darknet_diaries_llm/download_transcripts.py

import requests
from bs4 import BeautifulSoup

for i in range(1, 139):
    url = f"https://darknetdiaries.com/transcript/{i}"
    r = requests.get(url)
    soup = BeautifulSoup(r.text, 'html.parser')
    pre_section = soup.find('pre')
    title_section = soup.find('h1')

    if pre_section:
        text = pre_section.get_text()
        title = title_section.get_text()
        with open(f"data/episode_{i}.txt", "w") as f:
            f.write(
                f"Darknet Diaries - {title}\n" +
                text
            )
        print(title)