forked from phito/darknet_diaries_llm
25 lines
694 B
Python
25 lines
694 B
Python
import requests
|
|
import os
|
|
from bs4 import BeautifulSoup
|
|
|
|
folder_path = "transcripts"
|
|
|
|
if not os.path.exists(folder_path):
|
|
os.makedirs(folder_path)
|
|
|
|
for i in range(1, 139):
|
|
url = f"https://darknetdiaries.com/transcript/{i}"
|
|
r = requests.get(url)
|
|
soup = BeautifulSoup(r.text, 'html.parser')
|
|
pre_section = soup.find('pre')
|
|
title_section = soup.find('h1')
|
|
|
|
if pre_section:
|
|
transcript = pre_section.get_text()
|
|
ep, title = title_section.get_text().split(":", 1)
|
|
ep = ep.strip()
|
|
title = title.strip()
|
|
with open(f"{folder_path}/episode_{i}.txt", "w") as f:
|
|
f.write(f"{title}\n{transcript}")
|
|
print(f"{ep} {title}")
|