darknet_diaries_llm/download_transcripts.py

20 lines
551 B
Python
Raw Normal View History

2023-10-06 21:35:53 +02:00
import requests
from bs4 import BeautifulSoup
for i in range(1, 139):
url = f"https://darknetdiaries.com/transcript/{i}"
r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')
pre_section = soup.find('pre')
title_section = soup.find('h1')
2023-10-06 21:35:53 +02:00
if pre_section:
text = pre_section.get_text()
title = title_section.get_text()
2023-10-06 21:35:53 +02:00
with open(f"data/episode_{i}.txt", "w") as f:
f.write(
f"Darknet Diaries - {title}\n" +
text
)
print(title)