EndMove/darknet_diaries_llm
1
0
Fork 0
forked from phito/darknet_diaries_llm
Code Pull Requests Activity

Use DnD API

Browse Source
This commit is contained in:
Romain Quinet 2023-10-07 08:32:07 +02:00
parent c47ff3d9ed
commit bf3fd878ac
1 changed files with 17 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
download_transcripts.py
View File

@ -1,6 +1,7 @@
import requests import requests
import os import os
from bs4 import BeautifulSoup from bs4 import BeautifulSoup
import json
folder_path = "transcripts" folder_path = "transcripts"
@ -8,21 +9,22 @@ if not os.path.exists(folder_path):
os.makedirs(folder_path) os.makedirs(folder_path)
for i in range(1, 139): for i in range(1, 139):
try:
url = f"https://darknetdiaries.com/transcript/{i}" url = f"https://darknetdiaries.com/transcript/{i}"
r = requests.get(url) r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
transcript = soup.find('pre').get_text() transcript = soup.find('pre').get_text()
title_section = soup.find('h1').get_text()
url = f"https://darknetdiaries.com/episode/{i}" url = f"https://api.darknetdiaries.com/{i}.json"
r = requests.get(url) r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser') parsed_json = json.loads(r.text)
downloads = soup.find(id='downloads').get_text() title = parsed_json["episode_name"]
number = parsed_json["episode_number"]
downloads = parsed_json["total_downloads"]
ep, title = title_section.split(":", 1) with open(f"{folder_path}/episode_{number}.txt", "w") as f:
ep = ep.strip()
title = title.strip()
with open(f"{folder_path}/episode_{i}.txt", "w") as f:
f.write(f"{title}\n{downloads}\n{transcript}") f.write(f"{title}\n{downloads}\n{transcript}")
print(f"{ep} {title}") print(f"{number} {title}")
except Exception:
print(f"Failed scraping episode {i}")