forked from phito/darknet_diaries_llm
Use DnD API
This commit is contained in:
parent
c47ff3d9ed
commit
bf3fd878ac
@ -1,6 +1,7 @@
|
||||
import requests
|
||||
import os
|
||||
from bs4 import BeautifulSoup
|
||||
import json
|
||||
|
||||
folder_path = "transcripts"
|
||||
|
||||
@ -8,21 +9,22 @@ if not os.path.exists(folder_path):
|
||||
os.makedirs(folder_path)
|
||||
|
||||
for i in range(1, 139):
|
||||
url = f"https://darknetdiaries.com/transcript/{i}"
|
||||
r = requests.get(url)
|
||||
soup = BeautifulSoup(r.text, 'html.parser')
|
||||
try:
|
||||
url = f"https://darknetdiaries.com/transcript/{i}"
|
||||
r = requests.get(url)
|
||||
soup = BeautifulSoup(r.text, 'html.parser')
|
||||
|
||||
transcript = soup.find('pre').get_text()
|
||||
title_section = soup.find('h1').get_text()
|
||||
transcript = soup.find('pre').get_text()
|
||||
|
||||
url = f"https://darknetdiaries.com/episode/{i}"
|
||||
r = requests.get(url)
|
||||
soup = BeautifulSoup(r.text, 'html.parser')
|
||||
downloads = soup.find(id='downloads').get_text()
|
||||
url = f"https://api.darknetdiaries.com/{i}.json"
|
||||
r = requests.get(url)
|
||||
parsed_json = json.loads(r.text)
|
||||
title = parsed_json["episode_name"]
|
||||
number = parsed_json["episode_number"]
|
||||
downloads = parsed_json["total_downloads"]
|
||||
|
||||
ep, title = title_section.split(":", 1)
|
||||
ep = ep.strip()
|
||||
title = title.strip()
|
||||
with open(f"{folder_path}/episode_{i}.txt", "w") as f:
|
||||
f.write(f"{title}\n{downloads}\n{transcript}")
|
||||
print(f"{ep} {title}")
|
||||
with open(f"{folder_path}/episode_{number}.txt", "w") as f:
|
||||
f.write(f"{title}\n{downloads}\n{transcript}")
|
||||
print(f"{number} {title}")
|
||||
except Exception:
|
||||
print(f"Failed scraping episode {i}")
|
||||
|
Loading…
Reference in New Issue
Block a user