forked from phito/darknet_diaries_llm
feat: ups deps, add dotenv
This commit is contained in:
parent
0730db5727
commit
6d7600e47e
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,3 +1,4 @@
|
||||
.env
|
||||
/transcripts
|
||||
/index
|
||||
/.idea
|
||||
|
@ -5,17 +5,20 @@ import json
|
||||
|
||||
folder_path = "transcripts"
|
||||
|
||||
if __name__ == '__main__':
|
||||
if not os.path.exists(folder_path):
|
||||
os.makedirs(folder_path)
|
||||
|
||||
for i in range(1, 139):
|
||||
try:
|
||||
# fetch transcript
|
||||
url = f"https://darknetdiaries.com/transcript/{i}"
|
||||
r = requests.get(url)
|
||||
soup = BeautifulSoup(r.text, 'html.parser')
|
||||
|
||||
transcript = soup.find('pre').get_text()
|
||||
|
||||
# fetch transcript metadata
|
||||
url = f"https://api.darknetdiaries.com/{i}.json"
|
||||
r = requests.get(url)
|
||||
parsed_json = json.loads(r.text)
|
||||
@ -23,8 +26,9 @@ for i in range(1, 139):
|
||||
number = parsed_json["episode_number"]
|
||||
downloads = parsed_json["total_downloads"]
|
||||
|
||||
with open(f"{folder_path}/episode_{number}.txt", "w") as f:
|
||||
# write transcript
|
||||
with open(f"{folder_path}/episode_{number}.txt", "w", encoding='utf-8') as f:
|
||||
f.write(f"{title}\n{downloads}\n{transcript}")
|
||||
print(f"{number} {title}")
|
||||
except Exception:
|
||||
print(f"Failed scraping episode {i}")
|
||||
except Exception as err:
|
||||
print(f"Failed scraping episode {i} : [{err}]")
|
||||
|
10
main.py
10
main.py
@ -6,15 +6,23 @@ from llama_index.llms import OpenAI, ChatMessage, MessageRole
|
||||
from llama_index.prompts import ChatPromptTemplate
|
||||
# from llama_index import set_global_handler
|
||||
from llama_index.chat_engine.types import ChatMode
|
||||
from dotenv import load_dotenv
|
||||
import os
|
||||
import re
|
||||
|
||||
# set_global_handler("simple")
|
||||
|
||||
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256)
|
||||
# load .env
|
||||
load_dotenv()
|
||||
OPEN_API_KEY = os.getenv('OPEN_API_KEY')
|
||||
|
||||
# config llm context
|
||||
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256, api_key="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")
|
||||
service_context = ServiceContext.from_defaults(llm=llm)
|
||||
set_global_service_context(service_context)
|
||||
|
||||
# TODO split in small functions
|
||||
if __name__ == '__main__':
|
||||
if not os.path.exists("./index/lock"):
|
||||
documents = []
|
||||
for filename in os.listdir("./transcripts"):
|
||||
|
@ -1,8 +1,15 @@
|
||||
# =====================
|
||||
# Required dependencies
|
||||
# =====================
|
||||
llama-index==0.8.*
|
||||
beautifulsoup4==4.12.*
|
||||
# general deps
|
||||
requests~=2.31.0
|
||||
llama-index~=0.8.40
|
||||
beautifulsoup4~=4.12.2
|
||||
python-dotenv~=1.0.0
|
||||
|
||||
# llama sub deps
|
||||
transformers~=4.34.0
|
||||
torch~=2.1.0
|
||||
|
||||
# =====================
|
||||
# Development dependencies
|
||||
|
Loading…
Reference in New Issue
Block a user