feat: ups deps, add dotenv

This commit is contained in:
Jérémi N ‘EndMove’ 2023-10-07 16:55:18 +02:00
parent 0730db5727
commit 6d7600e47e
Signed by: EndMove
GPG Key ID: 65C4A02E1F5371A4
4 changed files with 124 additions and 104 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
.env
/transcripts
/index
/.idea

View File

@ -5,17 +5,20 @@ import json
folder_path = "transcripts"
if __name__ == '__main__':
if not os.path.exists(folder_path):
os.makedirs(folder_path)
for i in range(1, 139):
try:
# fetch transcript
url = f"https://darknetdiaries.com/transcript/{i}"
r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')
transcript = soup.find('pre').get_text()
# fetch transcript metadata
url = f"https://api.darknetdiaries.com/{i}.json"
r = requests.get(url)
parsed_json = json.loads(r.text)
@ -23,8 +26,9 @@ for i in range(1, 139):
number = parsed_json["episode_number"]
downloads = parsed_json["total_downloads"]
with open(f"{folder_path}/episode_{number}.txt", "w") as f:
# write transcript
with open(f"{folder_path}/episode_{number}.txt", "w", encoding='utf-8') as f:
f.write(f"{title}\n{downloads}\n{transcript}")
print(f"{number} {title}")
except Exception:
print(f"Failed scraping episode {i}")
except Exception as err:
print(f"Failed scraping episode {i} : [{err}]")

10
main.py
View File

@ -6,15 +6,23 @@ from llama_index.llms import OpenAI, ChatMessage, MessageRole
from llama_index.prompts import ChatPromptTemplate
# from llama_index import set_global_handler
from llama_index.chat_engine.types import ChatMode
from dotenv import load_dotenv
import os
import re
# set_global_handler("simple")
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256)
# load .env
load_dotenv()
OPEN_API_KEY = os.getenv('OPEN_API_KEY')
# config llm context
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256, api_key="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")
service_context = ServiceContext.from_defaults(llm=llm)
set_global_service_context(service_context)
# TODO split in small functions
if __name__ == '__main__':
if not os.path.exists("./index/lock"):
documents = []
for filename in os.listdir("./transcripts"):

View File

@ -1,8 +1,15 @@
# =====================
# Required dependencies
# =====================
llama-index==0.8.*
beautifulsoup4==4.12.*
# general deps
requests~=2.31.0
llama-index~=0.8.40
beautifulsoup4~=4.12.2
python-dotenv~=1.0.0
# llama sub deps
transformers~=4.34.0
torch~=2.1.0
# =====================
# Development dependencies