feat: ups deps, add dotenv

This commit is contained in:
Jérémi N ‘EndMove’ 2023-10-07 16:55:18 +02:00
parent 0730db5727
commit 6d7600e47e
Signed by untrusted user: EndMove
GPG Key ID: 65C4A02E1F5371A4
4 changed files with 124 additions and 104 deletions

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
.env
/transcripts /transcripts
/index /index
/.idea /.idea

View File

@ -5,17 +5,20 @@ import json
folder_path = "transcripts" folder_path = "transcripts"
if not os.path.exists(folder_path): if __name__ == '__main__':
if not os.path.exists(folder_path):
os.makedirs(folder_path) os.makedirs(folder_path)
for i in range(1, 139): for i in range(1, 139):
try: try:
# fetch transcript
url = f"https://darknetdiaries.com/transcript/{i}" url = f"https://darknetdiaries.com/transcript/{i}"
r = requests.get(url) r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser') soup = BeautifulSoup(r.text, 'html.parser')
transcript = soup.find('pre').get_text() transcript = soup.find('pre').get_text()
# fetch transcript metadata
url = f"https://api.darknetdiaries.com/{i}.json" url = f"https://api.darknetdiaries.com/{i}.json"
r = requests.get(url) r = requests.get(url)
parsed_json = json.loads(r.text) parsed_json = json.loads(r.text)
@ -23,8 +26,9 @@ for i in range(1, 139):
number = parsed_json["episode_number"] number = parsed_json["episode_number"]
downloads = parsed_json["total_downloads"] downloads = parsed_json["total_downloads"]
with open(f"{folder_path}/episode_{number}.txt", "w") as f: # write transcript
with open(f"{folder_path}/episode_{number}.txt", "w", encoding='utf-8') as f:
f.write(f"{title}\n{downloads}\n{transcript}") f.write(f"{title}\n{downloads}\n{transcript}")
print(f"{number} {title}") print(f"{number} {title}")
except Exception: except Exception as err:
print(f"Failed scraping episode {i}") print(f"Failed scraping episode {i} : [{err}]")

32
main.py
View File

@ -6,16 +6,24 @@ from llama_index.llms import OpenAI, ChatMessage, MessageRole
from llama_index.prompts import ChatPromptTemplate from llama_index.prompts import ChatPromptTemplate
# from llama_index import set_global_handler # from llama_index import set_global_handler
from llama_index.chat_engine.types import ChatMode from llama_index.chat_engine.types import ChatMode
from dotenv import load_dotenv
import os import os
import re import re
# set_global_handler("simple") # set_global_handler("simple")
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256) # load .env
load_dotenv()
OPEN_API_KEY = os.getenv('OPEN_API_KEY')
# config llm context
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256, api_key="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")
service_context = ServiceContext.from_defaults(llm=llm) service_context = ServiceContext.from_defaults(llm=llm)
set_global_service_context(service_context) set_global_service_context(service_context)
if not os.path.exists("./index/lock"): # TODO split in small functions
if __name__ == '__main__':
if not os.path.exists("./index/lock"):
documents = [] documents = []
for filename in os.listdir("./transcripts"): for filename in os.listdir("./transcripts"):
episode_number = re.search(r'\d+', filename).group() episode_number = re.search(r'\d+', filename).group()
@ -41,12 +49,12 @@ if not os.path.exists("./index/lock"):
index = VectorStoreIndex(nodes, show_progress=True) index = VectorStoreIndex(nodes, show_progress=True)
index.storage_context.persist(persist_dir="./index") index.storage_context.persist(persist_dir="./index")
open("./index/lock", 'a').close() open("./index/lock", 'a').close()
else: else:
print("Loading index...") print("Loading index...")
storage_context = StorageContext.from_defaults(persist_dir="./index") storage_context = StorageContext.from_defaults(persist_dir="./index")
index = load_index_from_storage(storage_context) index = load_index_from_storage(storage_context)
chat_text_qa_msgs = [ chat_text_qa_msgs = [
ChatMessage( ChatMessage(
role=MessageRole.SYSTEM, role=MessageRole.SYSTEM,
content=( content=(
@ -67,10 +75,10 @@ chat_text_qa_msgs = [
"answer the question: {query_str}\n" "answer the question: {query_str}\n"
) )
) )
] ]
text_qa_template = ChatPromptTemplate(chat_text_qa_msgs) text_qa_template = ChatPromptTemplate(chat_text_qa_msgs)
chat_refine_msgs = [ chat_refine_msgs = [
ChatMessage( ChatMessage(
role=MessageRole.SYSTEM, role=MessageRole.SYSTEM,
content="Always answer the question, even if the context isn't helpful.", content="Always answer the question, even if the context isn't helpful.",
@ -89,16 +97,16 @@ chat_refine_msgs = [
"Original Answer: {existing_answer}" "Original Answer: {existing_answer}"
), ),
), ),
] ]
refine_template = ChatPromptTemplate(chat_refine_msgs) refine_template = ChatPromptTemplate(chat_refine_msgs)
chat_engine = index.as_chat_engine( chat_engine = index.as_chat_engine(
text_qa_template=text_qa_template, text_qa_template=text_qa_template,
refine_template=refine_template, refine_template=refine_template,
chat_mode=ChatMode.OPENAI chat_mode=ChatMode.OPENAI
) )
while True: while True:
try: try:
chat_engine.chat_repl() chat_engine.chat_repl()
except KeyboardInterrupt: except KeyboardInterrupt:

View File

@ -1,8 +1,15 @@
# ===================== # =====================
# Required dependencies # Required dependencies
# ===================== # =====================
llama-index==0.8.* # general deps
beautifulsoup4==4.12.* requests~=2.31.0
llama-index~=0.8.40
beautifulsoup4~=4.12.2
python-dotenv~=1.0.0
# llama sub deps
transformers~=4.34.0
torch~=2.1.0
# ===================== # =====================
# Development dependencies # Development dependencies