feat: ups deps, add dotenv

.gitignore

@@ -1,3 +1,4 @@
+.env
 /transcripts
 /index
 /.idea

download_transcripts.py

@@ -5,17 +5,20 @@ import json
 
 folder_path = "transcripts"
 
-if not os.path.exists(folder_path):
+if __name__ == '__main__':
+    if not os.path.exists(folder_path):
         os.makedirs(folder_path)
 
-for i in range(1, 139):
+    for i in range(1, 139):
         try:
+            # fetch transcript
             url = f"https://darknetdiaries.com/transcript/{i}"
             r = requests.get(url)
             soup = BeautifulSoup(r.text, 'html.parser')
 
             transcript = soup.find('pre').get_text()
 
+            # fetch transcript metadata
             url = f"https://api.darknetdiaries.com/{i}.json"
             r = requests.get(url)
             parsed_json = json.loads(r.text)
@@ -23,8 +26,9 @@ for i in range(1, 139):
             number = parsed_json["episode_number"]
             downloads = parsed_json["total_downloads"]
 
-        with open(f"{folder_path}/episode_{number}.txt", "w") as f:
+            # write transcript
+            with open(f"{folder_path}/episode_{number}.txt", "w", encoding='utf-8') as f:
                 f.write(f"{title}\n{downloads}\n{transcript}")
             print(f"{number} {title}")
-    except Exception:
+        except Exception as err:
-        print(f"Failed scraping episode {i}")
+            print(f"Failed scraping episode {i} : [{err}]")

main.py

@@ -6,16 +6,24 @@ from llama_index.llms import OpenAI, ChatMessage, MessageRole
 from llama_index.prompts import ChatPromptTemplate
 # from llama_index import set_global_handler
 from llama_index.chat_engine.types import ChatMode
+from dotenv import load_dotenv
 import os
 import re
 
 # set_global_handler("simple")
 
-llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256)
+# load .env
+load_dotenv()
+OPEN_API_KEY = os.getenv('OPEN_API_KEY')
+
+# config llm context
+llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256, api_key="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")
 service_context = ServiceContext.from_defaults(llm=llm)
 set_global_service_context(service_context)
 
-if not os.path.exists("./index/lock"):
+# TODO split in small functions
+if __name__ == '__main__':
+    if not os.path.exists("./index/lock"):
         documents = []
         for filename in os.listdir("./transcripts"):
             episode_number = re.search(r'\d+', filename).group()
@@ -41,12 +49,12 @@ if not os.path.exists("./index/lock"):
         index = VectorStoreIndex(nodes, show_progress=True)
         index.storage_context.persist(persist_dir="./index")
         open("./index/lock", 'a').close()
-else:
+    else:
         print("Loading index...")
         storage_context = StorageContext.from_defaults(persist_dir="./index")
         index = load_index_from_storage(storage_context)
 
-chat_text_qa_msgs = [
+    chat_text_qa_msgs = [
         ChatMessage(
             role=MessageRole.SYSTEM,
             content=(
@@ -67,10 +75,10 @@ chat_text_qa_msgs = [
                 "answer the question: {query_str}\n"
             )
         )
-]
+    ]
-text_qa_template = ChatPromptTemplate(chat_text_qa_msgs)
+    text_qa_template = ChatPromptTemplate(chat_text_qa_msgs)
 
-chat_refine_msgs = [
+    chat_refine_msgs = [
         ChatMessage(
             role=MessageRole.SYSTEM,
             content="Always answer the question, even if the context isn't helpful.",
@@ -89,16 +97,16 @@ chat_refine_msgs = [
                 "Original Answer: {existing_answer}"
             ),
         ),
-]
+    ]
-refine_template = ChatPromptTemplate(chat_refine_msgs)
+    refine_template = ChatPromptTemplate(chat_refine_msgs)
 
-chat_engine = index.as_chat_engine(
+    chat_engine = index.as_chat_engine(
         text_qa_template=text_qa_template,
         refine_template=refine_template,
         chat_mode=ChatMode.OPENAI
-)
+    )
 
-while True:
+    while True:
         try:
             chat_engine.chat_repl()
         except KeyboardInterrupt:

requirements.txt

@@ -1,8 +1,15 @@
 # =====================
 # Required dependencies
 # =====================
-llama-index==0.8.*
+# general deps
-beautifulsoup4==4.12.*
+requests~=2.31.0
+llama-index~=0.8.40
+beautifulsoup4~=4.12.2
+python-dotenv~=1.0.0
+
+# llama sub deps
+transformers~=4.34.0
+torch~=2.1.0
 
 # =====================
 # Development dependencies