.editorconfig

@@ -0,0 +1,14 @@
+root = true
+
+[*]
+charset = utf-8
+end_of_line = lf
+indent_size = 4
+indent_style = space
+insert_final_newline = true
+trim_trailing_whitespace = true
+max_line_length = 120
+
+[*.md]
+trim_trailing_whitespace = false
+max_line_length = 0

.gitignore

@@ -1,3 +1,5 @@
+.env
 /transcripts
 /index
 /.idea
+/venv

README.md

@@ -9,9 +9,15 @@ Well, let's ask our LLM:
 
 ## How to run
 ### Install dependencies
-I have no idea what the correct way to install dependencies with python is. Somehow install these libraries and their dependencies:
+It is recommended to use a python version greater than or equal to ``3.10.0``.
-- llama_index
+Another stuff recommended, is to create a venv or use an IDE that supports venv creation, so all dependencies are installed locally to the project and not globally. If not, you can use https://virtualenv.pypa.io/en/latest/ to artificially create isolated environments.
-- beautifulsoup4
+
+Install the dependencies required to run the project by running the following command at the project root :
+
+```shell
+pip install -r requirements.txt
+```
+
 ### Execution
 Download transcripts:
 ```shell
@@ -31,6 +37,7 @@ python3 main.py
 On the first run, it will generate the index. This can take a while, but it will be cached on disk for the next runs.
 
 You can then ask it any questions about Darknet Diaries!
+
 ## Examples
 
 > What is the intro of the podcast?

download_transcripts.py

@@ -5,17 +5,20 @@ import json
 
 folder_path = "transcripts"
 
+if __name__ == '__main__':
     if not os.path.exists(folder_path):
         os.makedirs(folder_path)
 
     for i in range(1, 139):
         try:
+            # fetch transcript
             url = f"https://darknetdiaries.com/transcript/{i}"
             r = requests.get(url)
             soup = BeautifulSoup(r.text, 'html.parser')
 
             transcript = soup.find('pre').get_text()
 
+            # fetch transcript metadata
             url = f"https://api.darknetdiaries.com/{i}.json"
             r = requests.get(url)
             parsed_json = json.loads(r.text)
@@ -23,8 +26,9 @@ for i in range(1, 139):
             number = parsed_json["episode_number"]
             downloads = parsed_json["total_downloads"]
 
-        with open(f"{folder_path}/episode_{number}.txt", "w") as f:
+            # write transcript
+            with open(f"{folder_path}/episode_{number}.txt", "w", encoding='utf-8') as f:
                 f.write(f"{title}\n{downloads}\n{transcript}")
             print(f"{number} {title}")
-    except Exception:
+        except Exception as err:
-        print(f"Failed scraping episode {i}")
+            print(f"Failed scraping episode {i} : [{err}]")

main.py

@@ -4,17 +4,24 @@ from llama_index.node_parser import SimpleNodeParser
 from llama_index import VectorStoreIndex
 from llama_index.llms import OpenAI, ChatMessage, MessageRole
 from llama_index.prompts import ChatPromptTemplate
-from llama_index import set_global_handler
+# from llama_index import set_global_handler
 from llama_index.chat_engine.types import ChatMode
+from dotenv import load_dotenv
 import os
 import re
 
 # set_global_handler("simple")
 
-llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256)
+# load .env
+load_dotenv()
+OPEN_API_KEY = os.getenv('OPEN_API_KEY')
+
+# config llm context
+llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256, api_key=OPEN_API_KEY)
 service_context = ServiceContext.from_defaults(llm=llm)
 set_global_service_context(service_context)
 
+if __name__ == '__main__':
     if not os.path.exists("./index/lock"):
         documents = []
         for filename in os.listdir("./transcripts"):
@@ -103,4 +110,3 @@ while True:
             chat_engine.chat_repl()
         except KeyboardInterrupt:
             break
-

requirements.txt

@@ -0,0 +1,16 @@
+# =====================
+# Required dependencies
+# =====================
+# general deps
+requests~=2.31.0
+llama-index~=0.8.40
+beautifulsoup4~=4.12.2
+python-dotenv~=1.0.0
+
+# llama sub deps
+transformers~=4.34.0
+torch~=2.1.0
+
+# =====================
+# Development dependencies
+# =====================