update readme

README.MD

@@ -30,8 +30,7 @@ python3 main.py
 
 On the first run, it will generate the index. This can take a while, but it will be cached on disk for the next runs.
 
-You can then ask it any questions about Darknet Diaries! Currently, it does hallucinate a lot about episode numbers and titles. Other than that, it's pretty accurate!
-
+You can then ask it any questions about Darknet Diaries!
 ## Examples
 
 > What is the intro of the podcast?
@@ -67,4 +66,7 @@ You can then ask it any questions about Darknet Diaries! Currently, it does hall
 >>
 >>One of his memorable stories involves a physical penetration test where he had to break into a former employer's building. He used his knowledge of the building's layout and security mechanisms to gain access, even falling through a ceiling into a server room at one point.
 >>
->>The episode also covers a project where Jason was tasked with hacking into a large, worldwide bank. His job was to examine the bank's mobile app for any potential security vulnerabilities that could expose customer or sensitive information. The episode provides a detailed look into the world of penetration testing, highlighting the importance of robust security measures in both physical and digital spaces.
+>>The episode also covers a project where Jason was tasked with hacking into a large, worldwide bank. His job was to examine the bank's mobile app for any potential security vulnerabilities that could expose customer or sensitive information. The episode provides a detailed look into the world of penetration testing, highlighting the importance of robust security measures in both physical and digital spaces.
+> 
+> How many downloads does this episode have?
+>> Episode 130 of Darknet Diaries, titled "JASON'S PEN TEST", has 667,528 downloads.

download_transcripts.py

@@ -1,6 +1,7 @@
 import requests
 import os
 from bs4 import BeautifulSoup
+import json
 
 folder_path = "transcripts"
 
@@ -8,21 +9,22 @@ if not os.path.exists(folder_path):
     os.makedirs(folder_path)
 
 for i in range(1, 139):
-    url = f"https://darknetdiaries.com/transcript/{i}"
-    r = requests.get(url)
-    soup = BeautifulSoup(r.text, 'html.parser')
+    try:
+        url = f"https://darknetdiaries.com/transcript/{i}"
+        r = requests.get(url)
+        soup = BeautifulSoup(r.text, 'html.parser')
 
-    transcript = soup.find('pre').get_text()
-    title_section = soup.find('h1').get_text()
+        transcript = soup.find('pre').get_text()
 
-    url = f"https://darknetdiaries.com/episode/{i}"
-    r = requests.get(url)
-    soup = BeautifulSoup(r.text, 'html.parser')
-    downloads = soup.find(id='downloads').get_text()
+        url = f"https://api.darknetdiaries.com/{i}.json"
+        r = requests.get(url)
+        parsed_json = json.loads(r.text)
+        title = parsed_json["episode_name"]
+        number = parsed_json["episode_number"]
+        downloads = parsed_json["total_downloads"]
 
-    ep, title = title_section.split(":", 1)
-    ep = ep.strip()
-    title = title.strip()
-    with open(f"{folder_path}/episode_{i}.txt", "w") as f:
-        f.write(f"{title}\n{downloads}\n{transcript}")
-    print(f"{ep} {title}")
+        with open(f"{folder_path}/episode_{number}.txt", "w") as f:
+            f.write(f"{title}\n{downloads}\n{transcript}")
+        print(f"{number} {title}")
+    except Exception:
+        print(f"Failed scraping episode {i}")

main.py

@@ -4,9 +4,13 @@ from llama_index.node_parser import SimpleNodeParser
 from llama_index import VectorStoreIndex
 from llama_index.llms import OpenAI, ChatMessage, MessageRole
 from llama_index.prompts import ChatPromptTemplate
+from llama_index import set_global_handler
+from llama_index.chat_engine.types import ChatMode
 import os
 import re
 
+# set_global_handler("simple")
+
 llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256)
 service_context = ServiceContext.from_defaults(llm=llm)
 set_global_service_context(service_context)
@@ -90,16 +94,13 @@ refine_template = ChatPromptTemplate(chat_refine_msgs)
 
 chat_engine = index.as_chat_engine(
     text_qa_template=text_qa_template,
-    refine_template=refine_template
+    refine_template=refine_template,
+    chat_mode=ChatMode.OPENAI
 )
 
 while True:
     try:
-        user_prompt = input("Prompt: ")
-        streaming_response = chat_engine.stream_chat(user_prompt)
-        for token in streaming_response.response_gen:
-            print(token, end="")
-        print("\n")
+        chat_engine.chat_repl()
     except KeyboardInterrupt:
         break