download_transcripts.py

@@ -6,14 +6,8 @@ for i in range(1, 139):
     r = requests.get(url)
     soup = BeautifulSoup(r.text, 'html.parser')
     pre_section = soup.find('pre')
-    title_section = soup.find('h1')
 
     if pre_section:
         text = pre_section.get_text()
-        title = title_section.get_text()
         with open(f"data/episode_{i}.txt", "w") as f:
-            f.write(
-                f"Darknet Diaries - {title}\n" +
-                text
-            )
-        print(title)
+            f.write(text)

main.py

@@ -11,7 +11,6 @@ service_context = ServiceContext.from_defaults(llm=llm)
 set_global_service_context(service_context)
 
 if not os.path.exists("./index/lock"):
-    print("Generating index...")
     documents = []
     for filename in os.listdir("./data"):
         episode_number = re.search(r'\d+', filename).group()
@@ -32,17 +31,12 @@ if not os.path.exists("./index/lock"):
     index.storage_context.persist(persist_dir="./index")
     open("./index/lock", 'a').close()
 else:
-    print("Loading index...")
     storage_context = StorageContext.from_defaults(persist_dir="./index")
     index = load_index_from_storage(storage_context)
 
 template = (
-    "You have been trained on the Darknet Diaries podcast transcripts with data from october 6 2023."
-    "You are now an expert about it and will answer as such. You know about every episode up to number 138. \n"
-    "----------------\n"
-    "Here is the context: {context_str}"
-    "----------------\n"
-    "Please answer this question by referring to the podcast: {query_str}"
+    "You are now an expert on the Darknet Diaries podcast. \n"
+    "Please answer this question by referring to the podcast: {query_str}\n"
 )
 qa_template = PromptTemplate(template)
 query_engine = index.as_query_engine(text_qa_template=qa_template)