phito/darknet_diaries_llm
2
0
Fork 1
Code Issues Pull Requests 1 Packages Projects Releases Wiki Activity

feat: ups deps, add dotenv

Browse Source
This commit is contained in:
Jérémi N ‘EndMove’ 2023-10-07 16:55:18 +02:00
parent 0730db5727
commit 6d7600e47e
Signed by untrusted user: EndMove
GPG Key ID: 65C4A02E1F5371A4
4 changed files with 124 additions and 104 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.gitignore vendored
View File

@ -1,3 +1,4 @@
.env
/transcripts /transcripts
/index /index
/.idea /.idea

42
download_transcripts.py
View File

@ -5,26 +5,30 @@ import json
folder_path = "transcripts" folder_path = "transcripts"
if not os.path.exists(folder_path): if __name__ == '__main__':
os.makedirs(folder_path) if not os.path.exists(folder_path):
os.makedirs(folder_path)
for i in range(1, 139): for i in range(1, 139):
try: try:
url = f"https://darknetdiaries.com/transcript/{i}" # fetch transcript
r = requests.get(url) url = f"https://darknetdiaries.com/transcript/{i}"
soup = BeautifulSoup(r.text, 'html.parser') r = requests.get(url)
soup = BeautifulSoup(r.text, 'html.parser')
transcript = soup.find('pre').get_text() transcript = soup.find('pre').get_text()
url = f"https://api.darknetdiaries.com/{i}.json" # fetch transcript metadata
r = requests.get(url) url = f"https://api.darknetdiaries.com/{i}.json"
parsed_json = json.loads(r.text) r = requests.get(url)
title = parsed_json["episode_name"] parsed_json = json.loads(r.text)
number = parsed_json["episode_number"] title = parsed_json["episode_name"]
downloads = parsed_json["total_downloads"] number = parsed_json["episode_number"]
downloads = parsed_json["total_downloads"]
with open(f"{folder_path}/episode_{number}.txt", "w") as f: # write transcript
f.write(f"{title}\n{downloads}\n{transcript}") with open(f"{folder_path}/episode_{number}.txt", "w", encoding='utf-8') as f:
print(f"{number} {title}") f.write(f"{title}\n{downloads}\n{transcript}")
except Exception: print(f"{number} {title}")
print(f"Failed scraping episode {i}") except Exception as err:
print(f"Failed scraping episode {i} : [{err}]")

174
main.py
View File

@ -6,100 +6,108 @@ from llama_index.llms import OpenAI, ChatMessage, MessageRole
from llama_index.prompts import ChatPromptTemplate from llama_index.prompts import ChatPromptTemplate
# from llama_index import set_global_handler # from llama_index import set_global_handler
from llama_index.chat_engine.types import ChatMode from llama_index.chat_engine.types import ChatMode
from dotenv import load_dotenv
import os import os
import re import re
# set_global_handler("simple") # set_global_handler("simple")
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256) # load .env
load_dotenv()
OPEN_API_KEY = os.getenv('OPEN_API_KEY')
# config llm context
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256, api_key="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")
service_context = ServiceContext.from_defaults(llm=llm) service_context = ServiceContext.from_defaults(llm=llm)
set_global_service_context(service_context) set_global_service_context(service_context)
if not os.path.exists("./index/lock"): # TODO split in small functions
documents = [] if __name__ == '__main__':
for filename in os.listdir("./transcripts"): if not os.path.exists("./index/lock"):
episode_number = re.search(r'\d+', filename).group() documents = []
with open("./transcripts/" + filename, 'r') as f: for filename in os.listdir("./transcripts"):
title = f.readline().strip() episode_number = re.search(r'\d+', filename).group()
downloads = f.readline().strip() with open("./transcripts/" + filename, 'r') as f:
content = f.read() title = f.readline().strip()
document = Document( downloads = f.readline().strip()
text=content, content = f.read()
doc_id=filename, document = Document(
metadata={ text=content,
"episode_number": episode_number, doc_id=filename,
"episode_title": title, metadata={
"episode_downloads": downloads, "episode_number": episode_number,
"episode_url": f"https://darknetdiaries.com/episode/{episode_number}/" "episode_title": title,
} "episode_downloads": downloads,
) "episode_url": f"https://darknetdiaries.com/episode/{episode_number}/"
documents.append(document) }
)
documents.append(document)
parser = SimpleNodeParser.from_defaults() parser = SimpleNodeParser.from_defaults()
nodes = parser.get_nodes_from_documents(documents) nodes = parser.get_nodes_from_documents(documents)
index = VectorStoreIndex(nodes, show_progress=True) index = VectorStoreIndex(nodes, show_progress=True)
index.storage_context.persist(persist_dir="./index") index.storage_context.persist(persist_dir="./index")
open("./index/lock", 'a').close() open("./index/lock", 'a').close()
else: else:
print("Loading index...") print("Loading index...")
storage_context = StorageContext.from_defaults(persist_dir="./index") storage_context = StorageContext.from_defaults(persist_dir="./index")
index = load_index_from_storage(storage_context) index = load_index_from_storage(storage_context)
chat_text_qa_msgs = [ chat_text_qa_msgs = [
ChatMessage( ChatMessage(
role=MessageRole.SYSTEM, role=MessageRole.SYSTEM,
content=( content=(
"You have been trained on the Darknet Diaries podcast transcripts with data from october 6 2023." "You have been trained on the Darknet Diaries podcast transcripts with data from october 6 2023."
"You are an expert about it and will answer as such. You know about every episode up to number 138." "You are an expert about it and will answer as such. You know about every episode up to number 138."
"Always answer the question, even if the context isn't helpful." "Always answer the question, even if the context isn't helpful."
"Mention the number and title of the episodes you are referring to." "Mention the number and title of the episodes you are referring to."
) )
),
ChatMessage(
role=MessageRole.USER,
content=(
"Context information is below.\n"
"---------------------\n"
"{context_str}\n"
"---------------------\n"
"Given the context information and not prior knowledge,"
"answer the question: {query_str}\n"
)
)
]
text_qa_template = ChatPromptTemplate(chat_text_qa_msgs)
chat_refine_msgs = [
ChatMessage(
role=MessageRole.SYSTEM,
content="Always answer the question, even if the context isn't helpful.",
),
ChatMessage(
role=MessageRole.USER,
content=(
"We have the opportunity to refine the original answer "
"(only if needed) with some more context below.\n"
"------------\n"
"{context_msg}\n"
"------------\n"
"Given the new context, refine the original answer to better "
"answer the question: {query_str}. "
"If the context isn't useful, output the original answer again.\n"
"Original Answer: {existing_answer}"
), ),
), ChatMessage(
] role=MessageRole.USER,
refine_template = ChatPromptTemplate(chat_refine_msgs) content=(
"Context information is below.\n"
"---------------------\n"
"{context_str}\n"
"---------------------\n"
"Given the context information and not prior knowledge,"
"answer the question: {query_str}\n"
)
)
]
text_qa_template = ChatPromptTemplate(chat_text_qa_msgs)
chat_engine = index.as_chat_engine( chat_refine_msgs = [
text_qa_template=text_qa_template, ChatMessage(
refine_template=refine_template, role=MessageRole.SYSTEM,
chat_mode=ChatMode.OPENAI content="Always answer the question, even if the context isn't helpful.",
) ),
ChatMessage(
role=MessageRole.USER,
content=(
"We have the opportunity to refine the original answer "
"(only if needed) with some more context below.\n"
"------------\n"
"{context_msg}\n"
"------------\n"
"Given the new context, refine the original answer to better "
"answer the question: {query_str}. "
"If the context isn't useful, output the original answer again.\n"
"Original Answer: {existing_answer}"
),
),
]
refine_template = ChatPromptTemplate(chat_refine_msgs)
while True: chat_engine = index.as_chat_engine(
try: text_qa_template=text_qa_template,
chat_engine.chat_repl() refine_template=refine_template,
except KeyboardInterrupt: chat_mode=ChatMode.OPENAI
break )
while True:
try:
chat_engine.chat_repl()
except KeyboardInterrupt:
break

11
requirements.txt
View File

@ -1,8 +1,15 @@
# ===================== # =====================
# Required dependencies # Required dependencies
# ===================== # =====================
llama-index==0.8.* # general deps
beautifulsoup4==4.12.* requests~=2.31.0
llama-index~=0.8.40
beautifulsoup4~=4.12.2
python-dotenv~=1.0.0
# llama sub deps
transformers~=4.34.0
torch~=2.1.0
# ===================== # =====================
# Development dependencies # Development dependencies