forked from phito/darknet_diaries_llm
feat: ups deps, add dotenv
This commit is contained in:
parent
0730db5727
commit
6d7600e47e
1
.gitignore
vendored
1
.gitignore
vendored
@ -1,3 +1,4 @@
|
|||||||
|
.env
|
||||||
/transcripts
|
/transcripts
|
||||||
/index
|
/index
|
||||||
/.idea
|
/.idea
|
||||||
|
@ -5,26 +5,30 @@ import json
|
|||||||
|
|
||||||
folder_path = "transcripts"
|
folder_path = "transcripts"
|
||||||
|
|
||||||
if not os.path.exists(folder_path):
|
if __name__ == '__main__':
|
||||||
os.makedirs(folder_path)
|
if not os.path.exists(folder_path):
|
||||||
|
os.makedirs(folder_path)
|
||||||
|
|
||||||
for i in range(1, 139):
|
for i in range(1, 139):
|
||||||
try:
|
try:
|
||||||
url = f"https://darknetdiaries.com/transcript/{i}"
|
# fetch transcript
|
||||||
r = requests.get(url)
|
url = f"https://darknetdiaries.com/transcript/{i}"
|
||||||
soup = BeautifulSoup(r.text, 'html.parser')
|
r = requests.get(url)
|
||||||
|
soup = BeautifulSoup(r.text, 'html.parser')
|
||||||
|
|
||||||
transcript = soup.find('pre').get_text()
|
transcript = soup.find('pre').get_text()
|
||||||
|
|
||||||
url = f"https://api.darknetdiaries.com/{i}.json"
|
# fetch transcript metadata
|
||||||
r = requests.get(url)
|
url = f"https://api.darknetdiaries.com/{i}.json"
|
||||||
parsed_json = json.loads(r.text)
|
r = requests.get(url)
|
||||||
title = parsed_json["episode_name"]
|
parsed_json = json.loads(r.text)
|
||||||
number = parsed_json["episode_number"]
|
title = parsed_json["episode_name"]
|
||||||
downloads = parsed_json["total_downloads"]
|
number = parsed_json["episode_number"]
|
||||||
|
downloads = parsed_json["total_downloads"]
|
||||||
|
|
||||||
with open(f"{folder_path}/episode_{number}.txt", "w") as f:
|
# write transcript
|
||||||
f.write(f"{title}\n{downloads}\n{transcript}")
|
with open(f"{folder_path}/episode_{number}.txt", "w", encoding='utf-8') as f:
|
||||||
print(f"{number} {title}")
|
f.write(f"{title}\n{downloads}\n{transcript}")
|
||||||
except Exception:
|
print(f"{number} {title}")
|
||||||
print(f"Failed scraping episode {i}")
|
except Exception as err:
|
||||||
|
print(f"Failed scraping episode {i} : [{err}]")
|
||||||
|
174
main.py
174
main.py
@ -6,100 +6,108 @@ from llama_index.llms import OpenAI, ChatMessage, MessageRole
|
|||||||
from llama_index.prompts import ChatPromptTemplate
|
from llama_index.prompts import ChatPromptTemplate
|
||||||
# from llama_index import set_global_handler
|
# from llama_index import set_global_handler
|
||||||
from llama_index.chat_engine.types import ChatMode
|
from llama_index.chat_engine.types import ChatMode
|
||||||
|
from dotenv import load_dotenv
|
||||||
import os
|
import os
|
||||||
import re
|
import re
|
||||||
|
|
||||||
# set_global_handler("simple")
|
# set_global_handler("simple")
|
||||||
|
|
||||||
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256)
|
# load .env
|
||||||
|
load_dotenv()
|
||||||
|
OPEN_API_KEY = os.getenv('OPEN_API_KEY')
|
||||||
|
|
||||||
|
# config llm context
|
||||||
|
llm = OpenAI(model="gpt-4", temperature=0, max_tokens=256, api_key="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")
|
||||||
service_context = ServiceContext.from_defaults(llm=llm)
|
service_context = ServiceContext.from_defaults(llm=llm)
|
||||||
set_global_service_context(service_context)
|
set_global_service_context(service_context)
|
||||||
|
|
||||||
if not os.path.exists("./index/lock"):
|
# TODO split in small functions
|
||||||
documents = []
|
if __name__ == '__main__':
|
||||||
for filename in os.listdir("./transcripts"):
|
if not os.path.exists("./index/lock"):
|
||||||
episode_number = re.search(r'\d+', filename).group()
|
documents = []
|
||||||
with open("./transcripts/" + filename, 'r') as f:
|
for filename in os.listdir("./transcripts"):
|
||||||
title = f.readline().strip()
|
episode_number = re.search(r'\d+', filename).group()
|
||||||
downloads = f.readline().strip()
|
with open("./transcripts/" + filename, 'r') as f:
|
||||||
content = f.read()
|
title = f.readline().strip()
|
||||||
document = Document(
|
downloads = f.readline().strip()
|
||||||
text=content,
|
content = f.read()
|
||||||
doc_id=filename,
|
document = Document(
|
||||||
metadata={
|
text=content,
|
||||||
"episode_number": episode_number,
|
doc_id=filename,
|
||||||
"episode_title": title,
|
metadata={
|
||||||
"episode_downloads": downloads,
|
"episode_number": episode_number,
|
||||||
"episode_url": f"https://darknetdiaries.com/episode/{episode_number}/"
|
"episode_title": title,
|
||||||
}
|
"episode_downloads": downloads,
|
||||||
)
|
"episode_url": f"https://darknetdiaries.com/episode/{episode_number}/"
|
||||||
documents.append(document)
|
}
|
||||||
|
)
|
||||||
|
documents.append(document)
|
||||||
|
|
||||||
parser = SimpleNodeParser.from_defaults()
|
parser = SimpleNodeParser.from_defaults()
|
||||||
nodes = parser.get_nodes_from_documents(documents)
|
nodes = parser.get_nodes_from_documents(documents)
|
||||||
|
|
||||||
index = VectorStoreIndex(nodes, show_progress=True)
|
index = VectorStoreIndex(nodes, show_progress=True)
|
||||||
index.storage_context.persist(persist_dir="./index")
|
index.storage_context.persist(persist_dir="./index")
|
||||||
open("./index/lock", 'a').close()
|
open("./index/lock", 'a').close()
|
||||||
else:
|
else:
|
||||||
print("Loading index...")
|
print("Loading index...")
|
||||||
storage_context = StorageContext.from_defaults(persist_dir="./index")
|
storage_context = StorageContext.from_defaults(persist_dir="./index")
|
||||||
index = load_index_from_storage(storage_context)
|
index = load_index_from_storage(storage_context)
|
||||||
|
|
||||||
chat_text_qa_msgs = [
|
chat_text_qa_msgs = [
|
||||||
ChatMessage(
|
ChatMessage(
|
||||||
role=MessageRole.SYSTEM,
|
role=MessageRole.SYSTEM,
|
||||||
content=(
|
content=(
|
||||||
"You have been trained on the Darknet Diaries podcast transcripts with data from october 6 2023."
|
"You have been trained on the Darknet Diaries podcast transcripts with data from october 6 2023."
|
||||||
"You are an expert about it and will answer as such. You know about every episode up to number 138."
|
"You are an expert about it and will answer as such. You know about every episode up to number 138."
|
||||||
"Always answer the question, even if the context isn't helpful."
|
"Always answer the question, even if the context isn't helpful."
|
||||||
"Mention the number and title of the episodes you are referring to."
|
"Mention the number and title of the episodes you are referring to."
|
||||||
)
|
)
|
||||||
),
|
|
||||||
ChatMessage(
|
|
||||||
role=MessageRole.USER,
|
|
||||||
content=(
|
|
||||||
"Context information is below.\n"
|
|
||||||
"---------------------\n"
|
|
||||||
"{context_str}\n"
|
|
||||||
"---------------------\n"
|
|
||||||
"Given the context information and not prior knowledge,"
|
|
||||||
"answer the question: {query_str}\n"
|
|
||||||
)
|
|
||||||
)
|
|
||||||
]
|
|
||||||
text_qa_template = ChatPromptTemplate(chat_text_qa_msgs)
|
|
||||||
|
|
||||||
chat_refine_msgs = [
|
|
||||||
ChatMessage(
|
|
||||||
role=MessageRole.SYSTEM,
|
|
||||||
content="Always answer the question, even if the context isn't helpful.",
|
|
||||||
),
|
|
||||||
ChatMessage(
|
|
||||||
role=MessageRole.USER,
|
|
||||||
content=(
|
|
||||||
"We have the opportunity to refine the original answer "
|
|
||||||
"(only if needed) with some more context below.\n"
|
|
||||||
"------------\n"
|
|
||||||
"{context_msg}\n"
|
|
||||||
"------------\n"
|
|
||||||
"Given the new context, refine the original answer to better "
|
|
||||||
"answer the question: {query_str}. "
|
|
||||||
"If the context isn't useful, output the original answer again.\n"
|
|
||||||
"Original Answer: {existing_answer}"
|
|
||||||
),
|
),
|
||||||
),
|
ChatMessage(
|
||||||
]
|
role=MessageRole.USER,
|
||||||
refine_template = ChatPromptTemplate(chat_refine_msgs)
|
content=(
|
||||||
|
"Context information is below.\n"
|
||||||
|
"---------------------\n"
|
||||||
|
"{context_str}\n"
|
||||||
|
"---------------------\n"
|
||||||
|
"Given the context information and not prior knowledge,"
|
||||||
|
"answer the question: {query_str}\n"
|
||||||
|
)
|
||||||
|
)
|
||||||
|
]
|
||||||
|
text_qa_template = ChatPromptTemplate(chat_text_qa_msgs)
|
||||||
|
|
||||||
chat_engine = index.as_chat_engine(
|
chat_refine_msgs = [
|
||||||
text_qa_template=text_qa_template,
|
ChatMessage(
|
||||||
refine_template=refine_template,
|
role=MessageRole.SYSTEM,
|
||||||
chat_mode=ChatMode.OPENAI
|
content="Always answer the question, even if the context isn't helpful.",
|
||||||
)
|
),
|
||||||
|
ChatMessage(
|
||||||
|
role=MessageRole.USER,
|
||||||
|
content=(
|
||||||
|
"We have the opportunity to refine the original answer "
|
||||||
|
"(only if needed) with some more context below.\n"
|
||||||
|
"------------\n"
|
||||||
|
"{context_msg}\n"
|
||||||
|
"------------\n"
|
||||||
|
"Given the new context, refine the original answer to better "
|
||||||
|
"answer the question: {query_str}. "
|
||||||
|
"If the context isn't useful, output the original answer again.\n"
|
||||||
|
"Original Answer: {existing_answer}"
|
||||||
|
),
|
||||||
|
),
|
||||||
|
]
|
||||||
|
refine_template = ChatPromptTemplate(chat_refine_msgs)
|
||||||
|
|
||||||
while True:
|
chat_engine = index.as_chat_engine(
|
||||||
try:
|
text_qa_template=text_qa_template,
|
||||||
chat_engine.chat_repl()
|
refine_template=refine_template,
|
||||||
except KeyboardInterrupt:
|
chat_mode=ChatMode.OPENAI
|
||||||
break
|
)
|
||||||
|
|
||||||
|
while True:
|
||||||
|
try:
|
||||||
|
chat_engine.chat_repl()
|
||||||
|
except KeyboardInterrupt:
|
||||||
|
break
|
||||||
|
@ -1,8 +1,15 @@
|
|||||||
# =====================
|
# =====================
|
||||||
# Required dependencies
|
# Required dependencies
|
||||||
# =====================
|
# =====================
|
||||||
llama-index==0.8.*
|
# general deps
|
||||||
beautifulsoup4==4.12.*
|
requests~=2.31.0
|
||||||
|
llama-index~=0.8.40
|
||||||
|
beautifulsoup4~=4.12.2
|
||||||
|
python-dotenv~=1.0.0
|
||||||
|
|
||||||
|
# llama sub deps
|
||||||
|
transformers~=4.34.0
|
||||||
|
torch~=2.1.0
|
||||||
|
|
||||||
# =====================
|
# =====================
|
||||||
# Development dependencies
|
# Development dependencies
|
||||||
|
Loading…
Reference in New Issue
Block a user