Update main.py

main.py

@@ -14,21 +14,31 @@ HF_TOKEN = os.getenv("HF_TOKEN")
 if not HF_TOKEN:  
     raise ValueError("HF_TOKEN environment variable not set.")  
 
-repo_id = "meta-llama/Meta-Llama-3-8B-Instruct"  
+# repo_id = "meta-llama/Meta-Llama-3-8B-Instruct" 
+repo_id = "deepseek-ai/DeepSeek-R1-Distill-Llama-8B"
 llm_client = InferenceClient(  
     model=repo_id,  
     token=HF_TOKEN,  
 )  
 
-# Configure Llama index settings  
-Settings.llm = HuggingFaceInferenceAPI(  
-    model_name=repo_id,  
-    tokenizer_name=repo_id,  
-    context_window=3000,  
-    token=HF_TOKEN,  
-    max_new_tokens=512,  
-    generate_kwargs={"temperature": 0.1},  
-)  
+# # Configure Llama index settings  
+# Settings.llm = HuggingFaceInferenceAPI(  
+#     model_name=repo_id,  
+#     tokenizer_name=repo_id,  
+#     context_window=3000,  
+#     token=HF_TOKEN,  
+#     max_new_tokens=512,  
+#     generate_kwargs={"temperature": 0.1},  
+# )  
+# Configure Llama index settings with the new model
+Settings.llm = HuggingFaceInferenceAPI(
+    model_name=repo_id,
+    tokenizer_name=repo_id,  # Use the same tokenizer as the model
+    context_window=3000,
+    token=HF_TOKEN,
+    max_new_tokens=512,
+    generate_kwargs={"temperature": 0.1},
+)
 # Settings.embed_model = HuggingFaceEmbedding(  
 #     model_name="BAAI/bge-small-en-v1.5"  
 # )  
@@ -40,9 +50,12 @@ Settings.embed_model = HuggingFaceEmbedding(
     model_name="sentence-transformers/paraphrase-multilingual-mpnet-base-v2"
 )
 
+# # Configure tokenizer and model if required
+# tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base")
+# model = AutoModel.from_pretrained("xlm-roberta-base")
 # Configure tokenizer and model if required
-tokenizer = AutoTokenizer.from_pretrained("xlm-roberta-base")
-model = AutoModel.from_pretrained("xlm-roberta-base")
+tokenizer = AutoTokenizer.from_pretrained(repo_id)  # Use the tokenizer from the new model
+model = AutoModel.from_pretrained(repo_id)  # Load the new model
 
 PERSIST_DIR = "db"  
 PDF_DIRECTORY = 'data'