|
|
|
@ -106,8 +106,8 @@ class LLM: |
|
|
|
headers["X-Chosen-Backend"] = self.chosen_backend |
|
|
|
headers["X-Chosen-Backend"] = self.chosen_backend |
|
|
|
|
|
|
|
|
|
|
|
# If connected over VPN |
|
|
|
# If connected over VPN |
|
|
|
self.on_vpn = True |
|
|
|
self.on_vpn = on_vpn |
|
|
|
if on_vpn: |
|
|
|
if self.on_vpn: |
|
|
|
self.host_url = f"{os.getenv('LLM_URL')}:{os.getenv('LLM_PORT')}" |
|
|
|
self.host_url = f"{os.getenv('LLM_URL')}:{os.getenv('LLM_PORT')}" |
|
|
|
else: |
|
|
|
else: |
|
|
|
self.host_url = os.getenv("LLM_API_URL").rstrip("/api/chat/") |
|
|
|
self.host_url = os.getenv("LLM_API_URL").rstrip("/api/chat/") |
|
|
|
@ -151,7 +151,6 @@ class LLM: |
|
|
|
if model == "embeddings": |
|
|
|
if model == "embeddings": |
|
|
|
self.messages = [{"role": "user", "content": query}] |
|
|
|
self.messages = [{"role": "user", "content": query}] |
|
|
|
model = self.get_model("embeddings") |
|
|
|
model = self.get_model("embeddings") |
|
|
|
print_red(f"Using embeddings model: {model}") |
|
|
|
|
|
|
|
return model |
|
|
|
return model |
|
|
|
|
|
|
|
|
|
|
|
if messages: |
|
|
|
if messages: |
|
|
|
@ -525,7 +524,7 @@ class LLM: |
|
|
|
model = self._prepare_messages_and_model( |
|
|
|
model = self._prepare_messages_and_model( |
|
|
|
query, user_input, context, messages, images, model |
|
|
|
query, user_input, context, messages, images, model |
|
|
|
) |
|
|
|
) |
|
|
|
print_red(model) |
|
|
|
|
|
|
|
temperature = temperature if temperature else self.options["temperature"] |
|
|
|
temperature = temperature if temperature else self.options["temperature"] |
|
|
|
if think is None: |
|
|
|
if think is None: |
|
|
|
think = self.think |
|
|
|
think = self.think |
|
|
|
@ -533,7 +532,9 @@ class LLM: |
|
|
|
try: |
|
|
|
try: |
|
|
|
headers = self._build_headers(model) |
|
|
|
headers = self._build_headers(model) |
|
|
|
options = self._get_options(temperature) |
|
|
|
options = self._get_options(temperature) |
|
|
|
response = self._call_remote_api( |
|
|
|
|
|
|
|
|
|
|
|
# Call Ollama server |
|
|
|
|
|
|
|
response: ChatResponse = self._call_remote_api( |
|
|
|
model, tools, stream, options, format, headers, think=think |
|
|
|
model, tools, stream, options, format, headers, think=think |
|
|
|
) |
|
|
|
) |
|
|
|
|
|
|
|
|
|
|
|
@ -555,14 +556,7 @@ class LLM: |
|
|
|
if not self.chat: |
|
|
|
if not self.chat: |
|
|
|
self.messages = [self.messages[0]] |
|
|
|
self.messages = [self.messages[0]] |
|
|
|
|
|
|
|
|
|
|
|
#* Add attribute 'content' for compatibility with old version of _llm.py |
|
|
|
|
|
|
|
#* This is a copy of response.message.content |
|
|
|
|
|
|
|
#TODO Remove this when _llm.py is updated |
|
|
|
|
|
|
|
# Show a warning if using old version of _llm.py |
|
|
|
|
|
|
|
Warning = ("Please use reposen.message.content when ising _llm") |
|
|
|
Warning = ("Please use reposen.message.content when ising _llm") |
|
|
|
|
|
|
|
|
|
|
|
response.content = response.message |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
return response |
|
|
|
return response |
|
|
|
else: |
|
|
|
else: |
|
|
|
return "An error occurred." |
|
|
|
return "An error occurred." |
|
|
|
|