Add provider templates, scripts for syncing and embedding, and test cases

- Created a template for providers.yaml to define API providers and models.
- Added a new providers.yaml file with initial provider configurations.
- Implemented fix_things.py to update chunk documents in ArangoDB.
- Developed make_arango_embeddings.py to generate embeddings for talks and store them in ArangoDB.
- Introduced sync_talks.py to synchronize new speeches from riksdagen.se and process them.
- Added notes.md for documentation on riksdagsgruppen login details.
- Created test_make_arango_embeddings.py for integration testing of embedding generation.
- Implemented test_gpu.py to test image input handling with vLLM.

Makefile

@@ -1,4 +1,4 @@
-.PHONY: frontend backend reload nginx
+.PHONY: frontend backend reload nginx install-sync
 
 # install deps and build the React app
 frontend:
@@ -14,3 +14,11 @@ nginx:
 
 # build everything and reload nginx
 all: frontend nginx
+
+# install and enable the daily sync timer
+install-sync:
+	sudo cp etc/riksdagen-sync.service /etc/systemd/system/
+	sudo cp etc/riksdagen-sync.timer /etc/systemd/system/
+	sudo systemctl daemon-reload
+	sudo systemctl enable --now riksdagen-sync.timer
+	@echo "Timer installed. Check status with: systemctl list-timers | grep riksdagen"

_chromadb/chroma_client.py

@@ -13,12 +13,17 @@ from config import chromadb_path, embedding_model
 import re
 from typing import Dict, List, Any, Tuple, Optional
 from chromadb.utils.embedding_functions import OllamaEmbeddingFunction
+from chromadb.utils.embedding_functions import OllamaEmbeddingFunction
+
 
+from env_manager import set_env
+set_env()
 class ChromaClient:
     def __init__(self, path: str | None = chromadb_path):
         self.path: str = path
         self._client: ClientAPI = self._init_client()
-        self.embedding_function = OllamaEmbeddingFunction(model_name=embedding_model, url='192.168.1.10:33405')
+        embedding_url = os.getenv('LLM_EMBEDDINGS_PORT', '192.168.1.12:33405')
+        self.embedding_function = OllamaEmbeddingFunction(model_name=embedding_model, url=embedding_url)
 
     def _init_client(self) -> chromadb.PersistentClient:
         return chromadb.PersistentClient(path=self.path)
@@ -308,8 +313,8 @@ chroma_db = ChromaClient()
 if __name__ == "__main__":
    
     collection = chroma_db.get_collection(os.getenv("CHROMA_TALK_COLLECTION"))
-    print(collection.count())
+    query = 'betyg grundskola uppförande'
-    query = 'betyg grundskola'
+    print(f"Querying for: {query}")
     results = chroma_db.query_collection(
         collection=collection,
         query_texts=query,
@@ -317,15 +322,3 @@ if __name__ == "__main__":
     )
     for res in results:
         print(res['document'])
-    print('---')
-    col: Collection = chroma_db.get_collection(os.getenv("CHROMA_TALK_COLLECTION"))
-    print(col.get(limit=10))
-    results = col.query(query_texts=query, n_results=3)
-    for i in zip(
-        results['metadatas'][0],
-        results['documents'][0],
-        results['distances'][0],
-        results['ids'][0],
-    ):
-        print(i)
-

arango_client.py

@@ -1,7 +1,37 @@
-from _arango._arango import Arango
 import os
+from typing import List
+
+from ollama import Client
+from arango.collection import Collection
+
+from _arango._arango import Arango
+
+
+class CustomArango(Arango):
+    def __init__(self, db_name = 'riksdagen', user=None, password=None):
+        super().__init__(db_name, user, password)
+
 
-arango = Arango(
+    def make_embeddings(self, texts: List[str]) -> List[List[float]]:
+        """
+        Generate embeddings for a list of texts using Ollama.
+
+        Args:
+            texts (List[str]): List of text strings to embed.
+
+        Returns:
+            List[List[float]]: List of embedding vectors.
+        """
+        ollama_client = Client(host='192.168.1.12:33405')
+
+        embeddings = ollama_client.embed(
+            model="qwen3-embedding:latest",
+            input=texts,
+            dimensions=384
+        )
+        return embeddings.embeddings
+
+arango = CustomArango(
     db_name="riksdagen",
     user='riksdagen',
     password=os.getenv("ARANGO_PWD"),
@@ -9,4 +39,23 @@ arango = Arango(
 
 
 if __name__ == "__main__":
-    print(arango.db.collections())
+
+    embeddings = arango.make_embeddings(["Vilka åtgärder bör vidtas för att hantera klimatförändringar?"])
+    query = """LET query = @query_embedding
+
+FOR doc IN chunks
+  LET score = APPROX_NEAR_COSINE(doc.embedding, query)
+  SORT score DESC
+  LIMIT 5
+  RETURN {
+    _key: doc._key,
+    debate: doc.debate,
+    text: doc.text,
+    similarity: score
+  }
+  """
+    result = arango.db.aql.execute(query=query, bind_vars={"query_embedding": embeddings[0]})
+    for doc in result:
+        print(doc)
+        print('---')
+

backend/app.py

@@ -161,7 +161,8 @@ async def get_talk(talk_id: str) -> dict:
         "anforande_nummer",
         "replik",
         "url_session",
-        "url_audio"
+        "url_audio",
+        "summary"
     ]
     )
 

backend/services/chat.py

@@ -69,9 +69,20 @@ You can only request a tool use, not use it directly. After you request a tool,
 **When giving your final answer:**
 - Always start with a short summary of your findings, before any detailed analysis or tables.
 - Respond concisely, the user is not here for small talk. 
-- Make sure to include sources for your answer, but don't use the internal _id or chunk_index fields; instead, use date, title, etc. 
+- **IMPORTANT: Always format your answer using Markdown.** The frontend will convert it to HTML automatically.
-- When refering to a source, use foot notes like [1], [2], etc. at the end of the sentence where you mention it. *Remember to include a short bibliography at the end of your answer, listing all sources you used.*
+- **IMPORTANT: Use inline citation numbers for ALL source citations.** Use the format `[1]`, `[2]`, etc. directly after the statement that references the source.
-- Always format your final answer using Markdown (it will be translated to HTML by the frontend).
+- **CRITICAL: Citations must be plain square brackets with numbers inside: `[1]`, `[2]`, `[3]`. Do NOT use Markdown footnote syntax like `[^1]` or special Unicode brackets like `【1】`.**
+- **IMPORTANT: Always include a "Källor" (Sources) section at the end** with a numbered list matching your citations. Format each source as: `[1] Speaker name – Date – Brief context or quote`
+- Example of correct citation format:
+  ```
+  ROT-avdraget infördes 2009[1] och hade som syfte att minska svartarbete[2].
+
+  ## Källor
+  [1] Eva Andersson – 2009-01-15 – Debatt om ROT-avdrag
+  [2] Per Svensson – 2009-02-20 – Diskussion om byggbranschen
+  ```
+- Make sure citation numbers are sequential ([1], [2], [3]...) and that every citation has a matching entry in the Källor section.
+- Don't use internal _id or chunk_index fields in your answer; use human-readable information (speaker, date, topic).
 - Don't ever make up quotes or facts; if you don't have enough information, say that you don't know, or call another tool to find more information.
 - Answer in Swedish.
     """
@@ -198,10 +209,11 @@ You can only request a tool use, not use it directly. After you request a tool,
             # This avoids issues if there are multiple ChatCompletionMessage classes in the project
 
             # The following code should NOT be inside the if-block!
-            try:
+            # Use getattr so this doesn't raise AttributeError when the model
-                print_blue("Thinking:", response.reasoning_content)
+            # doesn't return a reasoning/thinking block (which is the normal case).
-            except Exception as e:
+            thinking = getattr(response, "reasoning_content", None)
-                print_red(f"[ChatService] Error printing thinking response: {e}")
+            if thinking:
+                print_blue("Thinking:", thinking)
             try:    
                 print_purple("Content:", response.content)
             except Exception as e:
@@ -209,11 +221,12 @@ You can only request a tool use, not use it directly. After you request a tool,
 
             tool_calls = getattr(response, "tool_calls", None)
             if tool_calls:
-                if response.reasoning_content:
+                reasoning_content_attr = getattr(response, "reasoning_content", None)
-                    if isinstance(response.reasoning_content, dict) and "content" in response.reasoning_content:
+                if reasoning_content_attr:
-                        reasoning_content = response.reasoning_content["content"]
+                    if isinstance(reasoning_content_attr, dict) and "content" in reasoning_content_attr:
+                        reasoning_content = reasoning_content_attr["content"]
                     else:
-                        reasoning_content = str(response.reasoning_content)
+                        reasoning_content = str(reasoning_content_attr)
                     current_messages.append(
                         {
                             "role": "assistant",
@@ -319,7 +332,7 @@ You can only request a tool use, not use it directly. After you request a tool,
                                 f"{tool_result_string[:12000]} (...) [truncated]"
                             )
 
-                    reminder = '\n\n**Remember that you can only use the information you get using the tools when giving your final answer. Do not make up any facts or quotes. If you do not have enough information, say that you do not know, or call another tool to find more information. Always give you final answer in Swedish.**'
+                    reminder = '\n\n**Remember:**\n- You can only use information from tool results when giving your final answer.\n- Do not make up facts or quotes.\n- If you lack information, say so or call another tool.\n- **Always use inline citations in the format [1], [2], [3] etc. Do NOT use [^1] or 【1】.**\n- **Always include a "Källor" section at the end with matching numbered sources.**\n- Always format your final answer in Markdown.\n- Always answer in Swedish.'
                     tool_message = {
                         "role": "tool",
                         "name": tool_name,
@@ -338,7 +351,11 @@ You can only request a tool use, not use it directly. After you request a tool,
                 continue
             elif response.content:
                 final_content = getattr(response, "content", "")
-                return final_content, collected_tables, active_focus_ids
+                final_message = FinalAnswer(
+                    final_answer=final_content,
+                    explanation="Model provided a direct answer without requiring additional tools."
+                )
+                return final_message, collected_tables, active_focus_ids
 
 
     def _get_tool_function(self, tool_name: str):

backend/services/llm_tools.py

@@ -8,22 +8,114 @@ from arango_client import arango
 from arango.exceptions import AQLQueryExecuteError
 from backend.services.search import (
     SearchService,
-)  # Import SearchService for use in the tool
+)
 from utils import detect_sql_syntax
 from _llm import LLM
 from pydantic import BaseModel, Field
 
-# * When to use an AQL tool vs vector/semantic search*
+
-# - Use AQL for exact predicates, structured filters, joins, grouping, aggregations,
+class HitDocument(BaseModel):
-#   date-range queries, or ArangoSearch indexed text search.
+    """
-#   Examples:
+    HitDocument is a Pydantic model that provides a normalized representation of a search hit across various tools, enabling consistent downstream handling.
-#     • Exact matches (by id, date, party, speaker).
+
-#     • Aggregations (counts, sums, min/max) and grouping (COLLECT).
+    Attributes:
-#     • Joins across collections with nested FOR.
+        id (Optional[str]): Fully qualified ArangoDB document identifier.
-#     • Range queries, pagination, sorted results and server-side window functions.
+        key (Optional[str]): Document key without collection prefix.
-# - Prefer vector/semantic search when you need fuzzy or semantic similarity
+        speaker (Optional[str]): Name of the speaker associated with the hit.
-#   (e.g., "find speeches similar in meaning to this paragraph"). Vector search is
+        party (Optional[str]): Party affiliation of the speaker.
-#   complementary to AQL, not a replacement for structured queries.
+        date (Optional[str]): ISO formatted document date (YYYY-MM-DD).
+        snippet (Optional[str]): Contextual snippet or highlight from the document.
+        text (Optional[str]): Full text of the document when available.
+        score (Optional[float]): Relevance score supplied by the executing tool.
+        metadata (Dict[str, Any]): Additional metadata specific to the originating tool that should be preserved.
+
+    Methods:
+        to_string() -> str:
+            Renders the hit as a human-readable string with uppercase labels, including all present fields and metadata.
+    """
+
+    """Normalized representation of a search hit across tools to enable consistent downstream handling."""
+    id: Optional[str] = Field(
+        default=None, description="Fully qualified ArangoDB document identifier."
+    )
+    key: Optional[str] = Field(
+        default=None, description="Document key without collection prefix."
+    )
+    speaker: Optional[str] = Field(
+        default=None, description="Name of the speaker associated with the hit."
+    )
+    party: Optional[str] = Field(
+        default=None, description="Party affiliation of the speaker."
+    )
+    date: Optional[str] = Field(
+        default=None, description="ISO formatted document date (YYYY-MM-DD)."
+    )
+    snippet: Optional[str] = Field(
+        default=None, description="Contextual snippet or highlight from the document."
+    )
+    text: Optional[str] = Field(
+        default=None, description="Full text of the document when available."
+    )
+    score: Optional[float] = Field(
+        default=None, description="Relevance score supplied by the executing tool."
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional metadata specific to the originating tool that should be preserved.",
+    )
+
+    def to_string(self, include_metadata: bool = True) -> str:
+        """
+        Render the object as a human-readable string with uppercase labels.
+
+        Args:
+            include_metadata (bool, optional): Whether to include metadata fields in the output. Defaults to True.
+
+        Returns:
+            str: A formatted string representation of the object, with each field and its value separated by double newlines, and field names in uppercase.
+        """
+        data: Dict[str, Any] = self.model_dump(exclude_none=True)
+        metadata: Dict[str, Any] = data.pop("metadata", {})
+        segments: List[str] = []
+        for field_name, field_value in data.items():
+            segments.append(f"{field_name.upper()}\n{field_value}")
+        for meta_key, meta_value in metadata.items():
+            segments.append(f"{meta_key.upper()}\n{meta_value}")
+        return "\n\n".join(segments)
+
+
+class HitsResponse(BaseModel):
+    """
+    HitsResponse is a Pydantic model that serves as a container for multiple HitDocument instances, providing utility methods for formatting and rendering the collection.
+
+    Attributes:
+        hits (List[HitDocument]): A list of collected search hits.
+
+    Methods:
+        to_string() -> str:
+            Returns a string representation of all hits, separated by a visual divider. If there are no hits, returns an empty string.
+    """
+
+    hits: List[HitDocument] = Field(
+        default_factory=list, description="Collected search hits."
+    )
+
+    def to_string(self, include_metadata=True) -> str:
+        """
+        Render all hits as a single string, separated by a visual divider.
+
+        Args:
+            include_metadata (bool, optional): Whether to include metadata in each hit's string representation. Defaults to True.
+
+        Returns:
+            str: A single string containing all hits, separated by "\n\n---\n\n". Returns an empty string if there are no hits.
+        """
+        """Render all hits as a single string separated by a visual divider."""
+        if not self.hits:
+            return ""
+        return "\n\n---\n\n".join(
+            hit.to_string(include_metadata=include_metadata) for hit in self.hits
+        )
 
 
 @register_tool()
@@ -54,12 +146,15 @@ def search_documents(query: str):
     class AQLResponseModel(BaseModel):
         query: str = Field(..., description="The generated AQL query string.")
         output_explanation: str = Field(
-            ..., description="A *very* short explanation of the expected output format, and how to interpret the results.",
+            ...,
-            examples=["Using COUNT INTO c, the result is a single integer count of matching documents.",]
+            description="A *very* short explanation of the expected output format, and how to interpret the results.",
+            examples=[
+                "Using COUNT INTO c, the result is a single integer count of matching documents.",
+            ],
         )
 
     tools = get_tools(specific_tools=["aql_query"])
-    aql_query_description = tools[0]['function']['description']
+    aql_query_description = tools[0]["function"]["description"]
 
     system_message = f"""You are an expert in converting natural language queries into AQL (ArangoDB Query Language) queries for the Riksdagen database. 
 The user will provide a query in natural language, and you must translate it into a valid AQL query that can be executed against the database.
@@ -127,7 +222,9 @@ Write AQL queries for reading data using only the allowed keywords and patterns
     query = response.content.query
     output_explanation = response.content.output_explanation
     result = aql_query(query)
-    if len(result) == 1: #TODO Is it a good idea to return as a single item if only one item?
+    if (
+        len(result) == 1
+    ):  # TODO Is it a good idea to return as a single item if only one item?
         result = result[0]
 
     result = f"The AQL used to answer your request was:\n```\n{query}\n```\n{output_explanation}\n\nThe result of the query is:\n{result}"
@@ -137,18 +234,18 @@ Write AQL queries for reading data using only the allowed keywords and patterns
 @register_tool()
 def aql_query(query: str) -> List[Dict[str, Any]]:
 
-
+    def remove_fields(
-    def remove_fields(doc: Dict[str, Any], fields_to_remove: List[str]) -> Dict[str, Any]:
+        doc: Dict[str, Any], fields_to_remove: List[str]
+    ) -> Dict[str, Any]:
         """Recursively remove specified fields from a document."""
         if not isinstance(doc, dict):
             return doc
-        
+
         for key in list(doc.keys()):
             if key in fields_to_remove:
                 del doc[key]
 
         return doc
-            
 
     """
     Execute a read-only AQL query against the Riksdag talks database.
@@ -257,7 +354,9 @@ def aql_query(query: str) -> List[Dict[str, Any]]:
     try:
         docs = []
         for doc in arango.execute_aql(query):
-            docs.append(remove_fields(doc, ['chunks'])) #TODO And other fields to remove?
+            docs.append(
+                remove_fields(doc, ["chunks"])
+            )  # TODO And other fields to remove?
         return docs
 
     except AQLQueryExecuteError as e:
@@ -312,7 +411,9 @@ def aql_query(query: str) -> List[Dict[str, Any]]:
         try:
             docs = []
             for doc in arango.execute_aql(query):
-                docs.append(remove_fields(doc, ['chunks'])) #TODO And other fields to remove?
+                docs.append(
+                    remove_fields(doc, ["chunks"])
+                )  # TODO And other fields to remove?
             return docs
         except AQLQueryExecuteError as e2:
             print_red(f"[Tools] Still got AQL execution error after rewrite: {str(e2)}")
@@ -324,150 +425,130 @@ def aql_query(query: str) -> List[Dict[str, Any]]:
                 return f"ERROR executing AQL query: {str(e)}.\nPlease see the aql_query tool documentation for correct usage and examples!"
     except Exception as e:
         import traceback
+
         tb = traceback.format_exc()
         print_red(f"[Tools] Unexpected error executing AQL query: {str(e)}\n{tb}")
         return f"ERROR executing AQL query: {str(e)}.\nPlease see the aql_query tool documentation for correct usage and examples!"
 
 
 @register_tool()
-def vector_search_talks(query: str, limit: int = 8) -> List[Dict[str, Any]]:
+def vector_search_talks(query: str, limit: int = 8) -> str:
     """
-    Använd det här verktyget för att göra en semantisk sökning bland anföranden i Riksdagen.
+    Semantic search among speeches in the Riksdagen database.
-    Använd när du vill:
-    - Hitta relevanta anföranden baserat på innebörden i en fråga eller ett ämne.
-    - Få sammanfattningar eller utdrag från anföranden som är relaterade till en specifik fråga.
-    - Söka tematiskt snarare än med exakta nyckelord.
-    När du genererar query-parametern, försök att formulera den som en naturlig språkfråga eller ett uttalande som fångar det du vill veta.
 
     Args:
-        query: The user's question.
+        query (str): The user's question.
-        limit: Number of hits to return. Default 8.
+        limit (int): Number of hits to return. Default 8.
 
     Returns:
-        List of speech snippets most relevant to the query.
+        str: Formatted string containing the top hits separated by dividers.
     """
     print_yellow(f"[Tools] vector_search_talks → query='{query}' (top_k={limit}).")
-    collection = chroma_db.get_collection(os.getenv("CHROMA_TALK_COLLECTION"))
-    results = collection.query(
-        query_texts=[query],
-        n_results=limit,
-    )
-
-    metadatas = results.get("metadatas") or []
-    documents = results.get("documents") or []
-    ids = results.get("ids") or []
-    distances = results.get("distances") or []
 
-    metadata_rows = metadatas[0] if metadatas else []
+    embeddings = arango.make_embeddings([query])[0]
-    document_rows = documents[0] if documents else []
+    query = """
-    id_rows = ids[0] if ids else []
+    LET query = @query_embedding
-    distance_rows = distances[0] if distances else []
+
-
+    FOR doc IN chunks
-    def _as_int(value: Any, default: int = -1) -> int:
+    LET score = APPROX_NEAR_COSINE(doc.embedding, query)
-        """
+    SORT score DESC
-        Normalize chunk indices returned by Chroma so downstream Pydantic validation succeeds.
+    LIMIT 5
-        """
+    RETURN {
-        if isinstance(value, bool):
+        text: doc.text,
-            return default
+        parent_id: doc.parent_id,
-        if isinstance(value, int):
+        collection: doc.collection,
-            return value
+        index: doc.index,
-        if isinstance(value, float) and value.is_integer():
+        score: score
-            return int(value)
+    }
-        if isinstance(value, str):
+    """
-            stripped = value.strip()
+    results = list(
-            if stripped.startswith("+"):
+        arango.execute_aql(
-                stripped = stripped[1:]
+            query=query, bind_vars={"query_embedding": embeddings}, batch_size=limit
-            if stripped.lstrip("-").isdigit():
+        )
-                return int(stripped)
-        return default
-
-    max_len = max(
-        len(metadata_rows),
-        len(document_rows),
-        len(id_rows),
-        len(distance_rows),
-        0,
     )
-
+    parent_ids = [doc["parent_id"] for doc in results]
-    hits: List[Dict[str, Any]] = []
+    parent_docs = fetch_documents(
-    for idx in range(max_len):
+        parent_ids, fields=["_id", "_key", "talare", "parti", "dok_datum", "chunks"]
-        metadata = metadata_rows[idx] if idx < len(metadata_rows) else {}
+    )
-        if not isinstance(metadata, dict):
+    hits: List[HitDocument] = []
-            metadata = {}
+    for n in range(len(results)):
-        _id = metadata.get("_id") or (id_rows[idx] if idx < len(id_rows) else None)
+        chunks_doc = results[n]
-        if not _id:
+        parent_doc = parent_docs[n]
-            continue
+        parent_chunks = parent_doc["chunks"]
-        chunk_index_raw = (
+        chunk_index = chunks_doc["index"]
-            metadata.get("chunk_index")
+
-            or metadata.get("index")
+        # Build snippet from neighboring chunks, extracting the 'text' field from each chunk dict
-            or metadata.get("chunkId")
+        snippet_segments: List[str] = []
+
+        def get_chunk_text(chunk: Any) -> str:
+            """Helper to extract text from a chunk dict or return the string itself."""
+            if isinstance(chunk, dict):
+                return chunk.get("text", "")
+            elif isinstance(chunk, str):
+                return chunk
+            return str(chunk)
+
+        if chunk_index > 0:
+            snippet_segments.append(get_chunk_text(parent_chunks[chunk_index - 1]))
+        snippet_segments.append(get_chunk_text(chunks_doc))  # current chunk
+        if chunk_index < len(parent_chunks) - 1:
+            snippet_segments.append(get_chunk_text(parent_chunks[chunk_index + 1]))
+
+        hits.append(
+            HitDocument(
+                id=parent_doc.get("_id"),
+                key=parent_doc.get("_key"),
+                speaker=parent_doc.get("talare"),
+                party=parent_doc.get("parti"),
+                date=parent_doc.get("dok_datum"),
+                snippet=" ".join(snippet_segments),
+                score=chunks_doc.get("score"),
+                metadata={
+                    "collection": chunks_doc.get("collection"),
+                    "parent_id": chunks_doc.get("parent_id"),
+                    "chunk_index": chunk_index,
+                },
+            )
         )
-        chunk_index = _as_int(chunk_index_raw)
+    return HitsResponse(hits=hits).to_string()
-        snippet_candidates: List[str] = []
-        for candidate in (
-            metadata.get("snippet"),
-            metadata.get("text"),
-            document_rows[idx] if idx < len(document_rows) else "",
-        ):
-            if isinstance(candidate, str) and candidate.strip():
-                snippet_candidates.append(candidate.strip())
-        snippet = snippet_candidates[0] if snippet_candidates else ""
-        hit = {
-            "_id": _id,
-            "_id": _id,
-            "chunk_index": chunk_index,
-            "heading": metadata.get("heading") or metadata.get("title") or metadata.get("talare"),
-            "snippet": snippet,
-            "debateurl": metadata.get("debateurl") or metadata.get("debate_url"),
-            "score": distance_rows[idx] if idx < len(distance_rows) else None,
-        }
-        hits.append(hit)
-    print_purple(f"[Tools] vector_search_talks assembled {len(hits)} hits.")
-    return hits
 
 
 @register_tool()
-def fetch_documents(_ids: list[str], collection: str = None, fields: dict = {}) -> list[Dict[str, Any]]:
+def fetch_documents(_ids: list[str], collection: str = "", fields: list = []) -> str:
-    """    
+    """
-    Fetches documents from the database by their IDs, with optional collection prefix and field filtering.
+    Fetches full documents by their _id from ArangoDB.
 
     Args:
-        _ids (list[str]): List of document IDs to fetch. If a single ID is provided, it will be converted to a list.
+        _ids: List of document IDs (e.g., ["talks/abc123", "talks/def456"])
-        collection (str, optional): Collection name to prefix to IDs if not already present. Defaults to None.
+        collection: Optional collection name (not used, kept for compatibility)
-        fields (dict, optional): Dictionary specifying which fields to include in the returned documents. If empty, all fields are returned. Defaults to {}.
+        fields: Optional list to specify which fields to return
 
-    Returns:
+    Note:
-        list[Dict[str, Any]]: List of documents fetched from the database. If 'fields' is specified, only those fields are included in each document.
+    - If `collection` is provided, it will be prepended to any IDs in `_ids` that do not already contain a collection prefix.
+    - If collection is `talks` and fields is empty, the `chunks` field will be removed from the returned documents to reduce payload size.
 
-    Raises:
+    Returns:
-        ValueError: If document IDs do not include the collection prefix and no collection is specified.
+        JSON string with document data or error message
-        """
+    """
+    # Check that collection is provided or items in _ids contain collection prefix
+    assert collection or all(
+        "/" in i for i in _ids
+    ), "Either collection must be provided or _ids must contain collection prefix."
 
+    if collection:
+        l = []
+        for i in _ids:
+            if "/" not in i:
+                l.append(f"{collection}/{i}")
+            else:
+                l.append(i)
 
-    if not isinstance(_ids, list):
+    query = """
-        if isinstance(_ids, str):
-            if '[' in _ids and ']' in _ids:
-                import json
-                try:
-                    _ids = json.loads(_ids)
-                except Exception as e:
-                    print_red(f"[Tools] Error parsing _ids as JSON list: {str(e)}. Treating as single ID.")
-                    _ids = [_ids]
-        _ids = [_ids]
-
-    _ids = [_id.replace('\\', "/") for _id in _ids]
-    if collection and '/' not in _ids[0]:
-        _ids = [f"{collection}/{_id.split('/')[-1]}" for _id in _ids]
-    elif '/' not in _ids[0]:
-        return f"ERROR FROM TOOL: When fetching documents by _id, you **must** include the collection prefix (e.g., 'talks/12345'). Or specify the collection parameter."
-
-    query = f"""
     FOR id IN @document_ids
         RETURN DOCUMENT(id)
     """
-    document_ids_string = f"""[{",".join(f'"{_id}"' for _id in _ids)}]"""
+
-    print_blue(f"[Tools] Fetch {query}, bind_vars={{'document_ids': {document_ids_string}}}")
+    docs = arango.execute_aql(query, bind_vars={"document_ids": _ids})
-    docs = arango.execute_aql(query, bind_vars={"document_ids": document_ids_string})
 
     if fields:
         l = []
@@ -478,7 +559,6 @@ def fetch_documents(_ids: list[str], collection: str = None, fields: dict = {})
     else:
         for _id in _ids:
             if _id.startswith("talks/"):
-                # If fetching a talk, also fetch its chunks
                 for doc in docs:
                     if "chunks" in doc:
                         del doc["chunks"]
@@ -575,7 +655,7 @@ def arango_search(
 
     **This tools has four special features/parameters:**
     1) `return_snippets=True` – If you want to get an overview of the results, use this parameter to get highlighted snippets instead of full documents. This is useful if you want to quickly see what the results are about, and then decide which _id:s to fetch in full.
-    2) `results_to_user=True` – If the user has asked for e.g. "a list of talks mentioning...", "I want to see...", "give me all speeches about..." – or in other ways indicates they want to see the actual results – use this parameter so the results are sent to the user as they are.
+    2) `results_to_user=True` – If the user has asked for e.g. "a list of talks mentioning...", "I want to see...", "give me all speeches about..." – or in other ways indicates they want to see the actual results – use this parameter so the results are sent to the user.
     3) `focus_ids` – If you want to do a search within the ID:s from the last search, set this parameter to True. This is only useful if you have done a previous where you've used `results_to_user=True`, and the user has then asked a follow-up question that requires a more specific search within the previous results.
     4) `intressent_ids` – If you want to filter the search by specific speaker IDs, use this parameter. It should be a list of speaker IDs (intressent_id).
 
@@ -643,12 +723,16 @@ def arango_search(
     focus_id_list: List[str] = []
     if focus_ids:
         if isinstance(focus_ids, list):
-            focus_id_list = [str(item) for item in focus_ids if isinstance(item, (str, int))]
+            focus_id_list = [
+                str(item) for item in focus_ids if isinstance(item, (str, int))
+            ]
         elif isinstance(focus_ids, str):
             try:
                 parsed = json.loads(focus_ids)
                 if isinstance(parsed, list):
-                    focus_id_list = [str(item) for item in parsed if isinstance(item, (str, int))]
+                    focus_id_list = [
+                        str(item) for item in parsed if isinstance(item, (str, int))
+                    ]
             except json.JSONDecodeError:
                 focus_id_list = [focus_ids]
         elif focus_ids is True:

backend/services/monitor_script.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""
+System Resource Monitor - Logs system stats to help diagnose SSH connectivity issues.
+
+This script monitors:
+- CPU usage
+- Memory usage
+- Disk usage
+- Network connectivity
+- SSH service status
+- System load
+- Active connections
+
+Run continuously to capture when the system becomes unreachable.
+"""
+
+import psutil
+import time
+import logging
+from datetime import datetime
+from pathlib import Path
+
+# Setup logging to file with rotation
+log_file = Path("/var/log/system_monitor.log")
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.FileHandler(log_file),
+        logging.StreamHandler()  # Also print to console
+    ]
+)
+
+def check_ssh_service() -> dict:
+    """
+    Check if SSH service is running.
+    
+    Returns:
+        dict: Service status information
+    """
+    try:
+        import subprocess
+        result = subprocess.run(
+            ['systemctl', 'is-active', 'ssh'],
+            capture_output=True,
+            text=True,
+            timeout=5
+        )
+        return {
+            'running': result.returncode == 0,
+            'status': result.stdout.strip()
+        }
+    except Exception as e:
+        return {'running': False, 'error': str(e)}
+
+def get_system_stats() -> dict:
+    """
+    Collect current system statistics.
+    
+    Returns:
+        dict: System statistics including CPU, memory, disk, network
+    """
+    # CPU usage
+    cpu_percent = psutil.cpu_percent(interval=1)
+    cpu_count = psutil.cpu_count()
+    
+    # Memory usage
+    memory = psutil.virtual_memory()
+    swap = psutil.swap_memory()
+    
+    # Disk usage
+    disk = psutil.disk_usage('/')
+    
+    # Network stats
+    net_io = psutil.net_io_counters()
+    
+    # System load (1, 5, 15 minute averages)
+    load_avg = psutil.getloadavg()
+    
+    # Number of connections
+    connections = len(psutil.net_connections())
+    
+    return {
+        'cpu_percent': cpu_percent,
+        'cpu_count': cpu_count,
+        'memory_percent': memory.percent,
+        'memory_available_gb': memory.available / (1024**3),
+        'swap_percent': swap.percent,
+        'disk_percent': disk.percent,
+        'disk_free_gb': disk.free / (1024**3),
+        'network_bytes_sent': net_io.bytes_sent,
+        'network_bytes_recv': net_io.bytes_recv,
+        'load_1min': load_avg[0],
+        'load_5min': load_avg[1],
+        'load_15min': load_avg[2],
+        'connections': connections
+    }
+
+def monitor_loop(interval_seconds: int = 60):
+    """
+    Main monitoring loop that logs system stats at regular intervals.
+    
+    Args:
+        interval_seconds: How often to log stats (default: 60 seconds)
+    """
+    logging.info("Starting system monitoring...")
+    
+    while True:
+        try:
+            stats = get_system_stats()
+            ssh_status = check_ssh_service()
+            
+            # Log current stats
+            log_message = (
+                f"CPU: {stats['cpu_percent']:.1f}% | "
+                f"MEM: {stats['memory_percent']:.1f}% ({stats['memory_available_gb']:.2f}GB free) | "
+                f"DISK: {stats['disk_percent']:.1f}% ({stats['disk_free_gb']:.2f}GB free) | "
+                f"LOAD: {stats['load_1min']:.2f} {stats['load_5min']:.2f} {stats['load_15min']:.2f} | "
+                f"CONN: {stats['connections']} | "
+                f"SSH: {ssh_status.get('status', 'unknown')}"
+            )
+            
+            # Warning thresholds
+            if stats['cpu_percent'] > 90:
+                logging.warning(f"HIGH CPU! {log_message}")
+            elif stats['memory_percent'] > 90:
+                logging.warning(f"HIGH MEMORY! {log_message}")
+            elif stats['disk_percent'] > 90:
+                logging.warning(f"HIGH DISK USAGE! {log_message}")
+            elif stats['load_1min'] > stats['cpu_count'] * 2:
+                logging.warning(f"HIGH LOAD! {log_message}")
+            elif not ssh_status.get('running'):
+                logging.error(f"SSH SERVICE DOWN! {log_message}")
+            else:
+                logging.info(log_message)
+            
+            time.sleep(interval_seconds)
+            
+        except Exception as e:
+            logging.error(f"Error in monitoring loop: {e}")
+            time.sleep(interval_seconds)
+
+if __name__ == "__main__":
+    monitor_loop(interval_seconds=60)  # Log every 60 seconds

config.py

@@ -6,7 +6,7 @@ embedding_model = os.getenv("LLM_MODEL_EMBEDDINGS")
 if embedding_model == "embeddinggemma":
     embedding_dimensions = 768
 else:
-    embedding_dimensions = None
+    embedding_dimensions = 384
 _llm_api_url = os.getenv("LLM_API_URL")
 llm_base_url = (_llm_api_url).rstrip("/")
 llm_api_key = os.getenv("LLM_API_KEY", os.getenv("LLM_API_PWD_LASSE", "not-set"))

etc/riksdagen-sync.service

@@ -0,0 +1,19 @@
+[Unit]
+Description=Riksdagen daily talk sync
+# Wait for network before starting
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+Type=oneshot
+User=lasse
+WorkingDirectory=/home/lasse/riksdagen
+# Loads ARANGO_PWD and other env vars from the project .env file
+EnvironmentFile=/home/lasse/riksdagen/.env
+ExecStart=/home/lasse/riksdagen/.venv/bin/python /home/lasse/riksdagen/scripts/sync_talks.py
+# Log stdout/stderr to the systemd journal (view with: journalctl -u riksdagen-sync)
+StandardOutput=journal
+StandardError=journal
+
+[Install]
+WantedBy=multi-user.target

etc/riksdagen-sync.timer

@@ -0,0 +1,11 @@
+[Unit]
+Description=Run riksdagen daily talk sync at 06:00
+
+[Timer]
+# Run every day at 06:00
+OnCalendar=*-*-* 06:00:00
+# If the server was off at 06:00, run the job as soon as it comes back up
+Persistent=true
+
+[Install]
+WantedBy=timers.target

frontend/src/App.tsx

@@ -256,7 +256,6 @@ function SearchView() {
 			const parsed = JSON.parse(stored) as {
 				query?: string;
 				filters?: SearchFilters;
-				results?: TalkHit[];
 				sortMode?: "relevance" | "date";
 				visibleCount?: number;
 				hasSearched?: boolean;
@@ -274,13 +273,22 @@ function SearchView() {
 					to_year: parsed.filters.to_year ?? undefined,
 				});
 			}
-			if (Array.isArray(parsed.results)) setResults(parsed.results);
 			if (parsed.sortMode === "date" || parsed.sortMode === "relevance") setSortMode(parsed.sortMode);
 			if (typeof parsed.visibleCount === "number") setVisibleCount(parsed.visibleCount);
 			if ("hasSearched" in parsed) setHasSearched(Boolean(parsed.hasSearched));
 			if ("lastError" in parsed) setLastError(parsed.lastError ?? null);
 			setSpeaker(parsed.speaker ?? null);
 			setSpeakerIds(Array.isArray(parsed.speakerIds) ? parsed.speakerIds : []);
+			// If we have a query and filters, re-run the search automatically
+			if (parsed.query && parsed.filters) {
+				const cleanQuery = stripMentions(parsed.query);
+				searchMutation.mutate({
+					q: cleanQuery,
+					...parsed.filters,
+					speaker_ids: parsed.speakerIds && parsed.speakerIds.length > 0 ? parsed.speakerIds : undefined,
+					include_snippets: true,
+				});
+			}
 		} catch (error) {
 			console.error("Failed to restore search state from session storage:", error);
 			window.sessionStorage.removeItem(SEARCH_STATE_KEY);
@@ -295,7 +303,6 @@ function SearchView() {
 		const persistableState = {
 			query,
 			filters,
-			results,
 			sortMode,
 			visibleCount,
 			hasSearched,
@@ -303,8 +310,9 @@ function SearchView() {
 			speaker,
 			speakerIds,
 		};
+		// Only store lightweight state (no results)
 		window.sessionStorage.setItem(SEARCH_STATE_KEY, JSON.stringify(persistableState));
-	}, [query, filters, results, sortMode, visibleCount, hasSearched, lastError, speaker, speakerIds, hasHydratedSearch]);
+	}, [query, filters, sortMode, visibleCount, hasSearched, lastError, speaker, speakerIds, hasHydratedSearch]);
 
 	useEffect(() => {
 		// Any filter change while the user is interacting should reset pagination back to the first page.

frontend/src/components/ChatPanel.tsx

@@ -297,13 +297,77 @@ export const ChatPanel = forwardRef<ChatPanelHandle, Props>(function ChatPanel(
         }
     };
 
+    /**
+     * Converts Markdown to HTML and replaces [1], [2], ... with <sup>1</sup> citations,
+     * but only outside of <a>, <code>, <pre>, <script>, <style>, and <sup> tags.
+     * This avoids breaking links, code, and already-superscripted numbers.
+     */
     const convertMarkdownToHtml = (markdown: string): string => {
-        /**
+        // Parse markdown to HTML
-         * Transform Markdown into safe HTML.
-         * RETURN_TRUSTED_TYPE=false guarantees a plain string for React.
-         */
         const rawHtml = marked.parse(markdown) as string;
-        const sanitized = DOMPurify.sanitize(rawHtml, { RETURN_TRUSTED_TYPE: false });
+
+        // Parse the HTML string into a DOM tree
+        const parser = new DOMParser();
+        const doc = parser.parseFromString(rawHtml, 'text/html');
+
+        // Define tags where we should NOT replace [n] with <sup>n</sup>
+        const SKIP_TAGS = new Set(['A', 'CODE', 'PRE', 'SCRIPT', 'STYLE', 'SUP']);
+
+        // Create a TreeWalker to traverse all text nodes in the body
+        const walker = doc.createTreeWalker(
+            doc.body,
+            NodeFilter.SHOW_TEXT,
+            {
+                acceptNode(node) {
+                    // Ignore empty or whitespace-only nodes
+                    if (!node.nodeValue || node.nodeValue.trim() === '') return NodeFilter.FILTER_REJECT;
+                    return NodeFilter.FILTER_ACCEPT;
+                }
+            } as any
+        );
+
+        let node: Node | null = walker.nextNode();
+        while (node) {
+            const parentEl = node.parentElement;
+            // Skip text nodes inside tags we don't want to touch
+            if (!parentEl || SKIP_TAGS.has(parentEl.tagName)) {
+                node = walker.nextNode();
+                continue;
+            }
+
+            // Replace [n] with <sup>n</sup> in this text node
+            const text = node.nodeValue!;
+            const parts: (Node)[] = [];
+            let lastIndex = 0;
+            const regex = /\[(\d+)\]/g;
+            let m: RegExpExecArray | null;
+            while ((m = regex.exec(text)) !== null) {
+                const idx = m.index;
+                // Add text before the match
+                if (idx > lastIndex) parts.push(document.createTextNode(text.slice(lastIndex, idx)));
+                // Add <sup>n</sup>
+                const sup = doc.createElement('sup');
+                sup.textContent = m[1];
+                parts.push(sup);
+                lastIndex = idx + m[0].length;
+            }
+            if (parts.length > 0) {
+                // Add any remaining text after the last match
+                if (lastIndex < text.length) parts.push(document.createTextNode(text.slice(lastIndex)));
+                // Replace the original text node with the new nodes
+                const parent = node.parentNode!;
+                for (const p of parts) parent.insertBefore(p, node);
+                parent.removeChild(node);
+            }
+
+            node = walker.nextNode();
+        }
+
+        // Serialize the DOM tree back to HTML
+        const htmlWithFootnotes = doc.body.innerHTML;
+
+        // Sanitize the HTML before returning
+        const sanitized = DOMPurify.sanitize(htmlWithFootnotes, { RETURN_TRUSTED_TYPE: false });
         return typeof sanitized === "string" ? sanitized : sanitized.toString();
     };
 
@@ -336,7 +400,7 @@ export const ChatPanel = forwardRef<ChatPanelHandle, Props>(function ChatPanel(
                                 type="button"
                                 className="chat-rail__item"
                                 data-active={turn.id === activeTurn?.id}
-                                data-distance={Math.min(index + 1, 4)} // Same cap for turns that are newer than the active one.
+                                data-distance={Math.min(index + 1, 4)} // Same cap for turns that are newer än the active one.
                                 onClick={() => setSelectedTurnId(turn.id)}
                                 title={turn.question}
                             >

frontend/src/components/TalkView.tsx

@@ -1,6 +1,8 @@
 import { useQuery } from "@tanstack/react-query";
 import { useParams, Link, useNavigate } from "react-router-dom";
 import { fetchTalk } from "../api";
+import { marked } from "marked";
+import DOMPurify from "dompurify";
 
 /**
  * TalkView component displays a single talk with full details.
@@ -51,8 +53,16 @@ export function TalkView() {
 		);
 	}
 
+	const convertMarkdownToHtml = (markdown: string): string => {
+		// Convert Markdown summaries to HTML while keeping the output safe to inject.
+		const rawHtml = marked.parse(markdown);
+		const sanitized = DOMPurify.sanitize(rawHtml);
+		return typeof sanitized === "string" ? sanitized : sanitized.toString();
+	};
+
 	// Fix image URLs from http to https
 	const imageUrl = talk.person?.bild_url_192?.replace('http://', 'https://');
+	const summaryHtml = talk.summary ? convertMarkdownToHtml(talk.summary) : null;
 	
 	const previousTalk = talk.navigation?.previous ?? null;
 	const nextTalk = talk.navigation?.next ?? null;
@@ -124,7 +134,7 @@ export function TalkView() {
 					<img 
 						src={imageUrl} 
 						alt={talk.talare}
-						className="talk-view__speaker-photo"
+						className="talk-view__speaker-photo talk-view__speaker-photo--enhanced"
 					/>
 				)}
 				<div className="talk-view__speaker-info">
@@ -172,6 +182,18 @@ export function TalkView() {
 				</dl>
 			</div>
 
+			{summaryHtml && (
+				<div className="panel talk-view__summary">
+					<div className="talk-view__summaryHeading">
+						<span role="img" aria-label="AI"></span> AI-genererad sammanfattning
+					</div>
+					<div
+						className="talk-view__summaryContent"
+						dangerouslySetInnerHTML={{ __html: summaryHtml }}
+					/>
+				</div>
+			)}
+
 			{/* Talk text */}
 			<div className="panel talk-view__text">
 				<div className="talk-view__content">

frontend/src/styles.css

@@ -1290,6 +1290,43 @@ textarea {
 	min-width: 120px; /* Ensures each cell has enough space for a button */
 }
 
+/* AI summary panel styles */
+.talk-view__summary {
+	background: linear-gradient(90deg, #f3f6ff 80%, #eaf1fb 100%);
+	border: 1.5px dashed #7ea2e6;
+	color: #223b57;
+	margin-bottom: 1.2rem;
+}
+
+.talk-view__summaryHeading {
+	font-size: 1.08rem;
+	font-weight: 600;
+	color: #3a4a6b;
+	display: flex;
+	align-items: center;
+	gap: 0.5em;
+	margin-bottom: 0.5em;
+	letter-spacing: 0.01em;
+}
+
+.talk-view__summaryContent {
+	font-size: 1.01rem;
+	color: #223b57;
+}
+
+.talk-view__summaryContent > :first-child {
+	margin-top: 0;
+}
+
+.talk-view__summary :last-child {
+	margin-bottom: 0;
+}
+
+/* This class increases contrast and brightness for speaker photos */
+.talk-view__speaker-photo--enhanced {
+  filter: contrast(1.3) brightness(1.1);
+}
+
 @media (max-width: 768px) {
 	.talk-view {
 		padding: 1.5rem 1rem;

home/Lasse/configs/mac_studion.conf

@@ -0,0 +1,10 @@
+[Interface]
+PrivateKey = yDRb0EYZkUZCuYax44lSBAP3vmN+mPdDQEh2hAQ10lY=
+Address = 10.156.168.2/24
+DNS = 1.1.1.1, 1.0.0.1
+
+[Peer]
+PublicKey = 6gwhWDypmpxrGaobEh8xZIXvRIKdp0pWH6YWZ9F8twY=
+PresharedKey = XAD5qpUMr0Ouz2azeXfH7J5tE3iSi5XJOdzdrUTSbRg=
+Endpoint = 98.128.172.165:51820
+AllowedIPs = 0.0.0.0/0, ::0/0

mcp_server/__init__.py

@@ -0,0 +1,6 @@
+"""
+Public entry points for the Riksdagen MCP server package.
+"""
+from .server import run
+
+__all__ = ("run",)

mcp_server/auth.py

@@ -0,0 +1,24 @@
+from __future__ import annotations
+
+import os
+import secrets
+
+
+def validate_token(provided_token: str) -> None:
+    """
+    Ensure the caller supplied the expected bearer token.
+
+    Args:
+        provided_token: Token received from the MCP client.
+
+    Raises:
+        RuntimeError: If the server token is not configured.
+        PermissionError: If the token is missing or incorrect.
+    """
+    expected_token = os.getenv("MCP_SERVER_TOKEN")
+    if not expected_token:
+        raise RuntimeError("MCP_SERVER_TOKEN environment variable must be set for authentication.")
+    if not provided_token:
+        raise PermissionError("Missing MCP access token.")
+    if not secrets.compare_digest(provided_token, expected_token):
+        raise PermissionError("Invalid MCP access token.")

mcp_server/check_cert.py

@@ -0,0 +1,130 @@
+import ssl
+import socket
+from datetime import datetime
+from typing import Any, Dict, List, Optional
+import argparse
+import pprint
+import sys  # added to detect whether --host was passed
+
+def fetch_certificate(host: str, port: int = 443, server_hostname: Optional[str] = None, timeout: float = 5.0) -> Dict[str, Any]:
+	"""
+	Fetch the TLS certificate from host:port. This function intentionally
+	uses a non-verifying SSL context to retrieve the certificate even if it
+	doesn't validate, so we can inspect its fields.
+
+	Parameters:
+	- host: TCP connect target (can be an IP or hostname)
+	- port: TCP port (default 443)
+	- server_hostname: SNI value to send. If None, server_hostname = host.
+	- timeout: socket connect timeout in seconds
+
+	Returns:
+	Dictionary with the peer certificate (as returned by SSLSocket.getpeercert()) and additional metadata.
+	"""
+	if server_hostname is None:
+		server_hostname = host
+
+	# Create an SSL context that does NOT verify so we can always fetch the cert.
+	context = ssl.create_default_context()
+	context.check_hostname = False
+	context.verify_mode = ssl.CERT_NONE
+
+	with socket.create_connection((host, port), timeout=timeout) as sock:
+		with context.wrap_socket(sock, server_hostname=server_hostname) as sslsock:
+			cert = sslsock.getpeercert()
+			peer_cipher = sslsock.cipher()
+			peertime = datetime.utcnow().isoformat() + "Z"
+
+	info: Dict[str, Any] = {"peer_certificate": cert, "cipher": peer_cipher, "fetched_at": peertime, "server_hostname_used": server_hostname}
+	return info
+
+def parse_san(cert: Dict[str, Any]) -> List[str]:
+	"""Return list of DNS names from subjectAltName (if any)."""
+	san = []
+	for typ, val in cert.get("subjectAltName", ()):
+		if typ.lower() == "dns":
+			san.append(val)
+	return san
+
+def format_subject(cert: Dict[str, Any]) -> str:
+	"""Return a short human-friendly subject string."""
+	subject = cert.get("subject", ())
+	parts = []
+	for rdn in subject:
+		for k, v in rdn:
+			parts.append(f"{k}={v}")
+	return ", ".join(parts)
+
+def check_hostname_match(cert: Dict[str, Any], hostname: str) -> bool:
+	"""
+	Check whether the certificate matches hostname using ssl.match_hostname.
+	Returns True if match, False otherwise.
+	"""
+	try:
+		ssl.match_hostname(cert, hostname)
+		return True
+	except Exception:
+		return False
+
+def print_report(host: str, port: int, server_hostname: Optional[str]) -> None:
+	"""Fetch certificate and print a readable report."""
+	info = fetch_certificate(host=host, port=port, server_hostname=server_hostname)
+	cert = info["peer_certificate"]
+
+	print(f"Connected target: {host}:{port}")
+	print(f"SNI sent: {info['server_hostname_used']}")
+	print(f"Cipher: {info['cipher']}")
+	print(f"Fetched at (UTC): {info['fetched_at']}")
+	print()
+
+	print("Subject:")
+	print(" ", format_subject(cert))
+	print()
+
+	print("Issuer:")
+	issuer = cert.get("issuer", ())
+	issuer_parts = []
+	for rdn in issuer:
+		for k, v in rdn:
+			issuer_parts.append(f"{k}={v}")
+	print(" ", ", ".join(issuer_parts))
+	print()
+
+	sans = parse_san(cert)
+	print("Subject Alternative Names (SANs):")
+	if sans:
+		for n in sans:
+			print("  -", n)
+	else:
+		print("  (none)")
+
+	not_before = cert.get("notBefore")
+	not_after = cert.get("notAfter")
+	print()
+	print("Validity:")
+	print("  notBefore:", not_before)
+	print("  notAfter: ", not_after)
+
+	match = check_hostname_match(cert, server_hostname or host)
+	print()
+	print(f"Hostname match for '{server_hostname or host}':", "YES" if match else "NO")
+
+	# For debugging show the full cert dict if requested
+	# pprint.pprint(cert)
+
+def main() -> None:
+	parser = argparse.ArgumentParser(description="Fetch and inspect TLS certificate from a host (SNI-aware).")
+	# make --host optional and default to api.rixdagen.se so running without args works
+	parser.add_argument("--host", "-H", required=False, default="api.rixdagen.se", help="Host or IP to connect to (TCP target). Defaults to api.rixdagen.se")
+	parser.add_argument("--port", "-p", type=int, default=443, help="Port to connect to (default 443).")
+	parser.add_argument("--sni", help="SNI hostname to send. If omitted, the --host value is used as SNI.")
+	args = parser.parse_args()
+
+	# Notify when using the default host for quick testing
+	if ("--host" not in sys.argv) and ("-H" not in sys.argv):
+		print("No --host provided: defaulting to api.rixdagen.se (you can override with --host or -H)")
+
+	print_report(host=args.host, port=args.port, server_hostname=args.sni)
+
+if __name__ == "__main__":
+	main()

mcp_server/server.py

@@ -0,0 +1,114 @@
+"""
+RiksdagenTools MCP server (HTTP only, compatible with current FastMCP version)
+"""
+from __future__ import annotations
+import asyncio
+import logging
+import os
+import inspect
+from typing import Any, Dict, List, Optional, Sequence
+from fastmcp import FastMCP
+from mcp_server.auth import validate_token
+from mcp_server import tools
+
+HOST = os.getenv("MCP_HOST", "127.0.0.1")
+PORT = int(os.getenv("MCP_PORT", "8010"))
+PATH = os.getenv("MCP_PATH", "/mcp")
+LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO").upper()
+
+logging.basicConfig(
+    level=LOG_LEVEL,
+    format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
+    datefmt="%Y-%m-%d %H:%M:%S",
+)
+log = logging.getLogger("mcp_server")
+
+app = FastMCP("RiksdagenTools")
+
+# --- tools unchanged ---
+@app.tool()
+async def search_documents(token: str, aql_query: str) -> Dict[str, Any]:
+    validate_token(token)
+    return await asyncio.to_thread(tools.search_documents, aql_query)
+
+@app.tool()
+async def aql_query(token: str, query: str) -> List[Dict[str, Any]]:
+    validate_token(token)
+    return await asyncio.to_thread(tools.run_aql_query, query)
+
+@app.tool()
+async def vector_search_talks(token: str, query: str, limit: int = 8) -> List[Dict[str, Any]]:
+    validate_token(token)
+    return await asyncio.to_thread(tools.vector_search, query, limit)
+
+@app.tool()
+async def fetch_documents(
+    token: str, document_ids: Sequence[str], fields: Optional[Sequence[str]] = None
+) -> List[Dict[str, Any]]:
+    validate_token(token)
+    return await asyncio.to_thread(
+        tools.fetch_documents, list(document_ids), list(fields) if fields else None
+    )
+
+@app.tool()
+async def arango_search(
+    token: str,
+    query: str,
+    limit: int = 20,
+    parties: Optional[Sequence[str]] = None,
+    people: Optional[Sequence[str]] = None,
+    from_year: Optional[int] = None,
+    to_year: Optional[int] = None,
+    return_snippets: bool = False,
+    focus_ids: Optional[Sequence[str]] = None,
+    speaker_ids: Optional[Sequence[str]] = None,
+) -> Dict[str, Any]:
+    validate_token(token)
+    return await asyncio.to_thread(
+        tools.arango_search,
+        query,
+        limit,
+        parties,
+        people,
+        from_year,
+        to_year,
+        return_snippets,
+        focus_ids,
+        speaker_ids,
+    )
+
+@app.tool()
+async def ping() -> str:
+    """
+    Lightweight test tool for connectivity checks.
+
+    Returns:
+        A short confirmation string ("ok"). This tool is intentionally
+        unauthenticated so it can be used to validate the transport/proxy
+        (e.g. nginx -> backend) without presenting credentials.
+    """
+    return "ok"
+
+# --- Entrypoint ---
+def run() -> None:
+    log.info(
+        "Starting RiksdagenTools MCP server (HTTP) on http://%s:%d%s",
+        HOST,
+        PORT,
+        PATH,
+    )
+    
+    try:
+        # Pass host, port, and path directly to run() method
+        app.run(
+            transport="streamable-http",
+            host=HOST,
+            port=PORT,
+            path=PATH,
+        )
+    except Exception:
+        log.exception("Unexpected error while running the MCP server.")
+        raise
+
+if __name__ == "__main__":
+    run()

mcp_server/test_mcp_client.py

@@ -0,0 +1,168 @@
+"""
+Test script for the RiksdagenTools MCP server (HTTP transport).
+This script connects to the MCP server via Streamable HTTP and tests your main tools.
+Ensure the MCP server is running and that the environment variable MCP_SERVER_TOKEN is set.
+Also adjust MCP_SERVER_URL if needed.
+"""
+
+import os
+import asyncio
+from typing import Any, Dict, List, Optional, Sequence
+
+from mcp.client.streamable_http import streamablehttp_client
+from mcp.client.session import ClientSession   # adjusted import per SDK version
+
+TOKEN: str = os.environ.get("MCP_SERVER_TOKEN", "2q89rwpfaiukdjshp298n3qw")
+SERVER_URL: str = os.environ.get("MCP_SERVER_URL", "https://api.rixdagen.se/mcp")  # use the public HTTPS endpoint by default so tests target the nginx proxy
+
+async def run_tests() -> None:
+    """
+    Attempt to connect to SERVER_URL and run the tool tests. On SSL certificate
+    verification failures, optionally retry against the backend IP if
+    MCP_FALLBACK_TO_IP=1 is set (or a custom MCP_FALLBACK_URL is provided).
+    """
+    async def run_with_url(url: str) -> None:
+        print(f"Connecting to server URL: {url}")
+        async with streamablehttp_client(
+            url=url,
+            headers={ "Authorization": f"Bearer {TOKEN}" }
+        ) as (read_stream, write_stream, get_session_id):
+            async with ClientSession(read_stream, write_stream) as session:
+                # initialize the session (if needed)
+                init_result = await session.initialize()
+                print("Initialized session:", init_result)
+
+                print("\nListing available tools...")
+                tools_info = await session.list_tools()
+                print("Tools:", [ tool.name for tool in tools_info.tools ])
+
+                # Test aql_query
+                print("\n== Testing aql_query ==")
+                result1 = await session.call_tool(
+                    "aql_query",
+                    arguments={
+                        "token": TOKEN,
+                        "query": "FOR doc IN talks LIMIT 2 RETURN { _id: doc._id, talare: doc.talare }"
+                    }
+                )
+                print("aql_query result:", result1)
+
+                # Test search_documents
+                print("\n== Testing search_documents ==")
+                result2 = await session.call_tool(
+                    "search_documents",
+                    arguments={
+                        "token": TOKEN,
+                        "aql_query": "FOR doc IN talks LIMIT 2 RETURN { _id: doc._id, talare: doc.talare }"
+                    }
+                )
+                print("search_documents result:", result2)
+
+                # Test vector_search_talks
+                print("\n== Testing vector_search_talks ==")
+                result3 = await session.call_tool(
+                    "vector_search_talks",
+                    arguments={
+                        "token": TOKEN,
+                        "query": "klimatförändringar",
+                        "limit": 2
+                    }
+                )
+                print("vector_search_talks result:", result3)
+
+                # Test fetch_documents
+                print("\n== Testing fetch_documents ==")
+                # try to pull out IDs from result3 if available
+                doc_ids: List[str]
+                maybe = result3
+                if hasattr(maybe, "output") and isinstance(maybe.output, list) and maybe.output:
+                    doc_ids = [ maybe.output[0].get("_id", "") ]
+                else:
+                    doc_ids = ["talks/1"]
+                result4 = await session.call_tool(
+                    "fetch_documents",
+                    arguments={
+                        "token": TOKEN,
+                        "document_ids": doc_ids
+                    }
+                )
+                print("fetch_documents result:", result4)
+
+                # Test arango_search
+                print("\n== Testing arango_search ==")
+                result5 = await session.call_tool(
+                    "arango_search",
+                    arguments={
+                        "token": TOKEN,
+                        "query": "klimat",
+                        "limit": 2
+                    }
+                )
+                print("arango_search result:", result5)
+
+    # try primary URL first
+    try:
+        await run_with_url(SERVER_URL)
+    except Exception as e:  # capture failures from streamablehttp_client / httpx
+        err_str = str(e).lower()
+        ssl_fail = "certificate_verify_failed" in err_str or "hostname mismatch" in err_str or "certificate verify failed" in err_str
+        gateway_fail = "502" in err_str or "bad gateway" in err_str or "502 bad gateway" in err_str
+
+        if gateway_fail:
+            print("Received 502 Bad Gateway from the proxy when connecting to the server URL.")
+            # If user explicitly set fallback env var, retry against backend IP or custom fallback
+            fallback_flag = os.environ.get("MCP_FALLBACK_TO_IP", "0").lower() in ("1", "true", "yes")
+            if fallback_flag:
+                fallback_url = os.environ.get("MCP_FALLBACK_URL", "http://127.0.0.1:8010/mcp")
+                print(f"Retrying with fallback URL (MCP_FALLBACK_URL or default backend): {fallback_url}")
+                await run_with_url(fallback_url)
+                return
+            print("")
+            print("Possible causes:")
+            print("- The proxy (nginx) couldn't reach the backend (backend down, wrong proxy_pass or path).")
+            print("- Proxy buffering or HTTP version issues interfering with streaming transport.")
+            print("")
+            print("Options:")
+            print("- Bypass the proxy and target the backend directly:")
+            print("    export MCP_SERVER_URL='http://127.0.0.1:8010/mcp'")
+            print("- Or enable automatic fallback to the backend (insecure) for testing:")
+            print("    export MCP_FALLBACK_TO_IP=1")
+            print("    # optionally override the fallback target")
+            print("    export MCP_FALLBACK_URL='http://127.0.0.1:8010/mcp'")
+            print("- Check the proxy's error log (e.g. /var/log/nginx/error.log) for upstream errors.")
+            print("")
+            # re-raise so caller still sees the error if they don't follow guidance
+            raise
+
+        if ssl_fail:
+            print("SSL certificate verification failed while connecting to the server URL.")
+            # If user explicitly set fallback env var, retry against backend IP or custom fallback
+            fallback_flag = os.environ.get("MCP_FALLBACK_TO_IP", "0").lower() in ("1", "true", "yes")
+            if fallback_flag:
+                fallback_url = os.environ.get("MCP_FALLBACK_URL", "http://192.168.1.10:8010/mcp")
+                print(f"Retrying with fallback URL (MCP_FALLBACK_URL or default backend IP): {fallback_url}")
+                await run_with_url(fallback_url)
+                return
+            # Otherwise give actionable guidance
+            print("")
+            print("Possible causes:")
+            print("- The TLS certificate served for api.rixdagen.se does not match that hostname on this machine.")
+            print("")
+            print("Options:")
+            print("- Set MCP_SERVER_URL to the backend HTTP address to bypass TLS: e.g.")
+            print("    export MCP_SERVER_URL='http://192.168.1.10:8010/mcp'")
+            print("- Or enable automatic fallback to the backend IP for testing (insecure):")
+            print("    export MCP_FALLBACK_TO_IP=1")
+            print("    # optionally override the fallback target")
+            print("    export MCP_FALLBACK_URL='http://192.168.1.10:8010/mcp'")
+            print("")
+            # re-raise so caller still sees the error if they don't follow guidance
+            raise
+        # Not an SSL or gateway failure: re-raise
+        raise
+
+def main() -> None:
+    asyncio.run(run_tests())
+
+if __name__ == "__main__":
+    main()

mcp_server/test_mcp_ping.py

@@ -0,0 +1,81 @@
+import os
+import sys
+import asyncio
+from typing import Tuple, Any
+
+from mcp.client.streamable_http import streamablehttp_client
+from mcp.client.session import ClientSession
+
+SERVER_URL: str = os.environ.get("MCP_SERVER_URL", "http://127.0.0.1:8010/mcp")
+TOKEN: str = os.environ.get("MCP_SERVER_TOKEN", "")
+
+
+async def _extract_ping_result(res: Any) -> Any:
+    """
+    Extract a sensible value from various CallToolResult shapes returned by the SDK.
+
+    Handles:
+    - objects with 'structuredContent' (dict) -> use 'result' or first value
+    - objects with 'output' attribute
+    - objects with 'content' list containing a TextContent with .text
+    - plain scalars
+    """
+    # structuredContent is common in newer SDK responses
+    if hasattr(res, "structuredContent") and isinstance(res.structuredContent, dict):
+        # prefer a 'result' key
+        if "result" in res.structuredContent:
+            return res.structuredContent["result"]
+        # fallback to any first value
+        for v in res.structuredContent.values():
+            return v
+
+    # older / alternative shape: an 'output' attribute
+    if hasattr(res, "output"):
+        return res.output
+
+    # textual content list (TextContent objects)
+    if hasattr(res, "content") and isinstance(res.content, list) and res.content:
+        first = res.content[0]
+        # Some SDK TextContent exposes 'text'
+        if hasattr(first, "text"):
+            return first.text
+        # fallback to stringifying the object
+        return str(first)
+
+    # final fallback: try direct indexing or string conversion
+    try:
+        return res["result"]
+    except Exception:
+        return str(res)
+
+
+async def check_ping(url: str, token: str) -> Tuple[bool, str]:
+    """
+    Connect to the MCP server at `url` and call the 'ping' tool.
+
+    Returns:
+        (ok, message) where ok is True if ping returned "ok", otherwise False.
+    """
+    headers = {"Authorization": f"Bearer {token}"} if token else {}
+    try:
+        async with streamablehttp_client(url=url, headers=headers) as (read_stream, write_stream, _):
+            async with ClientSession(read_stream, write_stream) as session:
+                await session.initialize()
+                res = await session.call_tool("ping", arguments={})
+                output = await _extract_ping_result(res)  # robust extractor
+                if output == "ok":
+                    return True, "ping -> ok"
+                return False, f"unexpected ping response: {output!r}"
+    except Exception as e:
+        return False, f"error connecting/calling ping: {e!r}"
+
+
+def main() -> None:
+    ok, msg = asyncio.run(check_ping(SERVER_URL, TOKEN))
+    print(msg)
+    if not ok:
+        sys.exit(1)
+
+
+if __name__ == "__main__":
+    main()

mcp_server/test_tools.py.py

@@ -0,0 +1,75 @@
+#!/usr/bin/env python3
+"""
+Quick test script for MCP server tools.
+Run:  python test_tools.py
+"""
+
+import asyncio
+import json
+from pprint import pprint
+
+from mcp_server import tools
+
+
+async def main():
+    print("🔍 Testing MCP server tools...\n")
+
+    # 1. Ping (simulated, since it's just "ok")
+    print("1️⃣  ping → ok")
+    print("   ✅", "ok\n")
+
+    # 2. Search documents with a trivial AQL (read-only)
+    try:
+        aql = "FOR d IN riksdagen LIMIT 1 RETURN d"
+        print("2️⃣  search_documents...")
+        result = await asyncio.to_thread(tools.search_documents, aql)
+        print("   ✅ rows:", result["row_count"])
+    except Exception as e:
+        print("   ❌ search_documents failed:", e)
+    print()
+
+    # 3. Run AQL query directly
+    try:
+        aql = "FOR d IN riksdagen LIMIT 1 RETURN d"
+        print("3️⃣  run_aql_query...")
+        rows = await asyncio.to_thread(tools.run_aql_query, aql)
+        print("   ✅ got", len(rows), "rows")
+    except Exception as e:
+        print("   ❌ run_aql_query failed:", e)
+    print()
+
+    # 4. Vector search (Chroma)
+    try:
+        print("4️⃣  vector_search_talks...")
+        hits = await asyncio.to_thread(tools.vector_search, "klimatpolitik", 3)
+        print(f"   ✅ got {len(hits)} hits")
+        if hits:
+            print("   →", hits[0])
+    except Exception as e:
+        print("   ❌ vector_search_talks failed:", e)
+    print()
+
+    # 5. Fetch documents
+    try:
+        print("5️⃣  fetch_documents (demo id)...")
+        # Adjust a known _id if needed; using a dummy for now
+        docs = await asyncio.to_thread(tools.fetch_documents, ["riksdagen/1"])
+        print("   ✅ got", len(docs), "docs")
+    except Exception as e:
+        print("   ❌ fetch_documents failed:", e)
+    print()
+
+    # 6. Arango search
+    try:
+        print("6️⃣  arango_search...")
+        result = await asyncio.to_thread(tools.arango_search, "budget", 3)
+        print("   ✅ got", len(result.get("results", [])), "hits")
+    except Exception as e:
+        print("   ❌ arango_search failed:", e)
+    print()
+
+    print("🏁 Done.\n")
+
+
+if __name__ == "__main__":
+    asyncio.run(main())

mcp_server/tools.py

@@ -0,0 +1,360 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Dict, Iterable, List, Optional, Sequence
+from pydantic import BaseModel, Field
+
+import env_manager
+
+env_manager.set_env()
+
+from arango.collection import Collection  # noqa: E402
+from arango_client import arango  # noqa: E402
+from backend.services.search import SearchService  # noqa: E402
+from _chromadb.chroma_client import chroma_db  # noqa: E402
+
+
+class HitDocument(BaseModel):
+    """
+    HitDocument is a Pydantic model that provides a normalized representation of a search hit across various tools, enabling consistent downstream handling.
+
+    Attributes:
+        id (Optional[str]): Fully qualified ArangoDB document identifier.
+        key (Optional[str]): Document key without collection prefix.
+        speaker (Optional[str]): Name of the speaker associated with the hit.
+        party (Optional[str]): Party affiliation of the speaker.
+        date (Optional[str]): ISO formatted document date (YYYY-MM-DD).
+        snippet (Optional[str]): Contextual snippet or highlight from the document.
+        text (Optional[str]): Full text of the document when available.
+        score (Optional[float]): Relevance score supplied by the executing tool.
+        metadata (Dict[str, Any]): Additional metadata specific to the originating tool that should be preserved.
+
+    Methods:
+        to_string() -> str:
+            Renders the hit as a human-readable string with uppercase labels, including all present fields and metadata.
+    """
+
+    """Normalized representation of a search hit across tools to enable consistent downstream handling."""
+    id: Optional[str] = Field(
+        default=None, description="Fully qualified ArangoDB document identifier."
+    )
+    key: Optional[str] = Field(
+        default=None, description="Document key without collection prefix."
+    )
+    speaker: Optional[str] = Field(
+        default=None, description="Name of the speaker associated with the hit."
+    )
+    party: Optional[str] = Field(
+        default=None, description="Party affiliation of the speaker."
+    )
+    date: Optional[str] = Field(
+        default=None, description="ISO formatted document date (YYYY-MM-DD)."
+    )
+    snippet: Optional[str] = Field(
+        default=None, description="Contextual snippet or highlight from the document."
+    )
+    text: Optional[str] = Field(
+        default=None, description="Full text of the document when available."
+    )
+    score: Optional[float] = Field(
+        default=None, description="Relevance score supplied by the executing tool."
+    )
+    metadata: Dict[str, Any] = Field(
+        default_factory=dict,
+        description="Additional metadata specific to the originating tool that should be preserved.",
+    )
+
+    def to_string(self, include_metadata: bool = True) -> str:
+        """
+        Render the object as a human-readable string with uppercase labels.
+
+        Args:
+            include_metadata (bool, optional): Whether to include metadata fields in the output. Defaults to True.
+
+        Returns:
+            str: A formatted string representation of the object, with each field and its value separated by double newlines, and field names in uppercase.
+        """
+        data: Dict[str, Any] = self.model_dump(exclude_none=True)
+        metadata: Dict[str, Any] = data.pop("metadata", {})
+        segments: List[str] = []
+        for field_name, field_value in data.items():
+            segments.append(f"{field_name.upper()}\n{field_value}")
+        for meta_key, meta_value in metadata.items():
+            segments.append(f"{meta_key.upper()}\n{meta_value}")
+        return "\n\n".join(segments)
+
+
+class HitsResponse(BaseModel):
+    """
+    HitsResponse is a Pydantic model that serves as a container for multiple HitDocument instances, providing utility methods for formatting and rendering the collection.
+
+    Attributes:
+        hits (List[HitDocument]): A list of collected search hits.
+
+    Methods:
+        to_string() -> str:
+            Returns a string representation of all hits, separated by a visual divider. If there are no hits, returns an empty string.
+    """
+
+    hits: List[HitDocument] = Field(
+        default_factory=list, description="Collected search hits."
+    )
+
+    def to_string(self, include_metadata=True) -> str:
+        """
+        Render all hits as a single string, separated by a visual divider.
+
+        Args:
+            include_metadata (bool, optional): Whether to include metadata in each hit's string representation. Defaults to True.
+
+        Returns:
+            str: A single string containing all hits, separated by "\n\n---\n\n". Returns an empty string if there are no hits.
+        """
+        """Render all hits as a single string separated by a visual divider."""
+        if not self.hits:
+            return ""
+        return "\n\n---\n\n".join(
+            hit.to_string(include_metadata=include_metadata) for hit in self.hits
+        )
+
+
+
+def ensure_read_only_aql(query: str) -> None:
+    """
+    Reject AQL statements that attempt to mutate data or omit a RETURN clause.
+
+    Args:
+        query: Raw AQL statement from the client.
+
+    Raises:
+        ValueError: If the query looks unsafe.
+    """
+    normalized = query.upper()
+    forbidden = (
+        "INSERT ",
+        "UPDATE ",
+        "UPSERT ",
+        "REMOVE ",
+        "REPLACE ",
+        "DELETE ",
+        "DROP ",
+        "TRUNCATE ",
+        "UPSERT ",
+        "MERGE ",
+    )
+    if any(keyword in normalized for keyword in forbidden):
+        raise ValueError("Only read-only AQL queries are allowed.")
+    if " RETURN " not in normalized and not normalized.strip().startswith("RETURN "):
+        raise ValueError("AQL queries must include a RETURN clause.")
+
+
+def strip_private_fields(document: Dict[str, Any]) -> Dict[str, Any]:
+    """
+    Remove large internal fields from a document dictionary.
+
+    Args:
+        document: Document returned by ArangoDB.
+
+    Returns:
+        Sanitized copy without chunk payloads.
+    """
+    if "chunks" in document:
+        del document["chunks"]
+    return document
+
+
+def search_documents(aql_query: str) -> Dict[str, Any]:
+    """
+    Execute a read-only AQL query and return the result set together with the query string.
+
+    Args:
+        aql_query: Read-only AQL statement supplied by the client.
+
+    Returns:
+        Dictionary containing the executed AQL string, row count, and result rows.
+    """
+    ensure_read_only_aql(aql_query)
+    rows = [strip_private_fields(doc) for doc in arango.execute_aql(aql_query)]
+    return {
+        "aql": aql_query,
+        "row_count": len(rows),
+        "rows": rows,
+    }
+
+
+def run_aql_query(aql_query: str) -> List[Dict[str, Any]]:
+    """
+    Execute a read-only AQL query and return the rows.
+
+    Args:
+        aql_query: Read-only AQL statement.
+
+    Returns:
+        List of result rows.
+    """
+    ensure_read_only_aql(aql_query)
+    return [strip_private_fields(doc) for doc in arango.execute_aql(aql_query)]
+
+
+def _get_existing_collection(name: str) -> Collection:
+    """
+    Fetch an existing Chroma collection without creating new data.
+
+    Args:
+        name: Collection identifier.
+
+    Returns:
+        The requested collection.
+
+    Raises:
+        ValueError: If the collection is absent.
+    """
+    available = {collection.name for collection in chroma_db._client.list_collections()}
+    if name not in available:
+        raise ValueError(f"Chroma collection '{name}' does not exist.")
+    return chroma_db._client.get_collection(name=name)
+
+
+def vector_search(query: str, limit: int) -> List[Dict[str, Any]]:
+    """
+    Perform semantic search against the pre-built Chroma collection.
+
+    Args:
+        query: Free-form search text.
+        limit: Maximum number of hits to return.
+
+    Returns:
+        List of hit dictionaries with metadata and scores.
+    """
+    collection_name = chroma_db.path.split("/")[-1]  # ...existing code...
+    chroma_collection = _get_existing_collection(collection_name)
+    results = chroma_collection.query(
+        query_texts=[query],
+        n_results=limit,
+    )
+    metadatas = results.get("metadatas") or []
+    documents = results.get("documents") or []
+    ids = results.get("ids") or []
+    distances = results.get("distances") or []
+
+    def as_int(value: Any, default: int = -1) -> int:
+        if isinstance(value, int):
+            return value
+        if isinstance(value, float) and value.is_integer():
+            return int(value)
+        if isinstance(value, str) and value.strip().lstrip("+-").isdigit():
+            return int(value)
+        return default
+
+    hits: List[Dict[str, Any]] = []
+    for index, metadata in enumerate(metadatas[0] if metadatas else []):
+        meta = metadata or {}
+        document = documents[0][index] if documents else ""
+        identifier = ids[0][index] if ids else ""
+        hit = {
+            "_id": meta.get("_id") or identifier,
+            "heading": meta.get("heading") or meta.get("title") or meta.get("talare"),
+            "snippet": meta.get("snippet") or meta.get("text") or document,
+            "debateurl": meta.get("debateurl") or meta.get("debate_url"),
+            "chunk_index": as_int(meta.get("chunk_index") or meta.get("index")),
+            "score": distances[0][index] if distances else None,
+        }
+        if hit["_id"]:
+            hits.append(hit)
+    return hits
+
+
+def fetch_documents(document_ids: Sequence[str], fields: Optional[Iterable[str]] = None) -> List[Dict[str, Any]]:
+    """
+    Pull full documents by _id while stripping heavy fields.
+
+    Args:
+        document_ids: Iterable with fully qualified Arango document ids.
+        fields: Optional subset of fields to return.
+
+    Returns:
+        List of sanitized documents.
+    """
+    ids = [doc_id.replace("\\", "/") for doc_id in document_ids]
+    query = """
+    FOR id IN @document_ids
+        RETURN DOCUMENT(id)
+    """
+    documents = arango.execute_aql(query, bind_vars={"document_ids": ids})
+    if fields:
+        return [{field: doc.get(field) for field in fields if field in doc} for doc in documents]
+    return [strip_private_fields(doc) for doc in documents]
+
+
+@dataclass
+class SearchPayload:
+    """
+    Lightweight container passed to SearchService.search.
+    """
+    q: str
+    parties: Optional[List[str]]
+    people: Optional[List[str]]
+    debates: Optional[List[str]]
+    from_year: Optional[int]
+    to_year: Optional[int]
+    limit: int
+    return_snippets: bool
+    focus_ids: Optional[List[str]]
+    speaker_ids: Optional[List[str]]
+    speaker: Optional[str] = None
+
+
+def arango_search(
+    query: str,
+    limit: int,
+    parties: Optional[Sequence[str]] = None,
+    people: Optional[Sequence[str]] = None,
+    from_year: Optional[int] = None,
+    to_year: Optional[int] = None,
+    return_snippets: bool = False,
+    focus_ids: Optional[Sequence[str]] = None,
+    speaker_ids: Optional[Sequence[str]] = None,
+) -> Dict[str, Any]:
+    """
+    Run an ArangoSearch query using the existing SearchService utilities.
+
+    Args:
+        query: Search expression (supports AND/OR/NOT and phrases).
+        limit: Maximum number of hits to return.
+        parties: Party filters.
+        people: Speaker name filters.
+        from_year: Start year filter.
+        to_year: End year filter.
+        return_snippets: Whether only snippets should be returned.
+        focus_ids: Optional list restricting the search scope.
+        speaker_ids: Optional list of speaker identifiers.
+
+    Returns:
+        Dictionary containing results, stats, limit flag, and focus_ids for follow-up queries.
+    """
+    payload = SearchPayload(
+        q=query,
+        parties=list(parties) if parties else None,
+        people=list(people) if people else None,
+        debates=None,
+        from_year=from_year,
+        to_year=to_year,
+        limit=limit,
+        return_snippets=return_snippets,
+        focus_ids=list(focus_ids) if focus_ids else None,
+        speaker_ids=list(speaker_ids) if speaker_ids else None,
+    )
+    service = SearchService()
+    results, stats, limit_reached = service.search(
+        payload=payload,
+        include_snippets=True,
+        return_snippets=return_snippets,
+        focus_ids=payload.focus_ids,
+    )
+    return {
+        "results": results,
+        "stats": stats,
+        "limit_reached": limit_reached,
+        "return_snippets": return_snippets,
+        "focus_ids": [hit["_id"] for hit in results if isinstance(hit, dict) and hit.get("_id")],
+    }

providers.template.yaml

@@ -0,0 +1,113 @@
+# Template for providers.yaml
+#
+# You can add any OpenAI API compatible provider to the "providers" list.
+# For each provider you must also specify a list of models, along with model abilities.
+#
+# All fields are required unless marked as optional.
+#
+# Refer to your provider's API documentation for specific
+# details such as model identifiers, capabilities etc
+#
+# Note: Since the OpenAI API is not a standard we can't guarantee that all
+# providers will work correctly with Raycast AI.
+#
+# To use this template rename as `providers.yaml`
+#
+providers:
+  - id: perplexity
+    name: Perplexity
+    base_url: https://api.perplexity.ai
+    # Specify at least one api key if authentication is required.
+    # Optional if authentication is not required or is provided elsewhere.
+    # If individual models require separate api keys, then specify a separate `key` for each model's `provider`
+    api_keys:
+      perplexity: PERPLEXITY_KEY
+    # Optional additional parameters sent to the `/chat/completions` endpoint
+    additional_parameters:
+      return_images: true
+      web_search_options:
+        search_context_size: medium
+    # Specify all models to use with the current provider
+    models:
+      - id: sonar # `id` must match the identifier used by the provider
+        name: Sonar # name visible in Raycast
+        provider: perplexity # Only required if mapping to a specific api key
+        description: Perplexity AI model for general-purpose queries # optional
+        context: 128000 # refer to provider's API documentation
+        # Optional abilities - all child properties are also optional.
+        # If you specify abilities incorrectly the model may fail to work as expected in Raycast AI.
+        # Refer to provider's API documentation for model abilities.
+        abilities:
+          temperature:
+            supported: true
+          vision:
+            supported: true
+          system_message:
+            supported: true
+          tools:
+            supported: false
+          reasoning_effort:
+            supported: false
+      - id: sonar-pro
+        name: Sonar Pro
+        description: Perplexity AI model for complex queries
+        context: 200000
+        abilities:
+          temperature:
+            supported: true
+          vision:
+            supported: true
+          system_message:
+            supported: true
+  # provider with multiple api keys
+  - id: my_provider
+    name: My Provider
+    base_url: http://localhost:4000
+    api_keys:
+      openai: OPENAI_KEY
+      anthropic: ANTHROPIC_KEY
+    models:
+      - id: gpt-4o
+        name: "GPT-4o"
+        context: 200000
+        provider: openai # matches "openai" in api_keys
+        abilities:
+          temperature:
+            supported: true
+          vision:
+            supported: true
+          system_message:
+            supported: true
+          tools:
+            supported: true
+      - id: claude-sonnet-4
+        name: "Claude Sonnet 4"
+        context: 200000
+        provider: anthropic # matches "anthropic" in api_keys
+        abilities:
+          temperature:
+            supported: true
+          vision:
+            supported: true
+          system_message:
+            supported: true
+          tools:
+            supported: true
+  - id: litellm
+    name: LiteLLM
+    base_url: http://localhost:4000
+    # No `api_keys` - authentication is provided by the LiteLLM config
+    models:
+      - id: anthropic/claude-sonnet-4-20250514
+        name: "Claude Sonnet 4"
+        context: 200000
+        abilities:
+          temperature:
+            supported: true
+          vision:
+            supported: true
+          system_message:
+            supported: true
+          tools:
+            supported: true
+

providers.yaml

@@ -0,0 +1,20 @@
+providers:
+  - id: vllm
+    name: vLLM Instance
+    base_url: https://lasseedfast.se/vllm
+    api_keys:
+      vllm: "ap98sfoiuajcnwe89sozchnsw9oeacislh"
+    models:
+      - id: "gpt-oss:20b"
+        name: "GPT OSS 20B (Main)"
+        provider: vllm
+        context: 16000
+        abilities:
+          temperature:
+            supported: true
+          vision:
+            supported: false
+          system_message:
+            supported: true
+          tools:
+            supported: true

scripts/convert_embeddings_to_lists.py

@@ -1,182 +0,0 @@
-#!/usr/bin/env python3
-"""
-Convert stored embeddings to plain Python lists for an existing Chroma collection.
-
-Usage:
-  # Dry run (inspect first 50 ids)
-  python scripts/convert_embeddings_to_lists.py --collection talks --limit 50 --dry-run
-
-  # Full run (no dry run)
-  python scripts/convert_embeddings_to_lists.py --collection talks
-
-Notes:
-- Run from your project root (same env you use to access chroma_db).
-- Back up chromadb_data before running.
-"""
-import argparse
-import json
-import os
-import time
-from pathlib import Path
-from typing import List
-import math
-import sys
-
-# Use the same imports/bootstrapping as you already have in your project
-# so the same chroma client and embedding function are loaded.
-# Adjust the import path if necessary.
-os.chdir("/home/lasse/riksdagen")
-sys.path.append("/home/lasse/riksdagen")
-
-import numpy as np
-from _chromadb.chroma_client import chroma_db
-
-CHECKPOINT_DIR = Path("var/chroma_repair")
-CHECKPOINT_DIR.mkdir(parents=True, exist_ok=True)
-
-def normalize_embedding(emb):
-    """
-    Convert a single embedding to a plain Python list[float].
-    Accepts numpy arrays, array-likes, lists.
-    """
-    # numpy ndarray
-    if isinstance(emb, np.ndarray):
-        return emb.tolist()
-    # Some array-likes (pandas/other) may have tolist()
-    if hasattr(emb, "tolist") and not isinstance(emb, list):
-        try:
-            return emb.tolist()
-        except Exception:
-            pass
-    # If it's already a list of numbers, convert elements to float
-    if isinstance(emb, list):
-        return [float(x) for x in emb]
-    # last resort: try iterating
-    try:
-        return [float(x) for x in emb]
-    except Exception:
-        raise ValueError("Cannot normalize embedding of type: %s" % type(emb))
-
-def chunked_iter(iterable, n):
-    it = iter(iterable)
-    while True:
-        chunk = []
-        try:
-            for _ in range(n):
-                chunk.append(next(it))
-        except StopIteration:
-            pass
-        if not chunk:
-            break
-        yield chunk
-
-def load_checkpoint(name):
-    path = CHECKPOINT_DIR / f"{name}.json"
-    if path.exists():
-        return json.load(path)
-    return {"last_index": 0, "processed_ids": []}
-
-def save_checkpoint(name, data):
-    path = CHECKPOINT_DIR / f"{name}.json"
-    with open(path, "w") as f:
-        json.dump(data, f)
-
-def main():
-    parser = argparse.ArgumentParser()
-    parser.add_argument("--collection", required=True, help="Chroma collection name (e.g. talks)")
-    parser.add_argument("--batch", type=int, default=1000, help="Batch size for update (default 1000)")
-    parser.add_argument("--dry-run", action="store_true", help="Dry run: don't write updates, just report")
-    parser.add_argument("--limit", type=int, default=None, help="Limit total number of ids to process (for testing)")
-    parser.add_argument("--checkpoint-name", default=None, help="Name for checkpoint file (defaults to collection name)")
-    args = parser.parse_args()
-
-    coll_name = args.collection
-    checkpoint_name = args.checkpoint_name or coll_name
-
-    print(f"Connecting to Chroma collection '{coll_name}'...")
-    col = chroma_db.get_collection(coll_name)
-
-    # Get the full list of ids. For 600k this should be okay to hold in memory,
-    # but if you need a more streaming approach, tell me and I can adapt.
-    all_info = col.get(include=[])  # may return {'ids': [...]} as in your env
-    ids = list(all_info.get("ids", []))
-    total_ids = len(ids)
-    if args.limit:
-        ids = ids[: args.limit]
-        total_process = len(ids)
-    else:
-        total_process = total_ids
-
-    print(f"Found {total_ids} ids in collection; will process {total_process} ids (limit={args.limit})")
-
-    # load checkpoint
-    ck = load_checkpoint(checkpoint_name)
-    start_index = ck.get("last_index", 0)
-    print(f"Resuming at index {start_index}")
-
-    # iterate in batches starting from last_index
-    processed = 0
-    for i in range(start_index, total_process, args.batch):
-        batch_ids = ids[i : i + args.batch]
-        print(f"\nProcessing batch {i}..{i+len(batch_ids)-1} (count={len(batch_ids)})")
-
-        # fetch full info for this batch (documents, metadatas, embeddings)
-        # we only need embeddings for this repair, but include docs/meta for verification if you want
-        try:
-            items = col.get(ids=batch_ids, include=["embeddings", "documents", "metadatas"])
-        except Exception as e:
-            print("Error fetching batch:", e)
-            # do a small retry after sleep
-            time.sleep(2)
-            items = col.get(ids=batch_ids, include=["embeddings", "documents", "metadatas"])
-
-        batch_embeddings = items.get("embeddings", [])
-        # items.get("ids") should match batch_ids order; if not, align by ids
-        ids_from_get = items.get("ids", batch_ids)
-        if len(ids_from_get) != len(batch_ids):
-            print("Warning: length mismatch between requested ids and returned ids")
-
-        # Normalize embeddings
-        normalized_embeddings = []
-        failed = False
-        for idx, emb in enumerate(batch_embeddings):
-            try:
-                norm = normalize_embedding(emb)
-            except Exception as e:
-                print(f"Failed to normalize embedding for id {ids_from_get[idx]}: {e}")
-                failed = True
-                break
-            normalized_embeddings.append(norm)
-
-        if failed:
-            print("Skipping this batch due to failures. You can adjust batch size and retry.")
-            break
-
-        # Dry-run: just print stats and continue
-        if args.dry_run:
-            # show a sample
-            sample_i = min(3, len(normalized_embeddings))
-            print("Sample normalized embedding lengths:", [len(normalized_embeddings[k]) for k in range(sample_i)])
-            # Optionally inspect first few floats
-            print("Sample values (first 6 floats):", [normalized_embeddings[k][:6] for k in range(sample_i)])
-        else:
-            # Update the collection in place (update will upsert embeddings for given ids)
-            try:
-                col.update(ids=ids_from_get, embeddings=normalized_embeddings)
-            except Exception as e:
-                print("Update failed, retrying once after short sleep:", e)
-                time.sleep(2)
-                col.update(ids=ids_from_get, embeddings=normalized_embeddings)
-
-            print(f"Updated {len(normalized_embeddings)} embeddings in collection '{coll_name}'")
-
-        # checkpoint progress
-        ck["last_index"] = i + len(batch_ids)
-        save_checkpoint(checkpoint_name, ck)
-        processed += len(batch_ids)
-
-    print(f"\nDone. Processed {processed} ids. Checkpoint saved to {CHECKPOINT_DIR / (checkpoint_name + '.json')}")
-    print("Reminder: run a few queries to validate search quality.")
-
-if __name__ == "__main__":
-    main()

scripts/debates.py

@@ -1,5 +1,4 @@
 import os
-import numpy as np
 from time import sleep
 import sys
 from pathlib import Path
@@ -18,7 +17,32 @@ from arango_client import arango
 from colorprinter import *
 
 
-from scripts.build_embeddings import assign_debate_ids
+def assign_debate_ids(
+    docs: list[dict], date: str
+) -> list[dict]:
+    """
+    Assigns a debate id to each talk in a list of talks for a given date.
+    A new debate starts whenever replik == False.
+    The debate id is a string: "{date}:{debate_index}".
+
+    Args:
+        docs (list[dict]): List of talk dicts, each with a 'replik' field.
+        date (str): The date string used as the prefix for debate IDs.
+
+    Returns:
+        list[dict]: The same dicts with a 'debate' field added.
+    """
+    debate_index = 0
+    current_debate_id = f"{date}:{debate_index}"
+    updated_docs = []
+    for doc in docs:
+        if not doc.get("replik", False):
+            debate_index += 1
+            current_debate_id = f"{date}:{debate_index}"
+        doc_with_debate = dict(doc)
+        doc_with_debate["debate"] = current_debate_id
+        updated_docs.append(doc_with_debate)
+    return updated_docs
 
 
 def make_debate_ids():
@@ -110,7 +134,7 @@ def process_debate_date(date: str, system_message: str) -> None:
         bind_vars={"date": date},
     ))
     for debate in debates:
-        llm = LLM(model="vllm", temperature=0.1, system_message=system_message)
+        llm = LLM(model="vllm", temperature=0.2, system_message=system_message)
         # Fetch the talks in this debate
         talks = arango.db.aql.execute(
             """

scripts/documents_to_arango.py

@@ -12,6 +12,9 @@ from arango_client import arango
 
 logging.basicConfig(level=logging.WARNING, format="%(asctime)s - %(levelname)s - %(message)s")
 
+# Module-level collection so it's available both when run directly and when imported
+arango_collection = arango.db.collection("talks")
+
 
 
 def clean_text(text: str) -> str:
@@ -114,7 +117,6 @@ if __name__ == "__main__":
     )
     for doc_key in cursor:
         already_processed.add(doc_key)
-    arango_collection = arango.db.collection("talks")
     folders_in_talks = os.listdir("talks")
     for folder in folders_in_talks:
         path = f"/home/lasse/riksdagen/talks/{folder}"

scripts/fix_things.py

@@ -0,0 +1,25 @@
+from arango_client import arango
+
+chunks_collection = arango.db.collection("chunks")
+
+q = """
+FOR chunk IN chunks
+FILTER chunk.parent_id == null
+RETURN chunk
+"""
+
+cursor = arango.db.aql.execute(q, batch_size=1000, count=True, ttl=360)
+updated_docs = []
+n = 0
+for doc in cursor:
+    n += 1
+    doc['collection'] = 'talks'
+    del doc['chroma_collecton']
+    del doc['chroma_id']
+    doc['parent_id'] = f"talks/{doc['_key'].split(':')[0]}"
+    updated_docs.append(doc)
+    if len(updated_docs) >= 100:
+        chunks_collection.update_many(updated_docs, merge=False, silent=True)
+        updated_docs = []
+        print(f"Updated {n} documents", end="\r")
+chunks_collection.update_many(updated_docs, merge=False, silent=True)

scripts/make_arango_embeddings.py

@@ -0,0 +1,173 @@
+import os
+import sys
+import logging
+
+# Silence the per-request HTTP logs from the ollama/httpx library
+logging.getLogger("httpx").setLevel(logging.WARNING)
+
+os.chdir("/home/lasse/riksdagen")
+sys.path.append("/home/lasse/riksdagen")
+
+from arango_client import arango
+from ollama import Client as Ollama
+from arango.collection import Collection
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import List, Dict
+from time import sleep
+from utils import TextChunker
+
+
+def make_embeddings(texts: List[str]) -> List[List[float]]:
+    """
+    Generate embeddings for a list of texts using Ollama.
+
+    Args:
+        texts (List[str]): List of text strings to embed.
+
+    Returns:
+        List[List[float]]: List of embedding vectors.
+    """
+    ollama_client = Ollama(host='192.168.1.12:33405')
+    embeddings = ollama_client.embed(
+        model="qwen3-embedding:latest",
+        input=texts,
+        dimensions=384,
+    )
+    return embeddings.embeddings
+
+
+def process_chunk_batch(chunk_batch: List[Dict]) -> List[Dict]:
+    """
+    Generate embeddings for a batch of chunks and attach them.
+
+    Args:
+        chunk_batch (List[Dict]): List of chunk dicts, each with a 'text' field.
+
+    Returns:
+        List[Dict]: Same list with an 'embedding' field added to each dict.
+    """
+    sleep(1)
+    texts = [chunk['text'] for chunk in chunk_batch]
+    embeddings = make_embeddings(texts)
+    for i, chunk in enumerate(chunk_batch):
+        chunk['embedding'] = embeddings[i]
+    return chunk_batch
+
+
+def make_arango_embeddings() -> int:
+    """
+    Chunks and embeds all talks that are not yet represented in the 'chunks' collection.
+
+    For each talk that has no chunks in the collection yet:
+      - If the talk document already has a 'chunks' field (legacy path), those are used.
+      - Otherwise the speech text is split into chunks using TextChunker.
+    Embedding vectors are generated via Ollama and stored in the 'chunks' collection.
+
+    Each chunk document in ArangoDB has:
+      _key      : "{talk_key}:{chunk_index}"  (unique within the collection)
+      text      : the chunk text
+      index     : chunk index within the talk
+      parent_id : "talks/{talk_key}"  (links back to the source talk)
+      collection: "talks"
+      embedding : the vector (list of floats)
+
+    Returns:
+        int: Total number of chunk documents inserted/updated.
+    """
+    if not arango.db.has_collection("chunks"):
+        chunks_collection: Collection = arango.db.create_collection("chunks")
+    else:
+        chunks_collection: Collection = arango.db.collection("chunks")
+
+    # Find every talk that has no entry yet in the chunks collection.
+    # The inner FOR loop returns [] if no match exists (acts as NOT EXISTS).
+    cursor = arango.db.aql.execute(
+        """
+        FOR p IN talks
+            FILTER p.anforandetext != null AND p.anforandetext != ""
+            FILTER (
+                FOR c IN chunks
+                FILTER c.parent_id == p._id
+                LIMIT 1
+                RETURN 1
+            ) == []
+            RETURN {
+                _key: p._key,
+                _id: p._id,
+                anforandetext: p.anforandetext,
+                chunks: p.chunks
+            }
+        """,
+        batch_size=1000,
+        ttl=360,
+    )
+
+    n = 0
+    embed_batch_size = 20  # Number of chunks per Ollama call
+    chunk_batches: List[List[Dict]] = []
+
+    for talk in cursor:
+        talk_key = talk["_key"]
+        parent_id = f"talks/{talk_key}"
+
+        if talk.get("chunks"):
+            # Legacy path: chunks were previously generated and stored on the talk document.
+            # Strip out the old ChromaDB-specific fields and assign a proper _key.
+            _chunks = []
+            for chunk in talk["chunks"]:
+                idx = chunk.get("index", 0)
+                _chunks.append({
+                    "_key": f"{talk_key}:{idx}",
+                    "text": chunk["text"],
+                    "index": idx,
+                    "parent_id": parent_id,
+                    "collection": "talks",
+                })
+        else:
+            # New path: chunk the speech text directly with TextChunker.
+            text = (talk.get("anforandetext") or "").strip()
+            text_chunks = TextChunker(chunk_limit=500).chunk(text)
+            _chunks = [
+                {
+                    "_key": f"{talk_key}:{idx}",
+                    "text": content,
+                    "index": idx,
+                    "parent_id": parent_id,
+                    "collection": "talks",
+                }
+                for idx, content in enumerate(text_chunks)
+                if content and content.strip()
+            ]
+
+        # Split into batches for embedding
+        for i in range(0, len(_chunks), embed_batch_size):
+            batch = _chunks[i : i + embed_batch_size]
+            if batch:
+                chunk_batches.append(batch)
+
+    # Embed all batches in parallel (Ollama calls are I/O-bound, threads are fine)
+    total_batches = len(chunk_batches)
+    completed_batches = 0
+    with ThreadPoolExecutor(max_workers=3) as executor:
+        futures = [executor.submit(process_chunk_batch, batch) for batch in chunk_batches]
+        processed_chunks: List[Dict] = []
+        for future in as_completed(futures):
+            result = future.result()
+            completed_batches += 1
+            processed_chunks.extend(result)
+            print(f"Embedding batches: {completed_batches}/{total_batches} | chunks ready to insert: {len(processed_chunks)}", end="\r")
+            # Insert in batches of 100 to keep HTTP payloads small
+            if len(processed_chunks) >= 100:
+                n += len(processed_chunks)
+                chunks_collection.insert_many(processed_chunks, overwrite=True)
+                processed_chunks = []
+        if processed_chunks:
+            n += len(processed_chunks)
+            chunks_collection.insert_many(processed_chunks, overwrite=True)
+
+    print(f"\nDone. Inserted/updated {n} chunks in ArangoDB.")
+    return n
+
+
+if __name__ == "__main__":
+    make_arango_embeddings()

scripts/notes.md

@@ -0,0 +1,5 @@
+### Inlogg riksdagsgruppen från Fojo-hackathon
+https://arango.lasseedfast.se  
+riksdagsgruppen  
+popre4-cygcuz-viHjyc  
+

scripts/sync_talks.py

@@ -0,0 +1,177 @@
+"""
+Synkroniserar nya anföranden från riksdagen.se till databasen och processar dem.
+
+Pipeline (körs dagligen via systemd timer):
+  1. Ladda ned årets anföranden från riksdagen.se (ersätter tidigare nerladdning)
+  2. Infoga nya anföranden i ArangoDB (hoppar över redan existerande)
+  3. Tilldela debatt-ID:n till anföranden som saknar det
+  4. Bygg embeddings för datum som saknar chunks
+  5. Generera sammanfattningar för datum som saknar summary
+
+Kör manuellt: python scripts/sync_talks.py
+"""
+
+import os
+import sys
+import logging
+from datetime import datetime
+from io import BytesIO
+from urllib.request import urlopen
+from zipfile import ZipFile
+
+# Säkerställ att vi kör från projektroten och att lokala moduler hittas
+os.chdir("/home/lasse/riksdagen")
+sys.path.append("/home/lasse/riksdagen")
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s [%(levelname)s] %(message)s",
+)
+logger = logging.getLogger(__name__)
+
+# Systemprompt som används av LLM:en vid sammanfattning av debatter
+SYSTEM_MESSAGE = """Din uppgift är att sammanfatta debatter i Sveriges riksdag. 
+Du kommer först att få enskilda tal som du ska sammanfatta var för sig, efter det ska du sammanfatta hela debatten.
+Sammanfattningarna ska vara på svenska och vara koncisa och informativa. 
+Det är viktigt att du förstår vad som är kärnan i varje tal och debatt, fokusera därför på de argument och sakförhållanden som framförs.
+"""
+
+
+def get_current_session_year() -> int:
+    """
+    Returnerar startåret för aktuell riksdagssession.
+
+    Riksdagssessionen löper september–augusti, så:
+      - Januari–Augusti 2026 → sessionen startade sep 2025 → returnerar 2025
+      - September–December 2025 → sessionen startade sep 2025 → returnerar 2025
+
+    Returns:
+        int: Fyrsiffrigt startår för aktuell session (t.ex. 2025).
+    """
+    now = datetime.now()
+    if now.month >= 9:
+        return now.year
+    else:
+        return now.year - 1
+
+
+def download_current_year(session_year: int) -> str:
+    """
+    Laddar ned och extraherar ZIP-arkivet för angiven riksdagssession,
+    och ersätter eventuella tidigare nerladdade filer för det året.
+
+    Riksdagen uppdaterar kontinuerligt samma ZIP-fil under pågående session,
+    så vi måste ladda ned den på nytt varje gång för att få med nya anföranden.
+
+    Args:
+        session_year (int): Sessionens startår (t.ex. 2025 för session 2025/26).
+
+    Returns:
+        str: Sökväg till katalogen dit filerna extraherades.
+    """
+    second_part = str(session_year + 1)[2:]  # t.ex. "26" för 2026
+    url = f"https://data.riksdagen.se/dataset/anforande/anforande-{session_year}{second_part}.json.zip"
+    folder_name = f"anforande-{session_year}{second_part}"
+    dir_path = os.path.join("talks", folder_name)
+
+    logger.info(f"Downloading {url} → {dir_path}")
+    os.makedirs(dir_path, exist_ok=True)
+
+    # Rensa gamla filer så vi får en färsk kopia
+    for f in os.listdir(dir_path):
+        os.remove(os.path.join(dir_path, f))
+
+    with urlopen(url) as resp:
+        with ZipFile(BytesIO(resp.read())) as zf:
+            zf.extractall(dir_path)
+
+    count = len(os.listdir(dir_path))
+    logger.info(f"Extracted {count} files to {dir_path}")
+    return dir_path
+
+
+def get_unsummarized_dates() -> list[str]:
+    """
+    Hämtar datum från ArangoDB som har anföranden utan sammanfattning.
+
+    Returns:
+        list[str]: Sorterad lista med datumsträngar, t.ex. ["2026-02-10", "2026-02-11"].
+    """
+    from arango_client import arango
+
+    cursor = arango.db.aql.execute(
+        """
+        FOR doc IN talks
+        FILTER doc.summary == null
+        RETURN DISTINCT doc.datum
+        """,
+        ttl=300,
+    )
+    dates = sorted(list(cursor))
+    logger.info(f"Found {len(dates)} dates with unsummarized talks")
+    return dates
+
+
+def sync() -> None:
+    """
+    Kör hela sync-pipelinen:
+      1. Ladda ned årets anföranden
+      2. Infoga nya anföranden i ArangoDB
+      3. Tilldela debatt-ID:n
+      4. Bygg embeddings för nya datum
+      5. Generera sammanfattningar för nya datum
+    """
+    logger.info("=== Starting daily riksdagen sync ===")
+
+    # --- Steg 1: Ladda ned ---
+    session_year = get_current_session_year()
+    logger.info(f"Current session year: {session_year}/{session_year + 1}")
+    dir_path = download_current_year(session_year)
+
+    # --- Steg 2: Infoga nya anföranden i ArangoDB ---
+    # update_folder() hämtar alla befintliga _key:s från databasen och hoppar
+    # över dem, så enbart nya anföranden infogas.
+    logger.info("Stage 2: Inserting new talks into ArangoDB...")
+    from scripts.documents_to_arango import update_folder
+
+    new_talks = update_folder(os.path.abspath(dir_path))
+    logger.info(f"Stage 2 complete: {new_talks} new talks inserted")
+
+    # --- Steg 3: Tilldela debatt-ID:n ---
+    # Anföranden som saknar fältet 'debate' grupperas i debatter baserat på
+    # datum och om de är repliker eller ej.
+    logger.info("Stage 3: Assigning debate IDs to talks missing them...")
+    from scripts.debates import make_debate_ids
+
+    make_debate_ids()
+    logger.info("Stage 3 complete")
+
+    # --- Steg 4: Chunk + bygg embeddings i ArangoDB ---
+    # make_arango_embeddings() hittar alla anföranden som saknar chunks i
+    # 'chunks'-kollektionen, chunkar texten, genererar vektorer via Ollama
+    # och lagrar allt direkt i ArangoDB. ChromaDB används inte.
+    logger.info("Stage 4: Chunking and embedding new talks into ArangoDB...")
+    from scripts.make_arango_embeddings import make_arango_embeddings
+
+    total_chunks = make_arango_embeddings()
+    logger.info(f"Stage 4 complete: {total_chunks} chunks created")
+
+    # --- Steg 5: Generera sammanfattningar ---
+    # process_debate_date() hoppar automatiskt över anföranden som redan har
+    # en sammanfattning, så det är säkert att köra igen.
+    new_dates = get_unsummarized_dates()
+    if new_dates:
+        logger.info(f"Stage 5: Generating summaries for {len(new_dates)} dates...")
+        from scripts.debates import process_debate_date
+
+        for date in new_dates:
+            process_debate_date(date, SYSTEM_MESSAGE)
+        logger.info(f"Stage 5 complete: summaries generated for {len(new_dates)} dates")
+    else:
+        logger.info("Stage 5: No unsummarized dates, skipping")
+
+    logger.info("=== Sync complete ===")
+
+
+if __name__ == "__main__":
+    sync()

scripts/test_make_arango_embeddings.py

@@ -0,0 +1,57 @@
+from arango_client import arango
+from scripts.make_arango_embeddings import process_chunk_batch
+from arango.collection import Collection
+from typing import List, Dict
+
+def test_full_make_arango_embeddings_for_one_talk() -> None:
+    """
+    Integration test for the full make_arango_embeddings chain:
+    - Fetches a specific talk document from ArangoDB.
+    - Processes its chunks to generate embeddings.
+    - Inserts/updates those chunks in the 'chunks' collection.
+    - Verifies that the chunks were updated in ArangoDB.
+
+    This test requires ArangoDB and Ollama to be running and accessible.
+    """
+    # The _id of the talk we want to process
+    target_id: str = "talks/000004cc-b896-e611-9441-00262d0d7125"
+    _key = target_id.split("/")[-1]
+
+    # Get the talks and chunks collections
+    talks_collection: Collection = arango.db.collection("talks")
+    chunks_collection: Collection = arango.db.collection("chunks")
+
+    # Fetch the talk document
+    talk: Dict = talks_collection.get(target_id)
+    assert talk is not None, f"Talk with _id {target_id} not found"
+    assert "chunks" in talk and talk["chunks"], "Talk has no chunks"
+
+    # Prepare chunks for embedding
+    processed_chunks: List[Dict] = []
+    for chunk in talk["chunks"]:
+        key: str = chunk["chroma_id"].split("/")[-1]
+        chunk["_key"] = key.split(":")[-1]
+        chunk["parent_id"] = target_id
+        chunk["collection"] = "talks"
+        # Remove fields not needed for embedding
+        if "chroma_id" in chunk:
+            del chunk["chroma_id"]
+        if "chroma_collecton" in chunk:
+            del chunk["chroma_collecton"]
+        processed_chunks.append(chunk)
+
+    # Generate embeddings for all chunks
+    processed_chunks = process_chunk_batch(processed_chunks)
+
+    # Insert/update chunks in the 'chunks' collection
+    chunks_collection.insert_many(processed_chunks, overwrite=True)
+
+    # Verify that the chunks were updated in ArangoDB
+    for chunk in processed_chunks:
+        db_chunk = chunks_collection.get(chunk["_key"])
+        assert db_chunk is not None, f"Chunk {_key} not found in DB"
+        assert "embedding" in db_chunk, "Chunk missing embedding in DB"
+        assert isinstance(db_chunk["embedding"], list), "Embedding is not a list"
+        print(f"Chunk {chunk['_key']} updated with embedding of length {len(db_chunk['embedding'])}")
+
+test_full_make_arango_embeddings_for_one_talk()

test_gpu

@@ -0,0 +1,70 @@
+import requests
+import time
+import sys
+import base64
+
+# Adjust if your server is remote
+VLLM_URL = "http://localhost:8010/v1/chat/completions"
+
+# Path to your image file
+IMAGE_PATH = "page_063.png"
+
+# Read and encode the image
+with open(IMAGE_PATH, "rb") as img_file:
+    image_data = img_file.read()
+
+# Convert image to base64
+image_base64 = base64.b64encode(image_data).decode("utf-8")
+
+# Prepare the messages
+messages = [
+    {"role": "system", "content": "You are a helpful assistant."},
+    {"role": "user", "content": "Describe the content of this image."},
+    {
+        "role": "user",
+        "content": [
+            {"type": "text", "text": "Please analyze the following image."},
+            {
+                "type": "image_url",
+                "image_url": {
+                    "url": f"data:image/png;base64,{image_base64}",
+                    "detail": "high"
+                }
+            }
+        ]
+    }
+]
+
+# Number of test iterations
+N = 10
+
+def test_vllm():
+    print(f"Starting vLLM image input test on {VLLM_URL}")
+    for i in range(1, N + 1):
+        payload = {
+            "model": "openbmb/MiniCPM-V-4_5-AWQ",
+            "messages": messages,
+            "max_tokens": 256,
+            "temperature": 0.7,
+        }
+
+        t0 = time.time()
+        try:
+            resp = requests.post(VLLM_URL, json=payload, timeout=120)
+            resp.raise_for_status()
+            data = resp.json()
+            output = data.get("choices", [{}])[0].get("message", {}).get("content", "")
+            print(output)
+            elapsed = time.time() - t0
+            print(f"[{i}/{N}] ✅ Response OK in {elapsed:.2f}s, output length={len(output)}")
+        except Exception as e:
+            print(f"[{i}/{N}] ❌ Error: {e}")
+            if "connection" in str(e).lower() or "timeout" in str(e).lower():
+                print("⚠️ Possible GPU hang or server crash. Stopping test.")
+                sys.exit(1)
+        time.sleep(1)  # small delay between requests
+
+    print("✅ Test completed successfully.")
+
+if __name__ == "__main__":
+    test_vllm()