diff --git a/_postgres/_postgres.py b/_postgres/_postgres.py index 0f1aa4e..d5a3ecd 100644 --- a/_postgres/_postgres.py +++ b/_postgres/_postgres.py @@ -199,23 +199,31 @@ class Postgres: def make_embeddings(self, texts: List[str]) -> List[List[float]]: # 1. Setup Client - client = OpenAI( - base_url=f"http://{os.environ.get('VLLM_EMBEDDING_HOST', '192.168.1.10:8003')}/v1", - api_key="vllm-key" - ) + from parliament import PARLIAMENT + + base_url = os.environ.get(PARLIAMENT.embeddings.base_url_env) + if not base_url: + raise RuntimeError( + f"{PARLIAMENT.embeddings.base_url_env} is not set. It must point at an " + f"OpenAI-compatible embeddings endpoint, e.g. http://localhost:8003/v1" + ) + client = OpenAI(base_url=base_url, api_key=os.environ.get("EMBEDDING_API_KEY", "none")) # 2. Request Embeddings # We pass 'dimensions' in the body. Qwen3 usually supports this. response = client.embeddings.create( input=texts, - model=os.environ.get("LLM_MODEL_EMBEDDING", "qwen3-embedding"), - extra_body={"dimensions": 384} + model=os.environ.get("LLM_MODEL_EMBEDDING", PARLIAMENT.embeddings.model), + extra_body={"dimensions": PARLIAMENT.embeddings.dimension}, ) # 3. Safety Slice # Even if the server returns the full 3584 dims by mistake, # this ensures your DB doesn't throw a dimension mismatch error. - return [emb.embedding[:384] for emb in response.data] + dim = PARLIAMENT.embeddings.dimension + # Truncate defensively: not every server honours the `dimensions` request, + # and a wider vector than the column would fail deep inside pgvector. + return [emb.embedding[:dim] for emb in response.data] def close(self): """Close all connections in the pool.""" diff --git a/backend/app.py b/backend/app.py index 409db55..7d6a9f6 100644 --- a/backend/app.py +++ b/backend/app.py @@ -10,7 +10,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi.responses import PlainTextResponse from postgres_client import pg -from info import debate_types, explainer, limit_warning, party_colors +from parliament import PARLIAMENT from .schemas import ( ChatRequest, ChatResponse, @@ -49,6 +49,52 @@ app.include_router(settings_router) app.include_router(names_autocomplete_router) +@app.on_event("startup") +def _verify_database_matches_config() -> None: + """Refuse to serve if the database disagrees with parliament.yaml. + + Both mismatches below fail silently rather than loudly, which is why they are + checked here: + + * A wrong text-search configuration makes every full-text query return almost + nothing, with no error — the app looks empty rather than broken. + * A vector column narrower or wider than `embeddings.dimension` fails inside + pgvector with a message that never mentions configuration. + """ + from parliament import PARLIAMENT + + try: + rows = pg.execute("SELECT current_setting('app.fts_config', true) AS cfg") + configured = PARLIAMENT.language.fts_config + actual = (rows[0]["cfg"] if rows else None) or None + if actual and actual != configured: + raise RuntimeError( + f"Database app.fts_config is {actual!r} but parliament.yaml declares " + f"{configured!r}. Fix with: " + f"ALTER DATABASE SET app.fts_config = '{configured}';" + ) + + rows = pg.execute( + "SELECT a.atttypmod AS typmod FROM pg_attribute a " + "JOIN pg_class c ON c.oid = a.attrelid " + "WHERE c.relname = 'chunks' AND a.attname = 'embedding'" + ) + if rows and rows[0]["typmod"] and rows[0]["typmod"] > 0: + actual_dim = rows[0]["typmod"] + if actual_dim != PARLIAMENT.embeddings.dimension: + raise RuntimeError( + f"chunks.embedding is vector({actual_dim}) but parliament.yaml " + f"declares embeddings.dimension={PARLIAMENT.embeddings.dimension}. " + f"Re-embedding is required to change this." + ) + except RuntimeError: + raise + except Exception as exc: + # A database that is merely unreachable is not a configuration error; + # let the request path report that in its own terms. + print(f"[startup] could not verify database configuration: {exc}") + + @app.on_event("startup") def _reap_abandoned_jobs() -> None: """Finalize research jobs whose child died while the API was down.""" @@ -74,12 +120,7 @@ def get_guide() -> str: @app.get("/api/meta") def meta(): - return { - "parties": party_colors, - "debate_types": debate_types, - "explainer": explainer, - "limit_warning": limit_warning, - } + return PARLIAMENT.public_meta() @app.post("/api/search", response_model=SearchResponse) diff --git a/backend/services/attribution.py b/backend/services/attribution.py index b9a96bf..2689a83 100644 --- a/backend/services/attribution.py +++ b/backend/services/attribution.py @@ -20,9 +20,15 @@ import re import unicodedata from typing import Any, Dict, List, Optional +from parliament import PARLIAMENT + +# How a party code looks in rendered text. Configured per parliament: Swedish +# codes are 1-3 uppercase letters including åäö, but other countries differ. +_PARTY = PARLIAMENT.party_defaults["code_pattern"] + # Matches [Name](/mp/12345) (PARTY) — the injected link with trailing party _SPEAKER_LINK_RE = re.compile( - r"\[([^\]]+)\]\(/mp/[^)]+\)\s*\(([A-ZÅÄÖ]{1,3})\)", + rf"\[([^\]]+)\]\(/mp/[^)]+\)\s*\(({_PARTY})\)", re.UNICODE, ) @@ -31,7 +37,7 @@ _SPEAKER_LINK_RE = re.compile( # We intentionally restrict "Name" to Title-case-ish tokens so we don't light up on # every parenthesised aside in the text. _PLAIN_SPEAKER_RE = re.compile( - r"(?, %s) && ..." tsq_params - list of values matching each %s in tsq_sql snippet_terms - raw terms used for snippet highlighting (informational) @@ -119,11 +124,11 @@ class SearchService: seen.add(clean) tsq_params.append(clean) if is_prefix: - return "to_tsquery('swedish', %s || ':*')" + return f"to_tsquery('{_FTS}', %s || ':*')" elif is_phrase: - return "phraseto_tsquery('swedish', %s)" + return f"phraseto_tsquery('{_FTS}', %s)" else: - return "plainto_tsquery('swedish', %s)" + return f"plainto_tsquery('{_FTS}', %s)" # MUST: all must match must_parts = [_add_term(t) for t in parsed.must_terms] @@ -226,9 +231,9 @@ class SearchService: order_by = "ts_rank_cd(t.search_vector, q.tsq) DESC, t.datum ASC, t.anforande_nummer ASC" if include_snippets: headline_col = ( - ", ts_headline('swedish', t.anforandetext, q.tsq, " + f", ts_headline('{_FTS}', t.anforandetext, q.tsq, " "'MaxWords=15, MinWords=8, MaxFragments=1') AS _headline" - ", ts_headline('swedish', t.anforandetext, q.tsq, " + f", ts_headline('{_FTS}', t.anforandetext, q.tsq, " "'MaxWords=60, MinWords=30, MaxFragments=4') AS _headline_long" ) else: @@ -300,7 +305,7 @@ class SearchService: snippet_long = text[:800] kammaraktivitet = doc.get("kammaraktivitet") - debate_info = debate_types.get(kammaraktivitet, {}) + debate_info = PARLIAMENT.activity_types.get(kammaraktivitet, {}) debate_type_title = ( debate_info.get("title", kammaraktivitet) if isinstance(debate_info, dict) @@ -429,9 +434,9 @@ class MotionSearchService(SearchService): order_by = "ts_rank_cd(m.search_vector, q.tsq) DESC, m.datum DESC" if include_snippets: headline_col = ( - ", ts_headline('swedish', m.text, q.tsq, " + f", ts_headline('{_FTS}', m.text, q.tsq, " "'MaxWords=15, MinWords=8, MaxFragments=1') AS _headline" - ", ts_headline('swedish', m.text, q.tsq, " + f", ts_headline('{_FTS}', m.text, q.tsq, " "'MaxWords=60, MinWords=30, MaxFragments=4') AS _headline_long" ) else: diff --git a/backend/services/snippets.py b/backend/services/snippets.py index 76c6b56..63a843f 100644 --- a/backend/services/snippets.py +++ b/backend/services/snippets.py @@ -2,12 +2,12 @@ from __future__ import annotations from typing import Iterable -from info import months_conversion, party_colors_lighten +from parliament import PARLIAMENT def datestring_to_iso(date_str: str) -> str: day, month_name, year = date_str.split(" ") - return f"{year}-{months_conversion[month_name]}-{day.zfill(2)}" + return f"{year}-{PARLIAMENT.language.months[month_name]}-{day.zfill(2)}" def _cleanup(text: str) -> str: @@ -50,4 +50,4 @@ def make_snippet(text: str, search_terms: Iterable[str], long: bool = False) -> def assign_party_highlight(party: str) -> str: - return party_colors_lighten.get(party, "#f0f0f0") + return PARLIAMENT.party_highlight_color(party) diff --git a/content/sv/explainer.md b/content/sv/explainer.md new file mode 100644 index 0000000..4d5c64c --- /dev/null +++ b/content/sv/explainer.md @@ -0,0 +1,12 @@ +Det här är en databas över vad svenska riksdagspolitiker har sagt i olika debatter i Riksdagen sedan 1993. +Datan kommer dels från data.riksdagen.se och dels från transkriberingar av vad som sänts i Riksdagens videotjänst (från år 2000). + +- Börja med att skriva ett eller flera sökord nedan. Du kan använda asterisk (\*), minus (-), citattecken (""), OR och år\:yyyy-yyyy. Sökningen +`energikris* baskraft OR kärnkraft "fossilfria energikällor" -vindkraft år:2015-2022` söker anföranden som\: + - nämner "energikris" (inkl. ex. "energikris*en*") + - nämner antingen "baskraft" *eller* "kärnkraft" + - nämner den *exakta frasen* "fossilfria energikällor" + - *inte* nämner "vindkraft" + - återfinns under åren 2015-2022 +- När du fått dina resultat kan du sedan klicka bort partier eller ändra vilka år och debattyper du är intresserad av. +- Under "Längre utdrag" kan du välja att se hela anförandet i text, och under texten finns länkar till Riksdagens Webb-TV och nedladdningsbart ljud (i de fall där debatten har sänts). diff --git a/content/sv/limit_warning.md b/content/sv/limit_warning.md new file mode 100644 index 0000000..372a7c4 --- /dev/null +++ b/content/sv/limit_warning.md @@ -0,0 +1,3 @@ +Din sökning ger fler än 10 000 träffar. Försök göra den mer specifik, exempelvis genom att +använda minustecken eller specificera årtal genom att skriva år\:yyyy-yyyy (ex. år:2019-2020, utan mellanrum efter kolon). +Gränsen på 10 000 träffar finns för att servern inte ska överbelastas. diff --git a/frontend/src/App.tsx b/frontend/src/App.tsx index 52aa3fd..9258905 100644 --- a/frontend/src/App.tsx +++ b/frontend/src/App.tsx @@ -3,6 +3,7 @@ import { BrowserRouter, Routes, Route, useSearchParams, useNavigate } from "reac import { useQuery, useMutation } from "@tanstack/react-query"; // Hooks for data fetching & mutations import { fetchMeta, searchTalks, sendFeedback, setSessionId } from "./api"; // API functions import type { SearchFilters, TalkHit } from "./types"; +import { setPhotoUrlTemplate } from "./utils/markdown"; import { Link } from "react-router-dom"; import { SearchPanel } from "./components/SearchPanel"; // Search form & filters import { StatsView } from "./components/StatsView"; // Stats visualization @@ -87,6 +88,20 @@ function SearchView() { queryFn: fetchMeta, }); + // Publish the configured palette as CSS custom properties. Doing it here rather + // than in the stylesheet is what lets a non-Swedish deployment render its own + // parties: CSS cannot read a dict, so hardcoded per-party rules could never adapt. + useEffect(() => { + if (!meta.data) return; + const root = document.documentElement; + for (const party of meta.data.parties) { + root.style.setProperty(`--party-${party.code}`, party.color); + } + root.style.setProperty("--party-na", meta.data.party_defaults.unknown_color); + setPhotoUrlTemplate(meta.data.urls?.person_photo ?? ""); + if (meta.data.site?.title) document.title = meta.data.site.title; + }, [meta.data]); + const feedback = useMutation({ mutationFn: sendFeedback, }); @@ -158,7 +173,7 @@ function SearchView() { }, [filters, speaker, speakerIds]); const selectedDebateLabels = useMemo(() => { - const mapping = meta.data?.debate_types ?? {}; + const mapping = meta.data?.activity_types ?? {}; return (filters.debates ?? []).map((code) => { const debateType = mapping[code]; return typeof debateType === 'string' ? debateType : debateType?.title ?? code; diff --git a/frontend/src/components/ChatPanel.tsx b/frontend/src/components/ChatPanel.tsx index 62a61da..e76c4d9 100644 --- a/frontend/src/components/ChatPanel.tsx +++ b/frontend/src/components/ChatPanel.tsx @@ -110,7 +110,7 @@ const SearchResultView = ({ card }: { card: LiveSearchCard }) => { )}
{result.speaker && {result.speaker}} - {result.party && {result.party}} + {result.party && {result.party}} {result.date && {result.date}} {result.snippet &&

{result.snippet}

}
diff --git a/frontend/src/components/ChatSnapshotView.tsx b/frontend/src/components/ChatSnapshotView.tsx index 42d1f8e..50c1390 100644 --- a/frontend/src/components/ChatSnapshotView.tsx +++ b/frontend/src/components/ChatSnapshotView.tsx @@ -66,7 +66,7 @@ function MpHeader({ intressent_id }: { intressent_id: string }) { onError={(e) => { (e.currentTarget as HTMLImageElement).style.display = "none"; }} />
{person.namn} - {person.parti && {person.parti}} + {person.parti && {person.parti}}

Fryst konversation med digital assistent – inte den riktiga personen.

diff --git a/frontend/src/components/ExplainerText.tsx b/frontend/src/components/ExplainerText.tsx index 9fcca2a..369cd0b 100644 --- a/frontend/src/components/ExplainerText.tsx +++ b/frontend/src/components/ExplainerText.tsx @@ -1,6 +1,10 @@ import React from "react"; -export function ExplainerText() { +type Contact = { email: string | null; url: string | null }; + +/** Contact details come from parliament.yaml, so a deployment can set its own + * without editing this component. Omitted entirely when unset. */ +export function ExplainerText({ contact }: { contact?: Contact }) { return (
Hur fungerar söket? @@ -27,10 +31,17 @@ export function ExplainerText() { När du fått resultat kan du filtrera partier, justera år och välja debatttyper. Under "Längre utdrag" hittar du hela tal med länkar till Webb-TV och ljud när de finns.

-

- Har du idéer eller hittar buggar? Mejla mig eller{" "} - skriv på Twitter. / Lasse Edfast, journalist. -

+ {contact?.email && ( +

+ Har du idéer eller hittar buggar? Mejla mig + {contact.url && ( + <> + {" "}eller läs mer om sidan + + )} + . +

+ )}
); diff --git a/frontend/src/components/MotionBody.tsx b/frontend/src/components/MotionBody.tsx index ea2e7ec..23e3daa 100644 --- a/frontend/src/components/MotionBody.tsx +++ b/frontend/src/components/MotionBody.tsx @@ -54,7 +54,7 @@ export function MotionBody({ motion }: { motion: Motion }) {

{primary?.namn ?? motion.talare}

)} {primary?.partibet && ( - {primary.partibet} + {primary.partibet} )}
diff --git a/frontend/src/components/MpChatPanel.tsx b/frontend/src/components/MpChatPanel.tsx index 2a1e7c1..b31cab8 100644 --- a/frontend/src/components/MpChatPanel.tsx +++ b/frontend/src/components/MpChatPanel.tsx @@ -443,7 +443,7 @@ export function MpChatPanel({ person, initialTalkId, sessionId }: Props) {

{person.namn}

{person.parti && ( - + {person.parti} )} diff --git a/frontend/src/components/ResultsTable.tsx b/frontend/src/components/ResultsTable.tsx index e6dce6d..0c4d093 100644 --- a/frontend/src/components/ResultsTable.tsx +++ b/frontend/src/components/ResultsTable.tsx @@ -94,6 +94,7 @@ export function ResultsTable({ results, exportResults, onLoadMore, nextBatchSize key={hit._id} className="results-table__row" data-party={hit.party ?? ""} + style={{ "--party-color": `var(--party-${hit.party ?? ""})` } as React.CSSProperties} onClick={handleRowClick} role="button" tabIndex={0} @@ -120,7 +121,7 @@ export function ResultsTable({ results, exportResults, onLoadMore, nextBatchSize )} - + {hit.party} diff --git a/frontend/src/components/SearchPanel.tsx b/frontend/src/components/SearchPanel.tsx index 4b2ff95..6c71fdf 100644 --- a/frontend/src/components/SearchPanel.tsx +++ b/frontend/src/components/SearchPanel.tsx @@ -57,9 +57,9 @@ export function SearchPanel({ onChatMentionSelect, chatInputRef, }: Props) { - const partyOptions = useMemo(() => Object.keys(meta?.parties ?? {}).filter(Boolean), [meta]); + const partyOptions = useMemo(() => (meta?.parties ?? []).filter((p) => p.active).map((p) => p.code), [meta]); const debateOptions = useMemo( - () => Object.entries(meta?.debate_types ?? {}), + () => Object.entries(meta?.activity_types ?? {}), [meta], ); diff --git a/frontend/src/components/StatsView.tsx b/frontend/src/components/StatsView.tsx index a7ce043..2097875 100644 --- a/frontend/src/components/StatsView.tsx +++ b/frontend/src/components/StatsView.tsx @@ -24,7 +24,7 @@ export function StatsView({ stats, meta }: Props) { {partyEntries.map(([party]) => ( - + p.code === party)?.color ?? meta?.party_defaults?.unknown_color ?? "#999"} /> ))} diff --git a/frontend/src/components/TalkDrawer.tsx b/frontend/src/components/TalkDrawer.tsx index 7212b28..2f23364 100644 --- a/frontend/src/components/TalkDrawer.tsx +++ b/frontend/src/components/TalkDrawer.tsx @@ -120,7 +120,7 @@ export function TalkDrawer() { ) : (

{talk.talare}

)} - + {talk.parti}
diff --git a/frontend/src/components/TalkView.tsx b/frontend/src/components/TalkView.tsx index a238d23..c92279b 100644 --- a/frontend/src/components/TalkView.tsx +++ b/frontend/src/components/TalkView.tsx @@ -150,7 +150,7 @@ export function TalkView() { ) : (

{talk.talare}

)} - + {talk.parti}
diff --git a/frontend/src/styles.css b/frontend/src/styles.css index 4c2f6ae..7317096 100644 --- a/frontend/src/styles.css +++ b/frontend/src/styles.css @@ -42,16 +42,8 @@ --danger: #a61f2c; --danger-soft: rgba(166, 31, 44, 0.08); - /* Party colors */ - --party-M: #52bdec; - --party-S: #e8112d; - --party-V: #b51a0e; - --party-MP: #83cf39; - --party-C: #009933; - --party-KD: #000077; - --party-L: #006ab3; - --party-SD: #dddd00; - --party-NYD: #ffff2b; + /* Party colours are injected at runtime from /api/meta (see App.tsx), so this + stylesheet carries no country-specific values. Only the fallback lives here. */ --party-na: #9aa5b8; /* Typography — self-hosted via @fontsource (see main.tsx). The "Variable" @@ -495,28 +487,18 @@ textarea { background: #e5e7eb; } -.party-chip[data-party="M"] { background: #52bdec33; color: #005884; } -.party-chip[data-party="S"] { background: #e8112d33; color: #8c0e1d; } -.party-chip[data-party="V"] { background: #b51a0e33; color: #8d170d; } -.party-chip[data-party="MP"] { background: #83cf3933; color: #2f6d16; } -.party-chip[data-party="C"] { background: #00993333; color: #0a5d24; } -.party-chip[data-party="KD"] { background: #00007733; color: #122b8f; } -.party-chip[data-party="L"] { background: #006ab333; color: #024270; } -.party-chip[data-party="SD"] { background: #dddd0033; color: #8b8b09; } -.party-chip[data-party="NYD"] { background: #ffff2b33; color: #8a7d05; } +/* One rule for every party. `--party-color` is set inline by the component from + the colour /api/meta supplied; color-mix derives the tint and the text shade, + which is why no per-party lightened palette needs maintaining. */ +.party-chip { + background: color-mix(in srgb, var(--party-color, var(--party-na)) 20%, transparent); + color: color-mix(in srgb, var(--party-color, var(--party-na)) 75%, black); +} .party-chip[data-party="-"], .party-chip[data-party=""] { background: #eceef3; color: var(--muted); } +.results-table__row td:first-child { border-left-color: var(--party-color, var(--party-na)); } /* Party-color left border on result rows */ -.results-table__row[data-party="M"] td:first-child { border-left-color: var(--party-M); } -.results-table__row[data-party="S"] td:first-child { border-left-color: var(--party-S); } -.results-table__row[data-party="V"] td:first-child { border-left-color: var(--party-V); } -.results-table__row[data-party="MP"] td:first-child { border-left-color: var(--party-MP); } -.results-table__row[data-party="C"] td:first-child { border-left-color: var(--party-C); } -.results-table__row[data-party="KD"] td:first-child { border-left-color: var(--party-KD); } -.results-table__row[data-party="L"] td:first-child { border-left-color: var(--party-L); } -.results-table__row[data-party="SD"] td:first-child { border-left-color: var(--party-SD); } -.results-table__row[data-party="NYD"] td:first-child { border-left-color: var(--party-NYD); } .results-table__row[data-party="-"] td:first-child, .results-table__row[data-party=""] td:first-child { border-left-color: var(--party-na); } diff --git a/frontend/src/types.ts b/frontend/src/types.ts index e54421f..4072dc6 100644 --- a/frontend/src/types.ts +++ b/frontend/src/types.ts @@ -55,11 +55,33 @@ export type DebateType = { description: string; }; +export type Party = { + code: string; + name: string; + color: string; + active: boolean; +}; + export type MetaResponse = { - parties: Record; - debate_types: Record; - explainer: string; - limit_warning: string; + parliament: { + name: string; + name_en: string; + country: string; + data_start_year: number; + }; + parties: Party[]; + party_defaults: { unknown_color: string; code_pattern: string }; + /** Keyed by the value stored in talks.kammaraktivitet. */ + activity_types: Record; + vocabulary: Record; + urls: Record; + site: { + title: string; + tagline: string; + explainer: string; + limit_warning: string; + contact: { email: string | null; url: string | null }; + }; }; export type ChatMessage = { diff --git a/frontend/src/utils/markdown.ts b/frontend/src/utils/markdown.ts index b9b90f4..051fac7 100644 --- a/frontend/src/utils/markdown.ts +++ b/frontend/src/utils/markdown.ts @@ -2,8 +2,16 @@ import { marked } from "marked"; import DOMPurify from "dompurify"; import type { ChatSource } from "../types"; -export const getMpPhotoUrl = (intressent_id: string): string => - `https://data.riksdagen.se/filarkiv/bilder/ledamot/${intressent_id}_192.jpg`; +// Set once from /api/meta (urls.person_photo). The Swedish default keeps portraits +// rendering during the first paint, before meta has arrived. +let photoUrlTemplate = "https://data.riksdagen.se/filarkiv/bilder/ledamot/{person_id}_192.jpg"; + +export const setPhotoUrlTemplate = (template: string): void => { + if (template) photoUrlTemplate = template; +}; + +export const getMpPhotoUrl = (personId: string): string => + photoUrlTemplate.replace("{person_id}", personId); /** * Converts Markdown to HTML and replaces [1], [2], ... with citations. diff --git a/info.py b/info.py deleted file mode 100644 index 3882e16..0000000 --- a/info.py +++ /dev/null @@ -1,180 +0,0 @@ -""" Information and constants are put here and imported into app.py. """ - -party_colors = { - "MP": "#83CF39", - "V": "#b51a0e", - "S": "#E8112d", - "C": "#009933", - "M": "#52BDEC", - "KD": "#000077", - "SD": "#DDDD00", - "L": "#006AB3", - "NYD": "#ffff2b", - '': 'white', - '-': 'white' -} - -select_columns = ''' - _id, - dok_id, - "anforandetext" AS "Text", - anforande_nummer AS number, - kammaraktivitet as debatetype, - talare AS "Talare", - datum AS "Datum", - year AS År, - debateurl AS url_session, - parti AS "Parti", - audiofileurl as url_audio, - startpos as start, - intressent_id - ''' - - - - - - -# 70 % lighter party colors. -party_colors_lighten = { - "MP": '#daf1c4', - "V": '#f8ada7', - "S": '#fab6bf', - "C": '#94ffb8', - "M": '#cbebf9', - "KD": "#b1b1ff", # 80 % - "SD": "#ffffa8", - "L": "#9cd6ff", - "NYD": "#ffffbf", - '': 'white', - '-': 'white' -} - -css = """ ' - -# css = ''' -# -# ''' - -months_conversion = { - 'januari': '01', - 'februari': '02', - 'mars': '03', - 'april': '04', - 'maj': '05', - 'juni': '06', - 'juli': '07', - 'augusti': '08', - 'september': '09', - 'oktober': '10', - 'november': '11', - 'december': '12' - } -explainer = """Det här är en databas över vad svenska riksdagspolitiker har sagt i olika debatter i Riksdagen sedan 1993. -Datan kommer dels från data.riksdagen.se och dels från transkriberingar av vad som sänts i Riksdagens videotjänst (från år 2000). -- Börja med att skriva ett eller flera sökord nedan. Du kan använda asterix (*), minus(-), citattecken (""), OR och år\:yyyy-yyyy. Sökningen -`energikris* baskraft OR kärnkraft "fossilfria energikällor" -vindkraft år:2015-2022` söker anföranden som\: - - nämner "energikris" (inkl. ex. "energikris*en*") - - nämner antingen "baskraft" *eller* "kärnkraft" - - nämner den *exakta frasen* "fossilfria energikällor" - - *inte* nämner "vindkraft" - - återfinns under åren 2015-2022 -- När du fått dina resultat kan sedan klicka bort parier eller ändra vilka år och debatttyper du är intresserad av. -- Under "Längre utdrag" kan du välja att se hela anförandet i text, och under texten finns länkar till Riksdagens Webb-TV och nedladdningsbart ljud (i de fall -där debatten har sänts). - -Berätta gärna hur du skulle vilja använda datan och om sånt som inte funkar. [Mejla mig](mailto:lasse@edfast.se) eller [skriv till mig på Twitter](https://twitter.com/lasseedfast). -Jag som gjort den här sidan heter [Lasse Edfast och är journalist](https://lasseedfast.se). -""" - -debate_types = { - "kam-vo": { - "title": "Beslut", - "description": "Riksdagen fattar det formella beslutet i ärendet." - }, - "bet": { - "title": "Debatt om beslut", - "description": "Ledamöterna debatterar ett förslag innan beslut fattas." - }, - "kam-fs": { - "title": "Frågestund", - "description": "Ministrar svarar direkt på frågor från ledamöter." - }, - "kam-ar": { - "title": "Information från regeringen", - "description": "Regeringen informerar riksdagen i en aktuell fråga." - }, - "ip": { - "title": "Interpellationsdebatt", - "description": "Debatt om en skriftlig fråga till en minister." - }, - "kam-sf": { - "title": "Statsministerns frågestund", - "description": "Statsministern svarar på frågor från ledamöter." - }, - "sam-ou": { - "title": "Öppen utfrågning", - "description": "Utskottet frågar ut experter eller ansvariga personer offentligt." - }, - "kam-ad": { - "title": "Aktuell debatt", - "description": "Snabb debatt om en aktuell politisk fråga." - }, - "kam-al": { - "title": "Allmänpolitisk debatt", - "description": "Ledamöterna debatterar valfria politiska ämnen." - }, - "kam-bu": { - "title": "Budgetdebatt", - "description": "Debatt om regeringens budgetförslag." - }, - "kam-bp": { - "title": "Bordläggning", - "description": "Ett ärende anmäls i kammaren inför fortsatt behandling." - }, - "kam-pd": { - "title": "Partiledardebatt", - "description": "Partiledarna debatterar politikens stora frågor." - }, - "kam-dv": { - "title": "Debatt med anledning av vårpropositionen", - "description": "Debatt om regeringens ekonomiska vårproposition." - }, - "sam-se": { - "title": "Öppet seminarium", - "description": "Öppet seminarium eller samtal om ett särskilt tema." - }, - "kam-ud": { - "title": "Utrikespolitisk debatt", - "description": "Debatt om Sveriges utrikespolitik." - } - } - -limit_warning = ''' - Din sökning ger fler än 10 000 träffar. Försök gör den mer specifik, exempelvis genom att - använda minustecken eller specificera årtal genom att skriva år\:yyyy-yyyy (ex. år:2019-2020, utan mellanrum efter kolon). - Gränsen på 10 000 träffar finns för att servern inte ska krascha och kommer att höjas när jag har en starkare server. - ''' \ No newline at end of file diff --git a/parliament.py b/parliament.py new file mode 100644 index 0000000..e7225ed --- /dev/null +++ b/parliament.py @@ -0,0 +1,218 @@ +"""Load parliament.yaml — every country-specific value in one place. + +Lives at the repository root rather than under ``backend/`` so the ingest CLI and +the maintenance scripts can import it without pulling in FastAPI. + + from parliament import PARLIAMENT + PARLIAMENT.language.fts_config # 'swedish' + PARLIAMENT.party_color('S') # '#E8112d' + PARLIAMENT.vocabulary['speech'] # 'anförande' + +Mirrors the loading style already used by backend/services/provider_registry.py: +a module-level singleton, ``yaml.safe_load``, frozen dataclasses. +""" +from __future__ import annotations + +import os +import re +from dataclasses import dataclass, field +from functools import cached_property +from pathlib import Path +from typing import Any, Optional + +import yaml + +_ROOT = Path(__file__).resolve().parent + +# A Postgres text-search configuration name. It is interpolated into SQL rather +# than passed as a parameter (identifiers cannot be bound), so it is validated +# on load and never trusted from arbitrary input. +_FTS_CONFIG_RE = re.compile(r"^[a-z_][a-z0-9_]*$") + + +class ConfigError(ValueError): + """parliament.yaml is missing, malformed, or internally inconsistent.""" + + +@dataclass(frozen=True) +class Party: + code: str + name: str + color: str + active: bool = True + + +@dataclass(frozen=True) +class Language: + fts_config: str + prompt_language: str + locale: str + preserve_characters: str + months: dict[str, str] = field(default_factory=dict) + + +@dataclass(frozen=True) +class Embeddings: + model: str + dimension: int + base_url_env: str + chunk_chars: int + + +@dataclass(frozen=True) +class Parliament: + """The active parliament's configuration.""" + + meta: dict[str, Any] + language: Language + vocabulary: dict[str, str] + parties: list[Party] + party_defaults: dict[str, str] + activity_types: dict[str, dict[str, str]] + document_subtypes: dict[str, str] + decisions: dict[str, dict[str, str]] + sessions: dict[str, Any] + ids: dict[str, dict[str, str]] + urls: dict[str, str] + sources: dict[str, Any] + embeddings: Embeddings + site: dict[str, Any] + path: Path + + # -- lookups ------------------------------------------------------------ + + @cached_property + def _by_code(self) -> dict[str, Party]: + return {p.code: p for p in self.parties} + + def party(self, code: Optional[str]) -> Optional[Party]: + return self._by_code.get((code or "").strip().upper()) + + def party_color(self, code: Optional[str]) -> str: + """Colour for a party code, or the neutral colour for unknown/independent.""" + found = self.party(code) + return found.color if found else self.party_defaults["unknown_color"] + + def party_highlight_color(self, code: Optional[str], amount: float = 0.75) -> str: + """A pale tint of the party colour, for text highlighting. + + Computed rather than configured — the predecessor kept a second + hand-maintained table of lightened colours that could drift from the first. + """ + found = self.party(code) + if not found: + return "#f0f0f0" + r, g, b = (int(found.color.lstrip("#")[i : i + 2], 16) for i in (0, 2, 4)) + blend = lambda c: round(c + (255 - c) * amount) # noqa: E731 + return f"#{blend(r):02x}{blend(g):02x}{blend(b):02x}" + + @cached_property + def party_codes(self) -> list[str]: + return [p.code for p in self.parties if p.active] + + def activity_title(self, code: Optional[str]) -> str: + return self.activity_types.get(code or "", {}).get("title", code or "") + + def person_photo_url(self, person_id: str) -> str: + return self.urls["person_photo"].format(person_id=person_id) + + def session_label(self, start_year: int) -> str: + """Render a session label, e.g. 2022 -> "2022/23".""" + return self.sessions["label_format"].format( + start=start_year, end_short=f"{(start_year + 1) % 100:02d}", end=start_year + 1 + ) + + def read_content(self, key: str) -> str: + """Read one of the markdown files referenced under `site:`.""" + rel = self.site.get(key) + if not rel: + return "" + path = _ROOT / rel + return path.read_text(encoding="utf-8") if path.exists() else "" + + # -- serialisation ------------------------------------------------------ + + def public_meta(self) -> dict[str, Any]: + """The payload served at GET /api/meta. + + Party colours ship to the client so the stylesheet does not have to + hardcode one rule per party — which is what made the previous CSS + Sweden-only in a way no configuration could fix. + """ + return { + "parliament": { + "name": self.meta.get("name"), + "name_en": self.meta.get("name_en"), + "country": self.meta.get("country"), + "data_start_year": self.meta.get("data_start_year"), + }, + "parties": [ + {"code": p.code, "name": p.name, "color": p.color, "active": p.active} + for p in self.parties + ], + "party_defaults": self.party_defaults, + "activity_types": self.activity_types, + "vocabulary": self.vocabulary, + "urls": self.urls, + "site": { + **{k: v for k, v in self.site.items() if not k.endswith("_file")}, + "explainer": self.read_content("explainer_file"), + "limit_warning": self.read_content("limit_warning_file"), + }, + } + + +def _require(data: dict, key: str) -> Any: + if key not in data: + raise ConfigError(f"parliament.yaml is missing the required `{key}:` section") + return data[key] + + +def load(path: Optional[Path] = None) -> Parliament: + """Read and validate a parliament configuration.""" + path = Path(path or os.environ.get("PARLIAMENT_CONFIG") or _ROOT / "parliament.yaml") + if not path.exists(): + raise ConfigError( + f"No parliament configuration at {path}. Copy parliament.yaml from the " + f"repository root, or set PARLIAMENT_CONFIG to point at yours." + ) + + data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} + + lang = Language(**_require(data, "language")) + if not _FTS_CONFIG_RE.match(lang.fts_config): + raise ConfigError( + f"language.fts_config {lang.fts_config!r} is not a valid Postgres " + f"identifier. It is interpolated into SQL, so it must match " + f"{_FTS_CONFIG_RE.pattern}. List valid names with: " + f"SELECT cfgname FROM pg_ts_config;" + ) + + parties = [Party(**p) for p in data.get("parties", [])] + if not parties: + raise ConfigError("parliament.yaml declares no parties") + + embeddings = Embeddings(**_require(data, "embeddings")) + if embeddings.dimension <= 0: + raise ConfigError("embeddings.dimension must be a positive integer") + + return Parliament( + meta=_require(data, "parliament"), + language=lang, + vocabulary=data.get("vocabulary", {}), + parties=parties, + party_defaults=data.get("party_defaults", {"unknown_color": "#9aa5b8"}), + activity_types=data.get("activity_types", {}), + document_subtypes=data.get("document_subtypes", {}), + decisions=data.get("decisions", {}), + sessions=data.get("sessions", {}), + ids=data.get("ids", {}), + urls=data.get("urls", {}), + sources=data.get("sources", {}), + embeddings=embeddings, + site=data.get("site", {}), + path=path, + ) + + +PARLIAMENT: Parliament = load() diff --git a/parliament.yaml b/parliament.yaml new file mode 100644 index 0000000..bc2cedc --- /dev/null +++ b/parliament.yaml @@ -0,0 +1,184 @@ +# Everything specific to one parliament, in one file. Nothing secret belongs here. +# +# This is the Swedish Riksdag configuration, shipped as the worked example. To +# target a different parliament, copy this file, change the values, and write an +# ingest adapter — see docs/PORTING.md. +# +# Override the path with PARLIAMENT_CONFIG=/etc/plenum/parliament.yaml so a +# deployment's own values live outside the repository and never appear in a diff. + +schema_version: 1 + +parliament: + id: se-riksdagen + name: Sveriges riksdag # in the language set below + name_en: Swedish Parliament + country: SE + chamber: unicameral # unicameral | lower | upper + open_data_url: https://data.riksdagen.se + data_start_year: 1993 + +language: + # Postgres text-search configuration. Must match the database's app.fts_config + # setting — the application refuses to start on a mismatch, because a silent + # mismatch returns near-zero rows with no error. + # See: SELECT cfgname FROM pg_ts_config; + fts_config: swedish + prompt_language: sv # selects prompts// and the answer language + locale: sv-SE + # Characters that must survive verbatim into search queries. Transliterating + # these silently breaks matching on most Swedish terms. + preserve_characters: "åäöÅÄÖ" + months: + januari: "01" + februari: "02" + mars: "03" + april: "04" + maj: "05" + juni: "06" + juli: "07" + augusti: "08" + september: "09" + oktober: "10" + november: "11" + december: "12" + +# Domain words injected into prompts, written in `prompt_language`. These exist +# because "motion", "yrkande" and "riksmöte" have no stable English equivalents; +# the database columns are neutral and the country's own word lives here. +vocabulary: + member: riksdagsledamot + member_plural: riksdagsledamöter + chamber: kammaren + speech: anförande + speech_plural: anföranden + document: motion + document_plural: motioner + proposal: yrkande + proposal_plural: yrkanden + committee: utskott + constituency: valkrets + session: riksmöte + +# Display order is list order. `active: false` keeps historical parties +# renderable without offering them as a filter. +parties: + - {code: S, name: Socialdemokraterna, color: "#E8112d", active: true} + - {code: M, name: Moderaterna, color: "#52BDEC", active: true} + - {code: SD, name: Sverigedemokraterna, color: "#DDDD00", active: true} + - {code: C, name: Centerpartiet, color: "#009933", active: true} + - {code: V, name: Vänsterpartiet, color: "#b51a0e", active: true} + - {code: KD, name: Kristdemokraterna, color: "#000077", active: true} + - {code: MP, name: Miljöpartiet de gröna, color: "#83CF39", active: true} + - {code: L, name: Liberalerna, color: "#006AB3", active: true} + - {code: NYD, name: Ny demokrati, color: "#ffff2b", active: false} + +party_defaults: + # Independents and unknowns. + unknown_color: "#9aa5b8" + # How a party code appears in rendered text, e.g. "Anna Andersson (MP)". + # Used to detect and verify attributions in generated answers. + code_pattern: "[A-ZÅÄÖ]{1,3}" + +# Keys are the values stored in talks.kammaraktivitet. Titles and descriptions +# are shown in the UI and are in `prompt_language`. +activity_types: + kam-vo: {title: Beslut, description: "Riksdagen fattar det formella beslutet i ärendet."} + bet: {title: Debatt om beslut, description: "Ledamöterna debatterar ett förslag innan beslut fattas."} + kam-fs: {title: Frågestund, description: "Ministrar svarar direkt på frågor från ledamöter."} + kam-ar: {title: Information från regeringen, description: "Regeringen informerar riksdagen i en aktuell fråga."} + ip: {title: Interpellationsdebatt, description: "Debatt om en skriftlig fråga till en minister."} + kam-sf: {title: Statsministerns frågestund, description: "Statsministern svarar på frågor från ledamöter."} + sam-ou: {title: Öppen utfrågning, description: "Utskottet frågar ut experter eller ansvariga personer offentligt."} + kam-ad: {title: Aktuell debatt, description: "Snabb debatt om en aktuell politisk fråga."} + kam-al: {title: Allmänpolitisk debatt, description: "Ledamöterna debatterar valfria politiska ämnen."} + kam-bu: {title: Budgetdebatt, description: "Debatt om regeringens budgetförslag."} + kam-bp: {title: Bordläggning, description: "Ett ärende anmäls i kammaren inför fortsatt behandling."} + kam-pd: {title: Partiledardebatt, description: "Partiledarna debatterar politikens stora frågor."} + kam-dv: {title: Debatt med anledning av vårpropositionen, description: "Debatt om regeringens ekonomiska vårproposition."} + sam-se: {title: Öppet seminarium, description: "Öppet seminarium eller samtal om ett särskilt tema."} + kam-ud: {title: Utrikespolitisk debatt, description: "Debatt om Sveriges utrikespolitik."} + +# Glosses for values stored in the data. The values themselves are never +# translated — only explained, so prompts can reason about them. +document_subtypes: + Enskild motion: "Motion från en eller ett fåtal ledamöter" + Kommittémotion: "Motion från ett partis utskottsgrupp" + Partimotion: "Motion från ett helt parti, ofta undertecknad av partiledaren" + +decisions: + Bifall: {label: Bifall, meaning: approved} + Avslag: {label: Avslag, meaning: rejected} + +# Annual parliamentary session. Deliberately not called "term": the European +# Parliament uses that for its five-year cycle, so it would be ambiguous. +sessions: + label_format: "{start}/{end_short}" # 2022 -> "2022/23" + starts_month: 9 + first_year: 1990 + +# Identifier shapes, templated into prompts and tool descriptions so the model +# can recognise and construct them without them being hardcoded in Python. +ids: + person_id: + example: "0448485371626" + pattern: "^[0-9]{10,13}$" + description: "Riksdagen intressent-id, a numeric string" + speech_id: + example: "GH09116-16" + pattern: "^[A-ZÅÄÖ0-9]+-[0-9]+$" + description: "{source_doc_id}-{sequence}" + doc_id: + example: "HD02846" + pattern: "^[A-ZÅÄÖ0-9]+$" + description: "Document id from the open-data portal" + debate_id: + example: "2021-06-17:42" + pattern: '^\d{4}-\d{2}-\d{2}:\d+$' + description: "{date}:{index within that day}" + +urls: + person_photo: "https://data.riksdagen.se/filarkiv/bilder/ledamot/{person_id}_192.jpg" + person_page: "https://www.riksdagen.se/sv/ledamoter-partier/ledamot/_{person_id}" + document_page: "https://www.riksdagen.se/sv/dokument-och-lagar/dokument/_{doc_id}" + +# Consumed by the ingest CLI. `adapter` names the module that knows this +# source's JSON shape; everything else is data. +sources: + adapter: ingest.adapters.riksdagen + speeches: + kind: zip-dataset + url_template: "https://data.riksdagen.se/dataset/anforande/anforande-{range}.json.zip" + dest_dir: talks + documents: + kind: zip-dataset + url_template: "https://data.riksdagen.se/dataset/dokument/mot-{range}.json.zip" + ranges: ["1990-1997", "1998-2001", "2002-2005", "2006-2009", + "2010-2013", "2014-2017", "2018-2021", "2022-2025"] + dest_dir: motioner + people: + kind: json + url: "https://data.riksdagen.se/personlista/?utformat=json" + dest_dir: personer + +embeddings: + model: qwen3-embedding # LLM_MODEL_EMBEDDING overrides + # Must equal the vector(N) width of the embedding columns. Checked at startup + # against the actual column type, because a mismatch fails deep inside pgvector + # with an error that says nothing about configuration. + dimension: 384 + base_url_env: EMBEDDING_BASE_URL + chunk_chars: 500 + +# Served to the frontend via GET /api/meta. +site: + title: rixdagen.se + tagline: "Sök i riksdagens anföranden och motioner" + # Markdown, so a deployment can rewrite its own copy without touching source. + explainer_file: content/sv/explainer.md + limit_warning_file: content/sv/limit_warning.md + # Upstream ships these empty. A deployment fills them in via its own + # PARLIAMENT_CONFIG file, which keeps personal contact details out of the repo. + contact: + email: null + url: null diff --git a/scripts/make_embeddings.py b/scripts/make_embeddings.py index bf7ca17..97c7ff0 100644 --- a/scripts/make_embeddings.py +++ b/scripts/make_embeddings.py @@ -27,6 +27,7 @@ logging.getLogger("httpx").setLevel(logging.WARNING) sys.path.insert(0, str(Path(__file__).resolve().parents[1])) import bootstrap # noqa: E402,F401 — sets cwd and sys.path to the project root +from parliament import PARLIAMENT from postgres_client import pg from utils import TextChunker @@ -36,7 +37,7 @@ logging.basicConfig( ) logger = logging.getLogger(__name__) -EMBED_DIM = 384 +EMBED_DIM = PARLIAMENT.embeddings.dimension EMBED_BATCH = 20 MAX_WORKERS = 3 INSERT_BATCH = 100