Add ingest adapter for the Riksdag's open data

ingest/adapters/riksdagen.py is the only module that knows what the Riksdag calls
its fields. Everything downstream works in plenum's column names, which is what
makes another parliament a matter of writing one file rather than editing the
application.

Writing it surfaced a bug the rename introduced: the loaders read the *source* JSON,
whose keys are the Riksdag's Swedish field names, and the mechanical pass renamed
those reads as if they were column names. doc.get("anforandetext") became
doc.get("text"), doc.get("talare") became doc.get("speaker_name"), and so on. The
next sync would have written empty speeches without erroring. This is exactly the
confusion an adapter exists to prevent, since it makes the source-name/column-name
boundary explicit instead of implicit.

The adapter captures what this source actually does, verified against real files:
repeated elements arrive as an object when there is one and a list when there are
several; null is serialised as the string "None"; speech text is HTML; and the
speaker field carries the party in parentheses, which has to come off or names stop
joining to the member register. Output was checked field by field against what
production stores for the same record.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>

ingest/__init__.py

@@ -0,0 +1,9 @@
+"""Fetch, adapt, and load a parliament's open data.
+
+    python -m ingest.cli fetch --source documents --range 2022-2025
+    python -m ingest.cli load  --source documents
+    python -m ingest.cli sync  --source all
+
+Sources are declared under `sources:` in parliament.yaml. The adapter named there
+is the only module that knows the source's own field names.
+"""

ingest/adapters/__init__.py

@@ -0,0 +1,16 @@
+"""Per-parliament adapters: source records in, plenum columns out."""
+from __future__ import annotations
+
+import importlib
+from typing import Any
+
+
+def load_adapter(module_path: str) -> Any:
+    """Import the adapter module named in parliament.yaml `sources.adapter`."""
+    try:
+        return importlib.import_module(module_path)
+    except ModuleNotFoundError as exc:
+        raise ModuleNotFoundError(
+            f"Could not import adapter {module_path!r} from parliament.yaml. "
+            f"See docs/PORTING.md for what an adapter must provide."
+        ) from exc

ingest/adapters/riksdagen.py

@@ -0,0 +1,273 @@
+"""Adapter for the Swedish Riksdag's open data (data.riksdagen.se).
+
+This module is the *only* place that knows what the Riksdag calls its fields.
+Everything downstream works in plenum's own column names. Copy this file as the
+starting point for another parliament — see docs/PORTING.md.
+
+Two things about this source that will probably bite you elsewhere too:
+
+* Its JSON is converted from XML, so a repeated element arrives as an object when
+  there is one of them and as a list when there are several. `_as_list` normalises
+  that; forgetting it produces code that works until the day a document has two
+  authors.
+* Null is serialised as the *string* `"None"`, which is truthy and will happily be
+  written to the database as text unless it is stripped.
+"""
+from __future__ import annotations
+
+import re
+from datetime import date, datetime
+from typing import Any, Iterable, Iterator, Optional
+
+# ── source quirks ─────────────────────────────────────────────────────────────
+
+
+def _as_list(value: Any) -> list:
+    """One child arrives as an object, several as a list, none as null."""
+    if value is None:
+        return []
+    return value if isinstance(value, list) else [value]
+
+
+def _clean(value: Any) -> Any:
+    """The API serialises null as the string "None"."""
+    if value is None or value == "None":
+        return None
+    return value
+
+
+def _parse_date(raw: Optional[str]) -> Optional[date]:
+    """Dates arrive as "YYYY-MM-DD HH:MM:SS", sometimes without the time."""
+    if not raw:
+        return None
+    for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%d"):
+        try:
+            return datetime.strptime(raw.strip()[: len(fmt) + 2], fmt).date()
+        except ValueError:
+            continue
+    return None
+
+
+def _session_start_year(session_label: Optional[str]) -> Optional[int]:
+    """"2022/23" -> 2022."""
+    if not session_label:
+        return None
+    try:
+        return int(str(session_label)[:4])
+    except ValueError:
+        return None
+
+
+_TITLE_PREFIXES = re.compile(
+    r"^(?:statsrådet|ministern|talman(?:nen)?|herr|fru)\s+", re.IGNORECASE
+)
+# The transcript appends the party in parentheses: "Mikael Dahlqvist (S)". The
+# party is stored in its own column, and leaving it here would break joins to the
+# member register and show up duplicated in the UI.
+_TRAILING_PARTY = re.compile(r"\s*\([A-ZÅÄÖ\-]{1,4}\)\s*$")
+
+
+def _clean_speaker(name: Optional[str]) -> Optional[str]:
+    """Strip honorifics and the trailing party, so names join to the register."""
+    if not name:
+        return None
+    name = _TITLE_PREFIXES.sub("", name.strip())
+    return _TRAILING_PARTY.sub("", name).strip() or None
+
+
+# ── speeches ──────────────────────────────────────────────────────────────────
+
+# Left: plenum column. Right: the Riksdag's field name. Written this way round
+# because the column names are the contract and the source names are the detail.
+SPEECH_FIELDS: dict[str, str] = {
+    "source_speech_id": "anforande_id",
+    "text": "anforandetext",
+    "section_title": "avsnittsrubrik",
+    "sequence": "anforande_nummer",
+    "activity_type": "kammaraktivitet",
+    "speaker_name": "talare",
+    "party": "parti",
+    "person_id": "intressent_id",
+    "source_datetime": "dok_datum",
+    "source_doc_id": "dok_id",
+    "related_doc_id": "rel_dok_id",
+    "source_doc_number": "dok_nummer",
+    "source_record_id": "dok_hangar_id",
+    "title": "dok_titel",
+}
+
+
+def adapt_speech(payload: dict) -> Optional[dict]:
+    """Turn one source record into a `speeches` row, or None if unusable."""
+    doc = payload.get("anforande") if "anforande" in payload else payload
+    if not doc:
+        return None
+
+    row = {col: _clean(doc.get(src)) for col, src in SPEECH_FIELDS.items()}
+
+    # The primary key is the protocol document plus the position within it. The
+    # source's own `anforande_id` is a UUID that changed in an earlier migration,
+    # so it is kept for reference but is not the key.
+    if not row["source_doc_id"] or row["sequence"] in (None, ""):
+        return None
+    row["id"] = f"{row['source_doc_id']}-{row['sequence']}"
+
+    row["speaker_name"] = _clean_speaker(row["speaker_name"])
+    # Speech text arrives as HTML fragments; the corpus stores plain text so that
+    # full-text search and snippet extraction do not have to strip tags per query.
+    row["text"] = _html_to_text(row["text"]) if row["text"] else None
+    row["date"] = _parse_date(row["source_datetime"])
+    row["year"] = _session_start_year(doc.get("dok_rm"))
+    row["session_year"] = row["year"]
+    # "Y"/"N", not a boolean.
+    row["is_reply"] = _clean(doc.get("replik")) == "Y"
+    return row
+
+
+# ── documents ─────────────────────────────────────────────────────────────────
+
+DOCUMENT_FIELDS: dict[str, str] = {
+    "doc_id": "dok_id",
+    "source_record_id": "hangar_id",
+    "session_label": "rm",
+    "designation": "beteckning",
+    "subtype": "subtyp",
+    "committee": "organ",
+    "status": "status",
+    "source_updated_at": "systemdatum",
+    "published_at": "publicerad",
+    "title": "titel",
+    "subtitle": "undertitel",
+    "url_text": "dokument_url_text",
+    "url_html": "dokument_url_html",
+}
+
+
+def adapt_document(payload: dict) -> Optional[dict]:
+    """Turn one `dokumentstatus` record into a `documents` row plus its children.
+
+    Returns a dict with keys `document`, `authors` and `proposals`; the caller
+    decides how to persist them.
+    """
+    status = payload.get("dokumentstatus") or payload
+    doc = status.get("dokument")
+    if not doc or not doc.get("dok_id"):
+        return None
+
+    row = {col: _clean(doc.get(src)) for col, src in DOCUMENT_FIELDS.items()}
+    row["doc_type"] = "motion"
+    row["date"] = _parse_date(doc.get("datum"))
+    row["session_year"] = _session_start_year(row["session_label"])
+
+    authors = []
+    for i, person in enumerate(_as_list((status.get("dokintressent") or {}).get("intressent"))):
+        authors.append({
+            "doc_id": row["doc_id"],
+            "ordinal": i,
+            "person_id": _clean(person.get("intressent_id")),
+            "name": _clean(person.get("namn")),
+            "party": _clean(person.get("partibet")),
+            "role": _clean(person.get("roll")),
+        })
+
+    proposals = []
+    for i, item in enumerate(_as_list((status.get("dokforslag") or {}).get("forslag"))):
+        proposals.append({
+            "id": f"{row['doc_id']}:{i}",
+            "doc_id": row["doc_id"],
+            "ordinal": i,
+            "number": _clean(item.get("nummer")),
+            "text": _clean(item.get("lydelse")),
+            "committee_recommendation": _clean(item.get("utskottet")),
+            "chamber_decision": _clean(item.get("kammaren")),
+            "handled_in": _clean(item.get("behandlas_i")),
+        })
+
+    # Denormalised so a party filter does not need a join.
+    row["parties"] = sorted({a["party"] for a in authors if a["party"]})
+    row["author_names"] = [a["name"] for a in authors if a["name"]]
+    row["num_proposals"] = len(proposals)
+    row["proposals_text"] = "\n".join(p["text"] for p in proposals if p["text"]) or None
+    row["proposals_raw"] = (status.get("dokforslag") or {}).get("forslag")
+    row["attachments"] = (status.get("dokbilaga") or {}).get("bilaga")
+
+    # `html` is the full text; documents that exist only as scanned PDFs have none,
+    # which is why has_text is stored rather than inferred at query time.
+    html = _clean(doc.get("html"))
+    row["text"] = _html_to_text(html) if html else None
+    row["has_text"] = bool(row["text"])
+    row["url_pdf"] = _pdf_url(status)
+
+    return {"document": row, "authors": authors, "proposals": proposals}
+
+
+def _html_to_text(html: str) -> Optional[str]:
+    from bs4 import BeautifulSoup
+
+    text = BeautifulSoup(html, "html.parser").get_text("\n")
+    return re.sub(r"\n{3,}", "\n\n", text).strip() or None
+
+
+def _pdf_url(status: dict) -> Optional[str]:
+    for attachment in _as_list((status.get("dokbilaga") or {}).get("bilaga")):
+        url = _clean(attachment.get("fil_url"))
+        if url and url.lower().endswith(".pdf"):
+            return url
+    return None
+
+
+# ── people ────────────────────────────────────────────────────────────────────
+
+PERSON_FIELDS: dict[str, str] = {
+    "person_id": "intressent_id",
+    "source_record_id": "hangar_id",
+    "source_record_guid": "hangar_guid",
+    "source_id": "sourceid",
+    "birth_year": "fodd_ar",
+    "gender": "kon",
+    "last_name": "efternamn",
+    "first_name": "tilltalsnamn",
+    "sort_name": "sorteringsnamn",
+    "home_town": "iort",
+    "party": "parti",
+    "constituency": "valkrets",
+    "status": "status",
+    "source_url": "person_url_xml",
+    "image_url_small": "bild_url_80",
+    "image_url_medium": "bild_url_192",
+    "image_url_large": "bild_url_max",
+}
+
+
+def adapt_person(payload: dict) -> Optional[dict]:
+    if not payload.get("intressent_id"):
+        return None
+    row = {col: _clean(payload.get(src)) for col, src in PERSON_FIELDS.items()}
+    row["name"] = " ".join(x for x in (row["first_name"], row["last_name"]) if x) or None
+    row["assignments"] = payload.get("personuppdrag")
+    row["contact_details"] = payload.get("personuppgift")
+    row["active"] = _clean(payload.get("status")) not in (None, "Avgången")
+    return row
+
+
+ADAPTERS = {
+    "speeches": adapt_speech,
+    "documents": adapt_document,
+    "people": adapt_person,
+}
+
+
+def adapt(kind: str, payload: dict) -> Optional[dict]:
+    """Adapt one record of the named kind."""
+    try:
+        return ADAPTERS[kind](payload)
+    except KeyError:
+        raise ValueError(f"Unknown record kind {kind!r}; expected one of {sorted(ADAPTERS)}")
+
+
+def adapt_many(kind: str, payloads: Iterable[dict]) -> Iterator[dict]:
+    """Adapt a stream, skipping records the source could not supply usably."""
+    for payload in payloads:
+        row = adapt(kind, payload)
+        if row is not None:
+            yield row