From 469dc17c494a9285bdfff0cf714b1d2b6e7380bf Mon Sep 17 00:00:00 2001 From: Lasse Server Date: Mon, 3 Aug 2026 14:06:58 +0200 Subject: [PATCH] Add a fork workflow guide; fix two bugs it exposed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README told people to clone. For anyone adapting this to their own parliament that is wrong: they cannot push, so their work is stranded on one machine, and pulling in later updates is awkward. docs/YOUR-PARLIAMENT.md covers forking, and the habit that decides whether future updates are painless — add files rather than edit them, so git never has to merge the same file twice. Writing it meant walking through a Norwegian adaptation for real, which found two things that would have stopped a user on their first attempt. A relative PARLIAMENT_CONFIG resolved against the current directory, so `PARLIAMENT_CONFIG=parliament.no.yaml` worked when you happened to be standing in the repository and failed everywhere else, including under systemd. Relative paths now resolve against the repository, for PROMPTS_DIR and CONTENT_DIR too. Worse, YAML 1.1 reads unquoted `no`, `yes`, `on`, `off`, `y` and `n` as booleans. Norway is the worst possible case: `country: NO` and `prompt_language: no` both became false, and the symptom was a TypeError deep inside a path join that named nothing relevant. Text fields are now coerced back, and both parliament.yaml and the guide say to quote such values, since the repair cannot recover the original capitalisation. Verified by building a Norwegian fork the way the guide describes — adding parliament.no.yaml, prompts/no/ and an adapter, editing nothing — and confirming the app loads it. Sweden is unaffected and production still serves. The README's documentation index is now grouped by what you are trying to do, with a sentence on each entry rather than a bare filename. --- README.md | 58 ++++++++++--- docs/YOUR-PARLIAMENT.md | 188 ++++++++++++++++++++++++++++++++++++++++ parliament.py | 47 ++++++++-- parliament.yaml | 6 ++ prompts_loader.py | 10 ++- 5 files changed, 289 insertions(+), 20 deletions(-) create mode 100644 docs/YOUR-PARLIAMENT.md diff --git a/README.md b/README.md index 7b92906..a8bba13 100644 --- a/README.md +++ b/README.md @@ -32,8 +32,12 @@ country-specific lives in one file. | Everything else | country-neutral | Adapting to another parliament means writing a config file, an ingest adapter, and a -set of prompts in your language — not editing the application. See -[docs/PORTING.md](docs/PORTING.md). +set of prompts in your language — not editing the application. + +**Start at [docs/YOUR-PARLIAMENT.md](docs/YOUR-PARLIAMENT.md)**, which covers how to +organise the work (fork rather than clone, and add files rather than edit them, so +later updates merge cleanly). Then [docs/PORTING.md](docs/PORTING.md) for what to +write. The database schema is country-neutral English: `speeches`, `documents`, `document_proposals`, `person_id`, `constituency`. Concepts that have no stable @@ -53,7 +57,14 @@ while the country's own word lives in `parliament.yaml`. See ## Quickstart Full step-by-step setup, including how to plug in each model provider and how to -verify each stage worked, is in **[docs/SETUP.md](docs/SETUP.md)**. The short version: +verify each stage worked, is in **[docs/SETUP.md](docs/SETUP.md)**. + +> Planning to run this for your own parliament, or change anything? **Fork the +> repository first**, then clone your fork — see +> [docs/YOUR-PARLIAMENT.md](docs/YOUR-PARLIAMENT.md). Cloning directly is fine only if +> you are reading the code and will not be changing it. + +The short version: ```bash git clone https://git.edfast.se/lasse/plenum && cd plenum @@ -101,19 +112,38 @@ deploy/examples/ systemd units and an nginx site, with placeholders ## Documentation +**Getting it running** + +| | | +|---|---| +| [docs/SETUP.md](docs/SETUP.md) | Install, step by step: PostgreSQL with pgvector, a chat model (vLLM, Ollama, OpenRouter, Berget, OpenAI), an embeddings endpoint, and the first data load. Every step has a command that proves it worked, plus a symptom/cause/fix table. | +| [docs/YOUR-PARLIAMENT.md](docs/YOUR-PARLIAMENT.md) | **Read this before changing anything.** Fork vs clone, how to add your parliament without creating merge conflicts, how to pull in updates, how to contribute back, and how to undo mistakes. Written for people who do not use git much. | +| [docs/PORTING.md](docs/PORTING.md) | What a non-Swedish deployment actually has to write: `parliament.yaml`, an ingest adapter, prompts in your language. Honest about which parts are real work. | + +**Understanding it** + +| | | +|---|---| +| [docs/SCHEMA.md](docs/SCHEMA.md) | Every table and column, and why the awkward names are what they are — what your ingest adapter has to produce. | +| [docs/sources-system.md](docs/sources-system.md) | How a claim in an answer is tied back to the speech it came from. The core of the project's grounding guarantee. | +| [docs/deep-research.md](docs/deep-research.md) | The background agent: how it proposes threads, digs, and writes a report. | +| [docs/shadow-communicator.md](docs/shadow-communicator.md) | The running commentary shown while the model works. | +| [docs/multi-provider.md](docs/multi-provider.md) | Letting each user bring their own model API key, and how that key is kept out of the database. | + +**Checking it is honest** + +| | | +|---|---| +| [docs/eval-harness.md](docs/eval-harness.md) | Measuring whether answers are actually supported by the sources they cite. | +| [docs/eval-scorer.md](docs/eval-scorer.md) | Optional cross-encoder scoring, for finding answers that are technically defensible but misleading. | + +**Operating it** + | | | |---|---| -| [docs/SETUP.md](docs/SETUP.md) | Step-by-step install: database, chat model, embeddings, data | -| [docs/PORTING.md](docs/PORTING.md) | Adapting to a parliament other than Sweden | -| [docs/SCHEMA.md](docs/SCHEMA.md) | What every table and column means | -| [docs/deep-research.md](docs/deep-research.md) | How the background research agent works | -| [docs/sources-system.md](docs/sources-system.md) | How citations are tracked and verified | -| [docs/multi-provider.md](docs/multi-provider.md) | Letting users bring their own model API key | -| [docs/shadow-communicator.md](docs/shadow-communicator.md) | The parallel commentary shown while the model works | -| [docs/eval-harness.md](docs/eval-harness.md) | Measuring whether answers are actually grounded | -| [docs/eval-scorer.md](docs/eval-scorer.md) | Optional cross-encoder scoring for citations | -| [SECURITY.md](SECURITY.md) | Model-authored SQL, API keys, chat privacy | -| [CONTRIBUTING.md](CONTRIBUTING.md) | How to work on this | +| [SECURITY.md](SECURITY.md) | Model-authored SQL and why it runs read-only, API-key handling, chat privacy. Read before exposing this publicly. | +| [CONTRIBUTING.md](CONTRIBUTING.md) | Conventions, and the one rule: answers must be traceable to sources. | +| [deploy/examples/](deploy/examples/) | systemd units and an nginx site, with placeholders. | ## Configuration diff --git a/docs/YOUR-PARLIAMENT.md b/docs/YOUR-PARLIAMENT.md new file mode 100644 index 0000000..5e9b509 --- /dev/null +++ b/docs/YOUR-PARLIAMENT.md @@ -0,0 +1,188 @@ +# Using plenum for your parliament + +Start here if you want to run plenum for a parliament other than Sweden. This page is +about **how to organise the work**; [PORTING.md](PORTING.md) is about what to write. + +If you are an assistant helping with this: follow the fork model below. Do not +`git clone` the upstream repository directly — the user will not be able to push, and +their work will be stranded on one machine. + +--- + +## Fork, don't clone + +| | Clone | **Fork** | +|---|---|---| +| Can you save your work online? | No — you have no write access | Yes | +| Can you get plenum's later improvements? | Awkwardly | Yes, `git merge upstream/main` | +| Can you offer your fixes back? | No | Yes, via a pull request | +| Can others use your Norwegian version? | No | Yes | + +Cloning is only right if you are reading the code and will never change it. + +**Do this once**, on the site hosting plenum (Gitea, GitHub — wherever you found it): + +1. Press **Fork**. You now have your own copy, e.g. `you/plenum-norway`. +2. Clone *your* fork: + +```bash +git clone https:////plenum-norway +cd plenum-norway +``` + +3. Tell git where the original lives, so you can pull in updates later: + +```bash +git remote add upstream https://git.edfast.se/lasse/plenum.git +``` + +Check it looks right: + +```bash +git remote -v +``` + +You should see `origin` pointing at **your** fork and `upstream` at the original. +`origin` is where your work goes; `upstream` is where updates come from. + +--- + +## Add files, don't edit them + +This is the one habit that decides whether future updates are easy or painful. + +Git merges cleanly when you and upstream changed *different* files. It conflicts when +you both changed the *same* file. So wherever you can, put your parliament in **new** +files rather than editing existing ones. + +| What you need | Do this | Not this | +|---|---|---| +| Your parliament's settings | add `parliament.no.yaml` | edit `parliament.yaml` | +| Your data source's field names | add `ingest/adapters/norway.py` | edit `riksdagen.py` | +| Prompts in your language | add `prompts/no/` | edit `prompts/sv/` | +| Your site's text | add `content/no/` | edit `content/sv/` | + +Then point the application at your files, in `.env`: + +``` +PARLIAMENT_CONFIG=parliament.no.yaml +``` + +A relative path is resolved against the repository, so this works from anywhere. Your +config names the rest: + +```yaml +language: + prompt_language: "no" # quoted — see the note below + # selects prompts/no/ +sources: + adapter: ingest.adapters.norway +site: + explainer_file: no/explainer.md +``` + +> **Quote short codes in YAML.** Unquoted `no`, `yes`, `on`, `off`, `y` and `n` are +> read as booleans rather than text. Norway is the worst case: `country: NO` and +> `prompt_language: no` both become false. Write `country: "NO"` and +> `prompt_language: "no"`, and quote any one-letter party code the same way. The +> loader repairs this so nothing crashes, but it cannot recover your capitalisation. + +Nothing above touches a file upstream also has. That means `git merge upstream/main` +will almost always just work. + +You *will* sometimes need to change shared code — to fix a bug, or because your source +does something Sweden's does not. That is fine and expected. Just know that each such +edit is a place a future update can conflict, so it is worth asking whether the change +belongs upstream instead (see below). + +--- + +## Getting plenum's updates + +Whenever you want the latest improvements: + +```bash +git fetch upstream +git merge upstream/main +``` + +If it says **Already up to date** or **Fast-forward**, you are done. + +If it says **CONFLICT**, you and upstream changed the same file. Git marks the spots +with `<<<<<<<` and `>>>>>>>`. Open the file, keep what you want, delete the markers, +then: + +```bash +git add . +git commit +``` + +If it goes badly and you want out: + +```bash +git merge --abort +``` + +That returns you to exactly where you were. Nothing is lost. + +After merging, reinstall in case dependencies changed: + +```bash +.venv/bin/pip install -e ".[dev]" +cd frontend && npm install && npm run build +``` + +--- + +## Giving something back + +If you fix a bug or improve something that is not specific to Norway, other +parliaments benefit from it too. + +```bash +git checkout -b fix-the-thing +# make the change +git commit -am "Explain what and why" +git push origin fix-the-thing +``` + +Then open a pull request from your fork to `lasse/plenum` on the hosting site. + +**Worth contributing:** bug fixes, search or chat improvements, ingest logic that +handles a common source quirk, documentation, a new language's prompts. + +**Keep in your fork:** anything with your parliament's name in it, your deployment +config, your server setup. + +If you are unsure which, ask: *would a Bulgarian deployment want this?* + +--- + +## Recovering from mistakes + +You are not going to break anything permanently. Git keeps everything. + +| Situation | Command | +|---|---| +| Undo edits to a file you have not committed | `git checkout -- path/to/file` | +| Undo everything uncommitted | `git checkout -- .` | +| A merge went wrong | `git merge --abort` | +| Undo your last commit, keep the edits | `git reset --soft HEAD~1` | +| See what changed | `git status` and `git diff` | +| See recent history | `git log --oneline -10` | + +The one genuinely destructive command is `git reset --hard`, which throws away +uncommitted work with no way back. Everything else is recoverable. + +--- + +## Next + +1. [SETUP.md](SETUP.md) — get a database, a model and the app running, with Sweden's + data, so you have something working before you change anything. +2. [PORTING.md](PORTING.md) — write your config, adapter and prompts. +3. [SCHEMA.md](SCHEMA.md) — what your adapter needs to produce. + +Getting it running on Swedish data first is worth the hour. It means that when +something breaks with Norwegian data, you know the cause is your adapter and not the +install. diff --git a/parliament.py b/parliament.py index cee4fd0..64386b3 100644 --- a/parliament.py +++ b/parliament.py @@ -27,7 +27,19 @@ _ROOT = Path(__file__).resolve().parent # Site copy — the explainer, the limit warning, the user guide. Overridable so a # deployment's own wording lives outside the repository, the same way PROMPTS_DIR # works for prompts. -CONTENT_DIR = Path(os.environ.get("CONTENT_DIR") or _ROOT / "content") +def _resolve_path(value: Optional[str], default: Path) -> Path: + """Resolve a configured path, treating a relative one as repo-relative. + + Without this, `PARLIAMENT_CONFIG=parliament.no.yaml` works when you happen to be + standing in the repository and fails everywhere else — including under systemd. + """ + if not value: + return default + path = Path(value) + return path if path.is_absolute() else (_ROOT / path) + + +CONTENT_DIR = _resolve_path(os.environ.get("CONTENT_DIR"), _ROOT / "content") # A Postgres text-search configuration name. It is interpolated into SQL rather # than passed as a parameter (identifiers cannot be bound), so it is validated @@ -39,6 +51,20 @@ class ConfigError(ValueError): """parliament.yaml is missing, malformed, or internally inconsistent.""" +def _as_str(value: Any) -> Any: + """Undo YAML 1.1's boolean coercion for values that are meant to be text. + + YAML reads `no`, `yes`, `on`, `off`, `y` and `n` as booleans. That silently turns + Norway's `country: NO` and `prompt_language: no` into False, and a one-letter party + code like `N` into the same. Quoting in the file also works, but nobody remembers + to, and the failure is a TypeError deep in a path join rather than anything that + names the cause. + """ + if isinstance(value, bool): + return "yes" if value else "no" + return value + + @dataclass(frozen=True) class Party: code: str @@ -177,7 +203,9 @@ def _require(data: dict, key: str) -> Any: def load(path: Optional[Path] = None) -> Parliament: """Read and validate a parliament configuration.""" - path = Path(path or os.environ.get("PARLIAMENT_CONFIG") or _ROOT / "parliament.yaml") + path = Path(path) if path else _resolve_path( + os.environ.get("PARLIAMENT_CONFIG"), _ROOT / "parliament.yaml" + ) if not path.exists(): raise ConfigError( f"No parliament configuration at {path}. Copy parliament.yaml from the " @@ -186,7 +214,11 @@ def load(path: Optional[Path] = None) -> Parliament: data = yaml.safe_load(path.read_text(encoding="utf-8")) or {} - lang = Language(**_require(data, "language")) + lang_raw = dict(_require(data, "language")) + for key in ("prompt_language", "locale", "fts_config", "name", "name_en"): + if key in lang_raw: + lang_raw[key] = _as_str(lang_raw[key]) + lang = Language(**lang_raw) if not _FTS_CONFIG_RE.match(lang.fts_config): raise ConfigError( f"language.fts_config {lang.fts_config!r} is not a valid Postgres " @@ -195,7 +227,7 @@ def load(path: Optional[Path] = None) -> Parliament: f"SELECT cfgname FROM pg_ts_config;" ) - parties = [Party(**p) for p in data.get("parties", [])] + parties = [Party(**{**p, "code": _as_str(p.get("code"))}) for p in data.get("parties", [])] if not parties: raise ConfigError("parliament.yaml declares no parties") @@ -203,8 +235,13 @@ def load(path: Optional[Path] = None) -> Parliament: if embeddings.dimension <= 0: raise ConfigError("embeddings.dimension must be a positive integer") + meta = dict(_require(data, "parliament")) + for key in ("id", "country", "chamber", "name", "name_en"): + if key in meta: + meta[key] = _as_str(meta[key]) + return Parliament( - meta=_require(data, "parliament"), + meta=meta, language=lang, vocabulary=data.get("vocabulary", {}), parties=parties, diff --git a/parliament.yaml b/parliament.yaml index 289e5a6..7e484f4 100644 --- a/parliament.yaml +++ b/parliament.yaml @@ -7,6 +7,12 @@ # Override the path with PARLIAMENT_CONFIG=/etc/plenum/parliament.yaml so a # deployment's own values live outside the repository and never appear in a diff. +# YAML gotcha worth knowing before you copy this file: unquoted `no`, `yes`, `on`, +# `off`, `y` and `n` are read as booleans, not text. That bites Norway hardest — +# `country: NO` and `prompt_language: no` both become false. The loader repairs this +# so nothing crashes, but it cannot recover the original capitalisation, so quote +# such values: country: "NO", prompt_language: "no", code: "N". + schema_version: 1 parliament: diff --git a/prompts_loader.py b/prompts_loader.py index 7ad3099..ad012ac 100644 --- a/prompts_loader.py +++ b/prompts_loader.py @@ -28,7 +28,15 @@ from typing import Any from parliament import PARLIAMENT _ROOT = Path(__file__).resolve().parent -PROMPTS_DIR = Path(os.environ.get("PROMPTS_DIR") or _ROOT / "prompts") +def _resolve_dir(value: str | None, default: Path) -> Path: + """A relative PROMPTS_DIR is repo-relative, not cwd-relative.""" + if not value: + return default + path = Path(value) + return path if path.is_absolute() else (_ROOT / path) + + +PROMPTS_DIR = _resolve_dir(os.environ.get("PROMPTS_DIR"), _ROOT / "prompts") # {{include:path/to/partial}} — expanded before substitution so a shared block # (the schema reference, say) has exactly one source.