You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
78 lines
3.0 KiB
78 lines
3.0 KiB
"""Command-line entry point for ingesting a parliament's open data. |
|
|
|
python -m ingest.cli fetch --source documents --range 2022-2025 |
|
python -m ingest.cli load --source documents |
|
python -m ingest.cli sync --source all |
|
|
|
`fetch` downloads, `load` adapts and upserts what is on disk, `sync` does both. |
|
Everything is resumable: archives already unpacked are skipped, and inserts use |
|
ON CONFLICT DO NOTHING, so re-running after an interruption is safe. |
|
""" |
|
from __future__ import annotations |
|
|
|
import argparse |
|
import sys |
|
from pathlib import Path |
|
|
|
sys.path.insert(0, str(Path(__file__).resolve().parents[1])) |
|
|
|
from ingest import pipeline # noqa: E402 |
|
from ingest.adapters import load_adapter # noqa: E402 |
|
from parliament import PARLIAMENT # noqa: E402 |
|
|
|
SOURCES = ("speeches", "documents", "people") |
|
|
|
|
|
def _adapter(): |
|
module_path = PARLIAMENT.sources.get("adapter") |
|
if not module_path: |
|
raise SystemExit("parliament.yaml has no `sources.adapter`. See docs/PORTING.md.") |
|
return load_adapter(module_path) |
|
|
|
|
|
def _resolve(source: str) -> list[str]: |
|
if source == "all": |
|
return [s for s in SOURCES if s in PARLIAMENT.sources] |
|
if source not in SOURCES: |
|
raise SystemExit(f"Unknown source {source!r}; expected one of {SOURCES + ('all',)}") |
|
return [source] |
|
|
|
|
|
def main(argv: list[str] | None = None) -> int: |
|
parser = argparse.ArgumentParser(prog="ingest", description=__doc__) |
|
parser.add_argument("command", choices=("fetch", "load", "sync")) |
|
parser.add_argument("--source", default="all", |
|
help=f"one of {', '.join(SOURCES)}, or 'all' (default)") |
|
parser.add_argument("--range", dest="ranges", action="append", |
|
help="archive range to fetch, e.g. 2022-2025. Repeatable. " |
|
"Defaults to the ranges in parliament.yaml.") |
|
parser.add_argument("--limit", type=int, |
|
help="stop after this many records — useful for a smoke test") |
|
args = parser.parse_args(argv) |
|
|
|
adapter = _adapter() |
|
print(f"parliament: {PARLIAMENT.meta.get('name')} adapter: {PARLIAMENT.sources['adapter']}") |
|
|
|
exit_code = 0 |
|
for source in _resolve(args.source): |
|
print(f"\n── {source} ──") |
|
try: |
|
if args.command in ("fetch", "sync"): |
|
pipeline.fetch(source, args.ranges) |
|
if args.command in ("load", "sync"): |
|
counts = pipeline.load(source, adapter.adapt, limit=args.limit) |
|
print(f" read {counts['read']}, wrote {counts['written']}, " |
|
f"skipped {counts['skipped']}") |
|
except Exception as exc: |
|
# One failing source should not abandon the others; a daily sync that |
|
# aborts because one archive moved is worse than a partial one. |
|
print(f" FAILED: {type(exc).__name__}: {exc}") |
|
exit_code = 1 |
|
|
|
if args.command in ("load", "sync"): |
|
print("\nNext: python scripts/make_embeddings.py") |
|
return exit_code |
|
|
|
|
|
if __name__ == "__main__": |
|
sys.exit(main())
|
|
|