You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
184 lines
6.8 KiB
184 lines
6.8 KiB
"""Export opt-in TEST conversations logged in Postgres for LLM analysis. |
|
|
|
Reads the eval_conversations table (written by |
|
backend/services/eval_conversation_logger.py when a conversation starts |
|
with "TEST ") and renders sessions either as Markdown designed to be |
|
pasted into an analysis LLM, or as raw JSON for programmatic judging. |
|
|
|
Usage: |
|
python scripts/eval_conversation_export.py --list |
|
python scripts/eval_conversation_export.py --session-id SID |
|
python scripts/eval_conversation_export.py --session-id SID --format json --out out.json |
|
python scripts/eval_conversation_export.py --key ROWID |
|
""" |
|
from __future__ import annotations |
|
|
|
import argparse |
|
import json |
|
import sys |
|
from pathlib import Path |
|
from typing import Any, Dict, List |
|
|
|
_ROOT = Path(__file__).resolve().parent.parent |
|
sys.path.insert(0, str(_ROOT)) |
|
|
|
from dotenv import load_dotenv |
|
load_dotenv() |
|
|
|
from postgres_client import pg |
|
|
|
|
|
def list_sessions() -> List[Dict[str, Any]]: |
|
return pg.execute( |
|
""" |
|
SELECT session_id, |
|
COUNT(*) AS turns, |
|
MAX(started_at) AS last, |
|
SUM(has_error::int) AS errors |
|
FROM eval_conversations |
|
GROUP BY session_id |
|
ORDER BY last DESC |
|
""" |
|
) |
|
|
|
|
|
def fetch_turns(session_id: str) -> List[Dict[str, Any]]: |
|
rows = pg.execute( |
|
"SELECT doc FROM eval_conversations WHERE session_id = %s ORDER BY started_at ASC", |
|
(session_id,), |
|
) |
|
return [r["doc"] for r in rows] |
|
|
|
|
|
def fetch_by_key(key: str) -> List[Dict[str, Any]]: |
|
rows = pg.execute("SELECT doc FROM eval_conversations WHERE id = %s", (int(key),)) |
|
return [r["doc"] for r in rows] |
|
|
|
|
|
def _preview(text: Any, limit: int = 400) -> str: |
|
s = str(text or "").replace("\n", " ").strip() |
|
return s if len(s) <= limit else s[:limit] + " (…)" |
|
|
|
|
|
def _render_event(e: Dict[str, Any]) -> str: |
|
t_s = e.get("t_ms", 0) / 1000 |
|
etype = e.get("type") |
|
if etype == "tool_call": |
|
args = json.dumps(e.get("args") or {}, ensure_ascii=False) |
|
return f"- `{t_s:7.1f}s` **tool_call #{e.get('iteration', '?')}** `{e.get('tool')}` args: {_preview(args)}" |
|
if etype == "tool_result": |
|
return ( |
|
f"- `{t_s:7.1f}s` tool_result `{e.get('tool')}` " |
|
f"({e.get('chars', '?')} chars): {_preview(e.get('content'), 300)}" |
|
) |
|
if etype == "status": |
|
return f"- `{t_s:7.1f}s` status: {_preview(e.get('message'))}" |
|
if etype == "insight": |
|
return f"- `{t_s:7.1f}s` insight: {_preview(e.get('message'))}" |
|
if etype == "search_card": |
|
ids = [r.get("_id") if isinstance(r, dict) else r for r in (e.get("results") or [])] |
|
return ( |
|
f"- `{t_s:7.1f}s` search_card query={_preview(e.get('query'), 120)!r} " |
|
f"total={e.get('total')} hits={ids}" |
|
) |
|
if etype == "stats_card": |
|
return f"- `{t_s:7.1f}s` stats_card rows={len(e.get('rows') or [])}" |
|
if etype == "tool_speakers": |
|
return f"- `{t_s:7.1f}s` tool_speakers: {e.get('person_ids')}" |
|
return f"- `{t_s:7.1f}s` {etype}: {_preview({k: v for k, v in e.items() if k not in ('i', 't_ms', 'type')})}" |
|
|
|
|
|
def render_markdown(turns: List[Dict[str, Any]]) -> str: |
|
if not turns: |
|
return "No logged turns found." |
|
lines: List[str] = [f"# Eval conversation `{turns[0].get('session_id')}`", ""] |
|
for doc in turns: |
|
lines.append(f"## Turn {doc.get('turn_index')} — {doc.get('started_at')}") |
|
lines.append( |
|
f"*duration: {doc.get('duration_s')}s · iterations: {doc.get('iterations')} · " |
|
f"stream: {doc.get('stream')} · provider: " |
|
f"{json.dumps((doc.get('request') or {}).get('provider'), ensure_ascii=False)}*" |
|
) |
|
lines.append("") |
|
messages = (doc.get("request") or {}).get("messages") or [] |
|
user_msgs = [m for m in messages if m.get("role") == "user"] |
|
if user_msgs: |
|
lines.append(f"**User:** {user_msgs[-1].get('content')}") |
|
lines.append("") |
|
lines.append("### Timeline") |
|
for e in doc.get("events") or []: |
|
lines.append(_render_event(e)) |
|
lines.append("") |
|
result = doc.get("result") |
|
if result: |
|
lines.append("### Answer") |
|
lines.append(result.get("answer") or "*(empty)*") |
|
lines.append("") |
|
sources = result.get("sources") or [] |
|
if sources: |
|
lines.append("### Sources") |
|
for n, s in enumerate(sources, 1): |
|
lines.append( |
|
f"{n}. `{s.get('_id')}` {s.get('speaker') or '?'} " |
|
f"({s.get('party') or '?'}, {s.get('date') or '?'}): " |
|
f"{_preview(s.get('snippet'), 200)}" |
|
) |
|
lines.append("") |
|
warnings = result.get("attribution_warnings") or [] |
|
if warnings: |
|
lines.append("### Attribution warnings") |
|
for w in warnings: |
|
lines.append(f"- {json.dumps(w, ensure_ascii=False)}") |
|
lines.append("") |
|
error = doc.get("error") |
|
if error: |
|
lines.append("### ERROR") |
|
lines.append(f"**{error.get('type')}**: {error.get('message')}") |
|
lines.append("```") |
|
lines.append((error.get("traceback") or "").strip()) |
|
lines.append("```") |
|
lines.append("") |
|
return "\n".join(lines) |
|
|
|
|
|
def main() -> None: |
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) |
|
parser.add_argument("--list", action="store_true", help="List logged sessions.") |
|
parser.add_argument("--session-id", help="Export all turns for a session.") |
|
parser.add_argument("--key", help="Export a single turn doc by _key.") |
|
parser.add_argument("--format", choices=["md", "json"], default="md") |
|
parser.add_argument("--out", help="Write output to file instead of stdout.") |
|
args = parser.parse_args() |
|
|
|
if args.list: |
|
sessions = list_sessions() |
|
if not sessions: |
|
print("No logged conversations.") |
|
return |
|
print(f"{'session_id':<40} {'turns':>5} {'errors':>6} last activity") |
|
for s in sessions: |
|
print(f"{s['session_id']:<40} {s['turns']:>5} {s['errors']:>6} {s['last']}") |
|
return |
|
|
|
if args.session_id: |
|
turns = fetch_turns(args.session_id) |
|
elif args.key: |
|
turns = fetch_by_key(args.key) |
|
else: |
|
parser.error("one of --list, --session-id or --key is required") |
|
return |
|
|
|
if args.format == "json": |
|
output = json.dumps(turns, ensure_ascii=False, indent=2, default=str) |
|
else: |
|
output = render_markdown(turns) |
|
|
|
if args.out: |
|
Path(args.out).write_text(output, encoding="utf-8") |
|
print(f"Wrote {len(turns)} turn(s) to {args.out}") |
|
else: |
|
print(output) |
|
|
|
|
|
if __name__ == "__main__": |
|
main()
|
|
|