"""Rich CLI viewer for semantic search benchmark JSONL artifacts.
Usage:
python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl
python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl --sort-by avg_latency_ms
python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl --filter-combo sqlite
python test-int/semantic/report.py .benchmarks/semantic-quality.jsonl --filter-suite paraphrase
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from rich.console import Console
from rich.table import Table
from rich.text import Text
def load_benchmarks(path: Path) -> list[dict]:
"""Load benchmark records from a JSONL file."""
records = []
with path.open("r", encoding="utf-8") as f:
for line in f:
line = line.strip()
if line:
records.append(json.loads(line))
return records
def _quality_cell(value: float) -> Text:
"""Color-code a quality metric value."""
text = f"{value:.3f}"
if value >= 0.90:
return Text(text, style="bold green")
if value >= 0.75:
return Text(text, style="green")
if value >= 0.50:
return Text(text, style="yellow")
if value > 0.0:
return Text(text, style="red")
return Text(text, style="dim red")
def _latency_cell(value: float) -> Text:
"""Color-code a latency value (ms)."""
text = f"{value:.1f}"
if value <= 5.0:
return Text(text, style="bold green")
if value <= 20.0:
return Text(text, style="green")
if value <= 100.0:
return Text(text, style="yellow")
return Text(text, style="red")
SORT_KEYS = {
"combo": lambda r: r["metrics"]["combo"],
"suite": lambda r: r["metrics"]["suite"],
"mode": lambda r: r["metrics"]["mode"],
"hit_at_1": lambda r: -r["metrics"].get("hit_at_1", 0),
"recall_at_5": lambda r: -r["metrics"].get("recall_at_5", 0),
"mrr_at_10": lambda r: -r["metrics"].get("mrr_at_10", 0),
"avg_latency_ms": lambda r: r["metrics"].get("avg_latency_ms", 0),
"total_time_ms": lambda r: r["metrics"].get("total_time_ms", 0),
}
def build_table(records: list[dict], title: str = "Semantic Search Benchmarks") -> Table:
"""Build a rich Table from benchmark records."""
table = Table(title=title, show_lines=False)
table.add_column("Combo", style="cyan", no_wrap=True)
table.add_column("Suite", style="magenta", no_wrap=True)
table.add_column("Mode", style="blue", no_wrap=True)
table.add_column("N", justify="right")
table.add_column("hit@1", justify="right", no_wrap=True)
table.add_column("R@5", justify="right", no_wrap=True)
table.add_column("MRR@10", justify="right", no_wrap=True)
table.add_column("avg ms", justify="right", no_wrap=True)
table.add_column("total ms", justify="right", no_wrap=True)
for rec in records:
m = rec["metrics"]
table.add_row(
m["combo"],
m["suite"],
m["mode"],
str(m["cases"]),
_quality_cell(m.get("hit_at_1", 0)),
_quality_cell(m.get("recall_at_5", 0)),
_quality_cell(m.get("mrr_at_10", 0)),
_latency_cell(m.get("avg_latency_ms", 0)),
_latency_cell(m.get("total_time_ms", 0)),
)
return table
def build_summary_table(records: list[dict]) -> Table:
"""Build a summary table comparing combos across suites."""
# Group by combo
combos: dict[str, dict[str, dict]] = {}
for rec in records:
m = rec["metrics"]
key = m["combo"]
if key not in combos:
combos[key] = {}
suite_mode = f"{m['suite']}/{m['mode']}"
combos[key][suite_mode] = m
table = Table(title="Summary: Best Recall@5 by Combo", show_lines=False)
table.add_column("Combo", style="cyan", no_wrap=True)
table.add_column("Lexical (best)", justify="right")
table.add_column("Paraphrase (best)", justify="right")
table.add_column("Avg Latency (best)", justify="right")
for combo_name, suite_modes in sorted(combos.items()):
# Find best recall@5 for lexical and paraphrase
lexical_best = max(
(v.get("recall_at_5", 0) for k, v in suite_modes.items() if k.startswith("lexical")),
default=0,
)
paraphrase_best = max(
(v.get("recall_at_5", 0) for k, v in suite_modes.items() if k.startswith("paraphrase")),
default=0,
)
# Find best (lowest) avg latency
avg_latencies = [
v.get("avg_latency_ms", 0)
for v in suite_modes.values()
if v.get("avg_latency_ms", 0) > 0
]
best_latency = min(avg_latencies) if avg_latencies else 0
table.add_row(
combo_name,
_quality_cell(lexical_best),
_quality_cell(paraphrase_best),
_latency_cell(best_latency),
)
return table
def main():
parser = argparse.ArgumentParser(description="View semantic search benchmark results")
parser.add_argument("path", type=Path, help="Path to JSONL benchmark artifact")
parser.add_argument(
"--sort-by", choices=list(SORT_KEYS.keys()), default="combo", help="Sort column"
)
parser.add_argument(
"--filter-combo", type=str, default=None, help="Filter by combo name (substring match)"
)
parser.add_argument(
"--filter-suite", type=str, default=None, help="Filter by suite name (substring match)"
)
parser.add_argument(
"--filter-mode", type=str, default=None, help="Filter by mode (fts, vector, hybrid)"
)
parser.add_argument("--no-summary", action="store_true", help="Skip the summary table")
args = parser.parse_args()
if not args.path.exists():
print(f"File not found: {args.path}", file=sys.stderr)
sys.exit(1)
records = load_benchmarks(args.path)
if not records:
print("No benchmark records found.", file=sys.stderr)
sys.exit(1)
# Apply filters
if args.filter_combo:
records = [r for r in records if args.filter_combo in r["metrics"]["combo"]]
if args.filter_suite:
records = [r for r in records if args.filter_suite in r["metrics"]["suite"]]
if args.filter_mode:
records = [r for r in records if args.filter_mode == r["metrics"]["mode"]]
if not records:
print("No records match the filters.", file=sys.stderr)
sys.exit(1)
# Sort
sort_fn = SORT_KEYS.get(args.sort_by, SORT_KEYS["combo"])
records.sort(key=sort_fn)
console = Console(width=max(120, Console().width))
# Timestamp from first record
timestamp = records[0].get("timestamp_utc", "unknown")
console.print(f"\n[dim]Benchmark run: {timestamp}[/dim]")
console.print(f"[dim]Records: {len(records)}[/dim]\n")
# Detail table
console.print(build_table(records))
# Summary table
if not args.no_summary:
console.print()
console.print(build_summary_table(records))
console.print()
if __name__ == "__main__":
main()
