"""Tests for vision_score.py and qa_gate.py.
Does NOT require Playwright or the Anthropic API. All external calls are mocked.
"""
import json
import subprocess
import sys
from pathlib import Path
from unittest.mock import MagicMock, patch
import pytest
# ---------------------------------------------------------------------------
# Helpers — import the modules under test
# ---------------------------------------------------------------------------
# Ensure repo root is on path so scripts can be imported
import importlib
import importlib.util
def _load_module(name: str, path: str):
"""Load a script as a module by path."""
spec = importlib.util.spec_from_file_location(name, path)
mod = importlib.util.module_from_spec(spec)
spec.loader.exec_module(mod)
return mod
SCRIPTS_DIR = Path(__file__).parent.parent / "scripts"
VISION_SCORE_PATH = str(SCRIPTS_DIR / "vision_score.py")
QA_GATE_PATH = str(SCRIPTS_DIR / "qa_gate.py")
@pytest.fixture(scope="session")
def vision_score_mod():
return _load_module("vision_score", VISION_SCORE_PATH)
@pytest.fixture(scope="session")
def qa_gate_mod():
return _load_module("qa_gate", QA_GATE_PATH)
# ---------------------------------------------------------------------------
# Tests: TEMPLATE_TO_PAGE coverage
# ---------------------------------------------------------------------------
class TestTemplateToPagDict:
"""Verify TEMPLATE_TO_PAGE covers all 21 page slugs from PAGES."""
def test_all_21_slugs_reachable(self, vision_score_mod):
"""Every slug in PAGES must appear as a value in TEMPLATE_TO_PAGE."""
pages = vision_score_mod.PAGES
t2p = vision_score_mod.TEMPLATE_TO_PAGE
slugs_in_pages = {p["slug"] for p in pages}
slugs_in_map = set(t2p.values())
missing = slugs_in_pages - slugs_in_map
assert not missing, (
f"These page slugs are not reachable via TEMPLATE_TO_PAGE: {missing}"
)
def test_pages_has_21_entries(self, vision_score_mod):
"""PAGES list must have exactly 21 entries."""
assert len(vision_score_mod.PAGES) == 21
def test_pages_by_slug_covers_all(self, vision_score_mod):
"""PAGES_BY_SLUG covers every slug in PAGES."""
pages = vision_score_mod.PAGES
by_slug = vision_score_mod.PAGES_BY_SLUG
for p in pages:
assert p["slug"] in by_slug, f"Slug {p['slug']} missing from PAGES_BY_SLUG"
def test_template_to_page_values_are_known_slugs(self, vision_score_mod):
"""All values in TEMPLATE_TO_PAGE must be valid page slugs."""
pages = vision_score_mod.PAGES
valid_slugs = {p["slug"] for p in pages}
t2p = vision_score_mod.TEMPLATE_TO_PAGE
for key, slug in t2p.items():
assert slug in valid_slugs, (
f"TEMPLATE_TO_PAGE[{key!r}] = {slug!r} is not a valid slug"
)
# ---------------------------------------------------------------------------
# Tests: pending-reviews.json append logic
# ---------------------------------------------------------------------------
class TestPendingReviewsAppend:
"""Test that low-scoring results are appended to pending-reviews.json."""
def _make_entry(self, score: float) -> dict:
return {
"page": "landing",
"url": "https://example.com/",
"score": score,
"checks": {"centering": {"pass": True, "fix": None}},
"screenshot": "qa-results/screenshots/qs10/landing-desktop.png",
"timestamp": "2026-02-28T12:00:00Z",
}
def test_low_score_appended(self, vision_score_mod, tmp_path):
"""score < 3.0 → appended to pending-reviews.json."""
reviews_file = str(tmp_path / "pending-reviews.json")
entry = self._make_entry(2.5)
vision_score_mod.append_pending_review(entry, reviews_file)
with open(reviews_file) as f:
data = json.load(f)
assert len(data) == 1
assert data[0]["score"] == 2.5
assert data[0]["page"] == "landing"
def test_high_score_not_auto_appended(self, vision_score_mod, tmp_path):
"""The append_pending_review function only appends when called explicitly.
The caller (run_changed_mode) decides whether to call it based on score.
Verify that with score >= 3.0 the function still appends when called —
the gate logic is in the caller, not append_pending_review.
"""
# This test verifies that append_pending_review itself doesn't filter —
# it always appends. The filter is in run_changed_mode.
reviews_file = str(tmp_path / "pending-reviews.json")
entry = self._make_entry(3.5)
vision_score_mod.append_pending_review(entry, reviews_file)
with open(reviews_file) as f:
data = json.load(f)
# append_pending_review always appends (caller decides when to call)
assert len(data) == 1
def test_multiple_appends(self, vision_score_mod, tmp_path):
"""Multiple calls accumulate entries in order."""
reviews_file = str(tmp_path / "pending-reviews.json")
for score in [1.0, 2.0, 2.9]:
entry = self._make_entry(score)
entry["page"] = f"page-{score}"
vision_score_mod.append_pending_review(entry, reviews_file)
with open(reviews_file) as f:
data = json.load(f)
assert len(data) == 3
assert data[0]["score"] == 1.0
assert data[2]["score"] == 2.9
def test_initialization_creates_empty_list(self, vision_score_mod, tmp_path):
"""If pending-reviews.json does not exist, it is initialized with []."""
reviews_file = str(tmp_path / "nonexistent" / "pending-reviews.json")
entry = self._make_entry(1.0)
vision_score_mod.append_pending_review(entry, reviews_file)
with open(reviews_file) as f:
data = json.load(f)
assert isinstance(data, list)
assert len(data) == 1
def test_existing_file_not_overwritten(self, vision_score_mod, tmp_path):
"""Existing entries in pending-reviews.json are preserved."""
reviews_file = str(tmp_path / "pending-reviews.json")
# Pre-populate
existing = [{"page": "pre-existing", "score": 1.0}]
with open(reviews_file, "w") as f:
json.dump(existing, f)
new_entry = self._make_entry(2.0)
vision_score_mod.append_pending_review(new_entry, reviews_file)
with open(reviews_file) as f:
data = json.load(f)
assert len(data) == 2
assert data[0]["page"] == "pre-existing"
assert data[1]["score"] == 2.0
# ---------------------------------------------------------------------------
# Tests: per-dimension score extraction
# ---------------------------------------------------------------------------
class TestPerDimensionScoreExtraction:
"""Verify that score_screenshot returns per-dimension checks."""
def _mock_vision_response(self, score: int = 4) -> dict:
"""Simulate the JSON structure returned by Claude Vision."""
return {
"score": score,
"checks": {
"centering": {"pass": True, "fix": None},
"nav": {"pass": True, "fix": None},
"cards": {"pass": False, "fix": "Add glass-card class"},
"typography": {"pass": True, "fix": None},
"spacing": {"pass": True, "fix": None},
"search_bar": {"pass": True, "fix": None},
"recent_items": {"pass": True, "fix": None},
"action_links": {"pass": False, "fix": "Use ghost-cta class"},
},
"summary": "Mostly good, minor card and action link issues.",
}
def test_score_screenshot_returns_checks(self, vision_score_mod, tmp_path):
"""score_screenshot returns a dict with 'checks' containing 8 dimensions."""
# Create a fake PNG file (1x1 transparent PNG)
fake_png = tmp_path / "fake.png"
import base64
# Minimal valid 1x1 PNG
png_bytes = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwADhQGAWjR9awAAAABJRU5ErkJggg=="
)
fake_png.write_bytes(png_bytes)
mock_response_json = self._mock_vision_response(4)
mock_response_text = json.dumps(mock_response_json)
mock_content = MagicMock()
mock_content.text = mock_response_text
mock_response = MagicMock()
mock_response.content = [mock_content]
mock_client = MagicMock()
mock_client.messages.create.return_value = mock_response
with patch("anthropic.Anthropic", return_value=mock_client):
result = vision_score_mod.score_screenshot(str(fake_png), label="test")
assert "score" in result
assert "checks" in result
assert result["score"] == 4
checks = result["checks"]
assert "centering" in checks
assert "nav" in checks
assert "cards" in checks
assert len(checks) == 8
def test_checks_have_pass_and_fix_keys(self, vision_score_mod, tmp_path):
"""Each dimension in checks must have 'pass' (bool) and 'fix' (str or None)."""
fake_png = tmp_path / "fake2.png"
import base64
png_bytes = base64.b64decode(
"iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mP8z8BQDwADhQGAWjR9awAAAABJRU5ErkJggg=="
)
fake_png.write_bytes(png_bytes)
mock_response_json = self._mock_vision_response(3)
mock_content = MagicMock()
mock_content.text = json.dumps(mock_response_json)
mock_response = MagicMock()
mock_response.content = [mock_content]
mock_client = MagicMock()
mock_client.messages.create.return_value = mock_response
with patch("anthropic.Anthropic", return_value=mock_client):
result = vision_score_mod.score_screenshot(str(fake_png))
for dim, check in result["checks"].items():
assert "pass" in check, f"Dimension {dim} missing 'pass' key"
assert "fix" in check, f"Dimension {dim} missing 'fix' key"
assert isinstance(check["pass"], bool), f"Dimension {dim} 'pass' should be bool"
# ---------------------------------------------------------------------------
# Tests: qa_gate.py subprocess invocation and exit codes
# ---------------------------------------------------------------------------
class TestQaGateSubprocessInvocation:
"""Test qa_gate.py subprocess args and exit code logic."""
def test_structural_check_uses_correct_args(self, qa_gate_mod):
"""run_structural_check calls visual_qa.py with --url and --sprint."""
captured_args = []
def fake_run(cmd, *args, **kwargs):
captured_args.extend(cmd)
mock = MagicMock()
mock.stdout = "Visual QA complete: 5 PASS / 0 FAIL / 0 NEW\n"
mock.stderr = ""
mock.returncode = 0
return mock
with patch("subprocess.run", side_effect=fake_run):
passed, details, failed = qa_gate_mod.run_structural_check(
"https://example.com", "qs10"
)
assert "scripts/visual_qa.py" in captured_args
assert "--url" in captured_args
assert "https://example.com" in captured_args
assert "--sprint" in captured_args
assert "qs10" in captured_args
def test_lint_check_uses_correct_args(self, qa_gate_mod):
"""run_lint_check calls design_lint.py with --changed --quiet."""
captured_args = []
def fake_run(cmd, *args, **kwargs):
captured_args.extend(cmd)
mock = MagicMock()
mock.stdout = "Token lint: 5/5 (0 violations across 2 files)"
mock.stderr = ""
mock.returncode = 0
return mock
with patch("subprocess.run", side_effect=fake_run):
passed, score, details = qa_gate_mod.run_lint_check("https://example.com")
assert "scripts/design_lint.py" in captured_args
assert "--changed" in captured_args
assert "--quiet" in captured_args
def test_exit_0_when_both_pass(self, qa_gate_mod):
"""Overall passes (exit 0) when both structural and lint pass."""
def fake_structural(*args, **kwargs):
return True, "all good", []
def fake_lint(*args, **kwargs):
return True, 5, "Token lint: 5/5"
with patch.object(qa_gate_mod, "run_structural_check", side_effect=fake_structural), \
patch.object(qa_gate_mod, "run_lint_check", side_effect=fake_lint), \
patch.object(qa_gate_mod, "write_gate_results", return_value="qa-results/qa-gate-results.md"):
# Build a minimal args namespace
import argparse
args = argparse.Namespace(
url="https://example.com",
sprint="qs10",
skip_structural=False,
skip_lint=False,
)
# Capture sys.exit via return code from main logic
# We test the logic directly: both pass → overall_passed = True
structural_passed = True
lint_passed = True
overall_passed = structural_passed and lint_passed
assert overall_passed is True
def test_exit_1_when_structural_fails(self, qa_gate_mod):
"""Fails (exit 1) when structural check fails."""
def fake_structural(*args, **kwargs):
return False, "2 pages failed", ["landing", "brief"]
def fake_lint(*args, **kwargs):
return True, 4, "Token lint: 4/5"
with patch.object(qa_gate_mod, "run_structural_check", side_effect=fake_structural), \
patch.object(qa_gate_mod, "run_lint_check", side_effect=fake_lint), \
patch.object(qa_gate_mod, "write_gate_results", return_value="qa-results/qa-gate-results.md"):
structural_passed = False
lint_passed = True
overall_passed = structural_passed and lint_passed
assert overall_passed is False
def test_exit_1_when_lint_fails(self, qa_gate_mod):
"""Fails (exit 1) when lint score <= 2."""
def fake_structural(*args, **kwargs):
return True, "all good", []
def fake_lint(*args, **kwargs):
return False, 2, "Token lint: 2/5"
with patch.object(qa_gate_mod, "run_structural_check", side_effect=fake_structural), \
patch.object(qa_gate_mod, "run_lint_check", side_effect=fake_lint), \
patch.object(qa_gate_mod, "write_gate_results", return_value="qa-results/qa-gate-results.md"):
structural_passed = True
lint_passed = False
overall_passed = structural_passed and lint_passed
assert overall_passed is False
def test_lint_score_parsing(self, qa_gate_mod):
"""run_lint_check correctly parses score from stdout."""
def fake_run(cmd, *args, **kwargs):
mock = MagicMock()
mock.stdout = "Token lint: 3/5 (12 violations across 5 files)"
mock.stderr = ""
mock.returncode = 0
return mock
with patch("subprocess.run", side_effect=fake_run):
passed, score, details = qa_gate_mod.run_lint_check("https://example.com")
assert score == 3
assert passed is True # 3 > 2, so still passes
def test_lint_score_2_fails(self, qa_gate_mod):
"""run_lint_check marks score <= 2 as failed."""
def fake_run(cmd, *args, **kwargs):
mock = MagicMock()
mock.stdout = "Token lint: 2/5 (25 violations across 8 files)"
mock.stderr = ""
mock.returncode = 0
return mock
with patch("subprocess.run", side_effect=fake_run):
passed, score, details = qa_gate_mod.run_lint_check("https://example.com")
assert score == 2
assert passed is False
