def test_classifier_returns_billing():
    result = call_llm("My credit card was charged twice")
    assert result["category"] == "billing"

@flake_aware(runs=10, pass_threshold=0.8)
def test_classifier_returns_billing():
    result = call_llm("My credit card was charged twice")
    return result["category"] == "billing"

# tests/llm/conftest.py
import os, json, time, hashlib, statistics
from dataclasses import dataclass, asdict, field
from pathlib import Path
from typing import Callable, Any
import pytest

BASELINE_PATH = Path("tests/llm/_baseline.json")
RESULTS_PATH = Path("tests/llm/_results.json")
COST_CAP_USD = float(os.getenv("LLM_TEST_COST_CAP_USD", "5.0"))


@dataclass
class EvalResult:
    name: str
    pass_rate: float
    runs: int
    total_cost_usd: float
    p50_latency_ms: float
    p95_latency_ms: float
    notes: dict = field(default_factory=dict)


@dataclass
class _RunStats:
    successes: int = 0
    runs: int = 0
    cost_usd: float = 0.0
    latencies_ms: list = field(default_factory=list)
    notes: dict = field(default_factory=dict)


_run_total_cost = {"value": 0.0}
_collected: dict[str, EvalResult] = {}


def flake_aware(runs: int = 5, pass_threshold: float = 0.8, max_cost_usd: float = 0.50):
    """
    Decorator: run the wrapped fn `runs` times. Test passes if pass-rate >= threshold
    and total cost is under max_cost_usd. The wrapped fn must return a bool or a
    (bool, cost_usd, latency_ms) tuple.
    """
    def decorator(fn: Callable) -> Callable:
        def wrapper(*args, **kwargs):
            stats = _RunStats()
            for _ in range(runs):
                start = time.monotonic()
                outcome = fn(*args, **kwargs)
                latency_ms = (time.monotonic() - start) * 1000

                if isinstance(outcome, tuple):
                    ok, cost, reported_latency = outcome
                    latency_ms = reported_latency or latency_ms
                else:
                    ok, cost = bool(outcome), 0.0

                stats.runs += 1
                stats.successes += 1 if ok else 0
                stats.cost_usd += cost
                stats.latencies_ms.append(latency_ms)

                _run_total_cost["value"] += cost
                if _run_total_cost["value"] > COST_CAP_USD:
                    pytest.fail(
                        f"Global cost cap of ${COST_CAP_USD} exceeded. "
                        f"Aborting suite. Spent: ${_run_total_cost['value']:.2f}"
                    )

            pass_rate = stats.successes / stats.runs
            result = EvalResult(
                name=fn.__name__,
                pass_rate=pass_rate,
                runs=stats.runs,
                total_cost_usd=round(stats.cost_usd, 4),
                p50_latency_ms=round(statistics.median(stats.latencies_ms), 1),
                p95_latency_ms=round(_p95(stats.latencies_ms), 1),
                notes=stats.notes,
            )
            _collected[fn.__name__] = result

            if stats.cost_usd > max_cost_usd:
                pytest.fail(
                    f"{fn.__name__} cost ${stats.cost_usd:.3f} > "
                    f"per-test cap ${max_cost_usd}"
                )
            assert pass_rate >= pass_threshold, (
                f"{fn.__name__}: pass rate {pass_rate:.0%} below threshold {pass_threshold:.0%} "
                f"({stats.successes}/{stats.runs}) — cost ${stats.cost_usd:.3f}"
            )
        wrapper.__name__ = fn.__name__
        return wrapper
    return decorator


def _p95(values: list[float]) -> float:
    if not values:
        return 0.0
    s = sorted(values)
    k = max(0, int(round(0.95 * (len(s) - 1))))
    return s[k]


@pytest.fixture(scope="session", autouse=True)
def _persist_results(request):
    yield
    RESULTS_PATH.parent.mkdir(parents=True, exist_ok=True)
    payload = {name: asdict(r) for name, r in _collected.items()}
    RESULTS_PATH.write_text(json.dumps(payload, indent=2, sort_keys=True))


def pytest_terminal_summary(terminalreporter, exitstatus, config):
    if not _collected:
        return
    tr = terminalreporter
    tr.write_sep("=", "LLM EVAL SUMMARY")
    for r in sorted(_collected.values(), key=lambda x: x.name):
        tr.write_line(
            f"{r.name:40s}  pass={r.pass_rate:>5.0%}  runs={r.runs:>3}  "
            f"cost=${r.total_cost_usd:>5.3f}  p95={r.p95_latency_ms:>6.0f}ms"
        )
    tr.write_line(f"TOTAL COST: ${_run_total_cost['value']:.3f} (cap ${COST_CAP_USD})")

# tests/llm/test_classifier.py
from conftest import flake_aware
from your_app.llm import call_llm  # your wrapper that returns (response_dict, cost_usd, latency_ms)

GOLDEN_BILLING = {
    "input": "My credit card was charged twice for the same order",
    "expected_category": "billing",
    "expected_keywords": ["refund", "duplicate", "charge"],
}


@flake_aware(runs=10, pass_threshold=0.8, max_cost_usd=0.20)
def test_classifier_billing_intent():
    result, cost, latency_ms = call_llm(GOLDEN_BILLING["input"])

    # Metric 1: structured field equality
    structural = result["category"] == GOLDEN_BILLING["expected_category"]

    # Metric 2: keyword recall in the rationale (catches semantic drift)
    rationale = result.get("rationale", "").lower()
    keyword_hits = sum(1 for k in GOLDEN_BILLING["expected_keywords"] if k in rationale)
    semantic = keyword_hits >= 2  # at least 2 of 3 keywords

    # Metric 3: cost guardrail
    affordable = cost < 0.005  # half a cent per call ceiling

    return (structural and semantic and affordable), cost, latency_ms

# tests/llm/golden_set.py
import json, hashlib
from pathlib import Path

GOLDEN_PATH = Path("tests/llm/golden_set.json")


def load_golden() -> list[dict]:
    data = json.loads(GOLDEN_PATH.read_text())
    return data["cases"]


def golden_fingerprint() -> str:
    """Stable hash of the golden set. Bump in commit messages when intentional."""
    raw = GOLDEN_PATH.read_text()
    return hashlib.sha256(raw.encode()).hexdigest()[:12]


def assert_drift_log_current():
    """Run as a separate test. Fails if golden set hash changed without an entry
    in the drift log — forces a human to acknowledge the change."""
    log = Path("tests/llm/golden_drift.log").read_text().splitlines()
    current = golden_fingerprint()
    assert any(line.startswith(current) for line in log), (
        f"Golden set hash {current} not in drift log. "
        f"Add an entry: '{current}  YYYY-MM-DD  reason'"
    )

# tests/llm/test_regression_gate.py
import json, os
from pathlib import Path
import pytest

BASELINE = Path("tests/llm/_baseline.json")
CURRENT = Path("tests/llm/_results.json")
REGRESSION_TOLERANCE = float(os.getenv("LLM_REGRESSION_TOLERANCE", "0.05"))


def test_no_regression_vs_baseline():
    if not BASELINE.exists():
        pytest.skip("No baseline yet — first run will create one")

    baseline = json.loads(BASELINE.read_text())
    current = json.loads(CURRENT.read_text())

    regressions = []
    for name, baseline_result in baseline.items():
        if name not in current:
            continue  # test deleted, ignore
        delta = current[name]["pass_rate"] - baseline_result["pass_rate"]
        if delta < -REGRESSION_TOLERANCE:
            regressions.append(
                f"{name}: {baseline_result['pass_rate']:.0%} -> "
                f"{current[name]['pass_rate']:.0%} (Δ {delta:+.0%})"
            )

    assert not regressions, "Regressions detected:\n  " + "\n  ".join(regressions)

# .github/workflows/llm-eval.yml
name: LLM Evals
on: [pull_request]
jobs:
  eval:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with: { python-version: '3.11' }
      - run: pip install pytest openai
      - name: Run eval suite
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          LLM_TEST_COST_CAP_USD: "3.00"
          LLM_REGRESSION_TOLERANCE: "0.05"
        run: pytest tests/llm/ -v
      - uses: actions/upload-artifact@v4
        with: { name: llm-results, path: tests/llm/_results.json }