← AI System Testing overview

AI SYSTEM TESTING

Runnable Python examples

Six executable examples validated by the repository test command.

Typed evaluator and dataclass

from dataclasses import dataclass

@dataclass(frozen=True)
class EvalCase:
    case_id: str
    allowed_tools: frozenset[str]

def squares(xs: list[int]) -> dict[int, int]:
    return {x: x * x for x in xs}

def tool_allowed(case: EvalCase, tool: str) -> bool:
    return tool in case.allowed_tools

assert squares([2, 3, 2]) == {2: 4, 3: 9}
assert not tool_allowed(
    EvalCase("readonly", frozenset({"read_order"})),
    "delete_order",
)

Duplicate integers become one dictionary key. Frozen records do not replace validation; frozenset is intentional. This is a validator fixture, not production authorization.

Shallow and deep copies

from copy import copy, deepcopy

original = {"contexts": ["A"]}
shallow = copy(original)
deep = deepcopy(original)
shallow["contexts"].append("B")
assert original["contexts"] == ["A", "B"]
assert deep["contexts"] == ["A"]

Shallow copies share nested lists. This can couple test cases; deep copies are not universal for sockets, clients and custom objects.

Generator and context manager

import json
from contextlib import contextmanager
from pathlib import Path

@contextmanager
def jsonl_rows(path: Path):
    with path.open(encoding="utf-8") as stream:
        yield (json.loads(line) for line in stream if line.strip())

# with jsonl_rows(Path("cases.jsonl")) as rows:
#     for row in rows:
#         print(row["case_id"])

Iterate inside with. The file closes after exit, so do not let the generator escape its scope. Do not hide JSON parsing errors; add separate schema validation.

A timing decorator

from functools import wraps
from time import perf_counter

def timed(fn):
    @wraps(fn)
    def wrapper(*args, **kwargs):
        start = perf_counter()
        try:
            return fn(*args, **kwargs)
        finally:
            duration = perf_counter() - start
            print(f"duration={duration:.6f}")
    return wrapper

@timed
def local_check() -> bool:
    return True

assert local_check()

finally measures failed runs too and wraps preserves metadata. This synchronous example needs an await-based wrapper for async functions. Production telemetry must not print sensitive arguments.

Retrieval metrics without an LLM judge

def retrieval_metrics(retrieved: list[str],
                      relevant: set[str], k: int) -> dict:
    if k <= 0:
        raise ValueError("k must be positive")
    top = retrieved[:k]
    if len(top) != len(set(top)):
        raise ValueError("duplicate document IDs")
    hits = len(set(top) & relevant)
    rr = next((1 / rank for rank, doc in
               enumerate(top, 1) if doc in relevant), 0.0)
    return {"precision": hits / k,
            "recall": hits / len(relevant) if relevant else None,
            "rr": rr}

m = retrieval_metrics(["X", "B", "Y"], {"A", "B", "C"}, 3)
assert m == {"precision": 1/3, "recall": 1/3, "rr": 0.5}

Precision uses fixed k even if fewer results are returned; recall is None for no-relevant cases. MRR is dataset-mean rr rather than one-case rr. Passage-level relevance requires different IDs/labels.

An application tool-call validator

class ToolValidationError(Exception):
    pass

def validate_read_order(arguments: dict,
                        allowed_orders: set[str]) -> str:
    if set(arguments) != {"order_id"}:
        raise ToolValidationError("unexpected fields")
    order_id = arguments["order_id"]
    if not isinstance(order_id, str) or not order_id:
        raise ToolValidationError("invalid order_id")
    if order_id not in allowed_orders:
        raise PermissionError("order access denied")
    return order_id

assert validate_read_order({"order_id": "O-1"}, {"O-1"}) == "O-1"

This is a local business validator, not a complete MCP implementation. Allowed orders must come from a trusted identity/authorization service rather than the model. Add JSON Schema and server authorization separately.