AI SYSTEM TESTING
Runnable Python examples
Six executable examples validated by the repository test command.
Typed evaluator and dataclass
from dataclasses import dataclass
@dataclass(frozen=True)
class EvalCase:
case_id: str
allowed_tools: frozenset[str]
def squares(xs: list[int]) -> dict[int, int]:
return {x: x * x for x in xs}
def tool_allowed(case: EvalCase, tool: str) -> bool:
return tool in case.allowed_tools
assert squares([2, 3, 2]) == {2: 4, 3: 9}
assert not tool_allowed(
EvalCase("readonly", frozenset({"read_order"})),
"delete_order",
)Duplicate integers become one dictionary key. Frozen records do not replace validation; frozenset is intentional. This is a validator fixture, not production authorization.
Shallow and deep copies
from copy import copy, deepcopy
original = {"contexts": ["A"]}
shallow = copy(original)
deep = deepcopy(original)
shallow["contexts"].append("B")
assert original["contexts"] == ["A", "B"]
assert deep["contexts"] == ["A"]Shallow copies share nested lists. This can couple test cases; deep copies are not universal for sockets, clients and custom objects.
Generator and context manager
import json
from contextlib import contextmanager
from pathlib import Path
@contextmanager
def jsonl_rows(path: Path):
with path.open(encoding="utf-8") as stream:
yield (json.loads(line) for line in stream if line.strip())
# with jsonl_rows(Path("cases.jsonl")) as rows:
# for row in rows:
# print(row["case_id"])Iterate inside with. The file closes after exit, so do not let the generator escape its scope. Do not hide JSON parsing errors; add separate schema validation.
A timing decorator
from functools import wraps
from time import perf_counter
def timed(fn):
@wraps(fn)
def wrapper(*args, **kwargs):
start = perf_counter()
try:
return fn(*args, **kwargs)
finally:
duration = perf_counter() - start
print(f"duration={duration:.6f}")
return wrapper
@timed
def local_check() -> bool:
return True
assert local_check()finally measures failed runs too and wraps preserves metadata. This synchronous example needs an await-based wrapper for async functions. Production telemetry must not print sensitive arguments.
Retrieval metrics without an LLM judge
def retrieval_metrics(retrieved: list[str],
relevant: set[str], k: int) -> dict:
if k <= 0:
raise ValueError("k must be positive")
top = retrieved[:k]
if len(top) != len(set(top)):
raise ValueError("duplicate document IDs")
hits = len(set(top) & relevant)
rr = next((1 / rank for rank, doc in
enumerate(top, 1) if doc in relevant), 0.0)
return {"precision": hits / k,
"recall": hits / len(relevant) if relevant else None,
"rr": rr}
m = retrieval_metrics(["X", "B", "Y"], {"A", "B", "C"}, 3)
assert m == {"precision": 1/3, "recall": 1/3, "rr": 0.5}Precision uses fixed k even if fewer results are returned; recall is None for no-relevant cases. MRR is dataset-mean rr rather than one-case rr. Passage-level relevance requires different IDs/labels.
An application tool-call validator
class ToolValidationError(Exception):
pass
def validate_read_order(arguments: dict,
allowed_orders: set[str]) -> str:
if set(arguments) != {"order_id"}:
raise ToolValidationError("unexpected fields")
order_id = arguments["order_id"]
if not isinstance(order_id, str) or not order_id:
raise ToolValidationError("invalid order_id")
if order_id not in allowed_orders:
raise PermissionError("order access denied")
return order_id
assert validate_read_order({"order_id": "O-1"}, {"O-1"}) == "O-1"This is a local business validator, not a complete MCP implementation. Allowed orders must come from a trusted identity/authorization service rather than the model. Add JSON Schema and server authorization separately.