1from openai import OpenAI
2from dataclasses import dataclass
3from typing import Callable
4import json, statistics
5
6client = OpenAI()
7
8@dataclass
9class EvalCase:
10 task: str
11 expected_keywords: list[str]
12 forbidden_keywords: list[str] = None
13 max_steps: int = 10
14
15@dataclass
16class EvalResult:
17 case: EvalCase
18 actual_output: str
19 steps_taken: int
20 passed: bool
21 score: float
22 failure_reason: str = ""
23
24def llm_judge(task: str, response: str, criteria: str) -> float:
25 result = client.chat.completions.create(
26 model="gpt-4o",
27 messages=[{"role": "user", "content": f"Task: {task}
28Response: {response}
29
30Criteria: {criteria}
31
32Score 0.0-1.0. Return JSON: {{"score": 0.0, "reason": "..."}}"}],
33 response_format={"type": "json_object"}
34 ).choices[0].message.content
35 return json.loads(result)["score"]
36
37def run_eval_suite(agent_fn: Callable, cases: list[EvalCase]) -> dict:
38 results = []
39 for case in cases:
40 output = agent_fn(case.task)
41 kw_pass = all(kw.lower() in output.lower() for kw in case.expected_keywords)
42 score = llm_judge(case.task, output, "Is the response helpful, accurate, and complete?")
43 results.append(EvalResult(
44 case=case, actual_output=output, steps_taken=1,
45 passed=kw_pass, score=score,
46 failure_reason="" if kw_pass else f"Missing: {[k for k in case.expected_keywords if k.lower() not in output.lower()]}"
47 ))
48 passing = [r for r in results if r.passed]
49 return {
50 "pass_rate": len(passing) / len(results),
51 "avg_score": statistics.mean(r.score for r in results),
52 "results": results,
53 }
54
55EVAL_SUITE = [
56 EvalCase("What is the capital of France?", expected_keywords=["Paris"]),
57 EvalCase("Write a Python hello world", expected_keywords=["print", "hello"]),
58 EvalCase("Explain MVCC in PostgreSQL", expected_keywords=["MVCC", "transaction", "version"]),
59]
