Pythoneval_harness.py
Copy
1
from openai import OpenAI
2
from dataclasses import dataclass, field
3
from typing import Callable
4
import json, statistics
5
 
6
client = OpenAI(max_retries=3)
7
 
8
@dataclass
9
class EvalCase:
10
    id: str
11
    task: str
12
    expected_keywords: list[str] = field(default_factory=list)
13
    criteria: str = "Is this response helpful, accurate, and complete?"
14
    max_tokens: int = 500
15
 
16
@dataclass
17
class EvalResult:
18
    case: EvalCase
19
    output: str
20
    keyword_pass: bool
21
    llm_score: float
22
    llm_reason: str
23
    passed: bool
24
 
25
def llm_judge(task: str, response: str, criteria: str) -> tuple[float, str]:
26
    resp = client.chat.completions.create(
27
        model="gpt-4o-mini",
28
        messages=[{"role": "user", "content": f"Task: {task}
29
Response: {response}
30
 
31
Criteria: {criteria}
32
 
33
Score 0.0-1.0. Return JSON: {{"score": 0.0, "reason": "..."}}"}],
34
        response_format={"type": "json_object"},
35
        temperature=0
36
    ).choices[0].message.content
37
    data = json.loads(resp)
38
    return data["score"], data["reason"]
39
 
40
def run_eval(agent_fn: Callable, cases: list[EvalCase]) -> dict:
41
    results = []
42
    for case in cases:
43
        output = agent_fn(case.task)
44
        kw_pass = all(kw.lower() in output.lower() for kw in case.expected_keywords)
45
        score, reason = llm_judge(case.task, output, case.criteria)
46
        result = EvalResult(case=case, output=output, keyword_pass=kw_pass, llm_score=score, llm_reason=reason, passed=kw_pass and score >= 0.7)
47
        results.append(result)
48
        status = "✓" if result.passed else "✗"
49
        print(f"{status} [{case.id}] score={score:.2f} | {case.task[:50]}")
50
    return {
51
        "pass_rate": sum(1 for r in results if r.passed) / len(results),
52
        "avg_score": statistics.mean(r.llm_score for r in results),
53
        "results": results
54
    }
55
 
56
# Test with a simple agent
57
def simple_agent(task: str) -> str:
58
    return client.chat.completions.create(
59
        model="gpt-4o-mini", temperature=0,
60
        messages=[{"role": "user", "content": task}]
61
    ).choices[0].message.content
62
 
63
SUITE = [
64
    EvalCase("q1", "What is the capital of France?", ["Paris"]),
65
    EvalCase("q2", "Write a Python hello world", ["print", "Hello"]),
66
    EvalCase("q3", "Explain what an API is", ["interface", "application"]),
67
    EvalCase("q4", "What is 2 + 2?", ["4"]),
68
    EvalCase("q5", "List 3 Python web frameworks", ["Flask", "Django", "FastAPI"]),
69
]
70
 
71
if __name__ == "__main__":
72
    report = run_eval(simple_agent, SUITE)
73
    print(f"
74
Pass rate: {report['pass_rate']:.0%}  Avg score: {report['avg_score']:.2f}")
75