Pythoneval_harness.py
Copy
1
import os
2
import json
3
import time
4
from dataclasses import dataclass, asdict
5
 
6
import openai
7
 
8
client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
9
 
10
@dataclass
11
class EvalCase:
12
    id: str
13
    input: str
14
    expected: str | None = None  # For exact-match evals
15
    criteria: str | None = None  # For LLM-judge evals
16
 
17
@dataclass
18
class EvalResult:
19
    case_id: str
20
    model: str
21
    response: str
22
    score: float      # 0.0–1.0 for exact, 1–5 for LLM judge
23
    scorer: str
24
    latency_ms: float
25
    passed: bool
26
 
27
EVAL_DATASET = [
28
    EvalCase("q1", "What is 2+2?", expected="4"),
29
    EvalCase("q2", "Name the capital of France.", expected="Paris"),
30
    EvalCase(
31
        "q3",
32
        "Explain recursion to a 10-year-old.",
33
        criteria="Is the explanation clear, uses an analogy, and avoids jargon?",
34
    ),
35
]
36
 
37
def exact_match(response: str, expected: str) -> tuple[float, bool]:
38
    clean = response.strip().lower()
39
    exp = expected.strip().lower()
40
    score = 1.0 if exp in clean or clean == exp else 0.0
41
    return score, score == 1.0
42
 
43
def llm_judge(response: str, criteria: str) -> tuple[float, bool]:
44
    """Score 1-5 using GPT-4o as judge."""
45
    prompt = f"""Score this response 1-5 based on the criteria.
46
Criteria: {criteria}
47
Response: {response}
48
Output JSON only: {{"score": <1-5>, "reason": "<brief reason>"}}"""
49
    
50
    judge_response = client.chat.completions.create(
51
        model="gpt-4o-mini",
52
        messages=[{"role": "user", "content": prompt}],
53
        max_tokens=100,
54
        temperature=0,
55
        response_format={"type": "json_object"},
56
    )
57
    result = json.loads(judge_response.choices[0].message.content)
58
    score = result.get("score", 1)
59
    return float(score), score >= 4
60
 
61
def run_evals(model: str, dataset: list[EvalCase]) -> list[EvalResult]:
62
    results = []
63
    for case in dataset:
64
        start = time.perf_counter()
65
        response = client.chat.completions.create(
66
            model=model,
67
            messages=[{"role": "user", "content": case.input}],
68
            max_tokens=200,
69
            temperature=0,
70
        )
71
        latency = (time.perf_counter() - start) * 1000
72
        text = response.choices[0].message.content
73
        
74
        if case.expected:
75
            score, passed = exact_match(text, case.expected)
76
            scorer = "exact_match"
77
        elif case.criteria:
78
            score, passed = llm_judge(text, case.criteria)
79
            scorer = "llm_judge"
80
        else:
81
            continue
82
        
83
        results.append(EvalResult(case.id, model, text, score, scorer, latency, passed))
84
    return results
85
 
86
# Run evals
87
for model in ["gpt-4o-mini"]:
88
    results = run_evals(model, EVAL_DATASET)
89
    passed = sum(1 for r in results if r.passed)
90
    print(f"{model}: {passed}/{len(results)} passed")
91
    
92
    # Save results
93
    with open(f"evals_{model.replace('/', '-')}.jsonl", "w") as f:
94
        for r in results:
95
            f.write(json.dumps(asdict(r)) + "\n")
96