Pythonllm_eval.py
Copy
1
import json
2
from openai import OpenAI
3
 
4
client = OpenAI()
5
 
6
EVAL_DATASET = [
7
    {"question": "What is gradient descent?", "reference": "An optimization algorithm that iteratively adjusts model weights to minimize a loss function."},
8
    {"question": "What is backpropagation?", "reference": "An algorithm that computes gradients of the loss with respect to model weights using the chain rule."},
9
    # Add 8 more...
10
]
11
 
12
def get_app_response(question: str) -> str:
13
    resp = client.chat.completions.create(
14
        model="gpt-4o-mini",
15
        messages=[{"role": "user", "content": question}],
16
    )
17
    return resp.choices[0].message.content or ""
18
 
19
def judge_response(question: str, reference: str, actual: str) -> dict:
20
    prompt = f"""Score the ACTUAL answer vs REFERENCE on a 1-5 scale for correctness.
21
 
22
Question: {question}
23
Reference: {reference}
24
Actual: {actual}
25
 
26
Return JSON: {{"score": <1-5>, "reason": "<one sentence>"}}"""
27
 
28
    resp = client.chat.completions.create(
29
        model="gpt-4o",
30
        messages=[{"role": "user", "content": prompt}],
31
        response_format={"type": "json_object"},
32
    )
33
    return json.loads(resp.choices[0].message.content or "{}")
34
 
35
scores = []
36
for item in EVAL_DATASET:
37
    actual = get_app_response(item["question"])
38
    result = judge_response(item["question"], item["reference"], actual)
39
    scores.append(result.get("score", 0))
40
    print(f"Q: {item['question'][:50]}... Score: {result.get('score')} - {result.get('reason')}")
41
 
42
print(f"\nMean score: {sum(scores)/len(scores):.2f}")
43