Pythoneval_framework.py
Copy
1
import json
2
import pandas as pd
3
from openai import OpenAI
4
 
5
client = OpenAI()
6
 
7
def lcs_length(a: str, b: str) -> int:
8
    a_words, b_words = a.split(), b.split()
9
    m, n = len(a_words), len(b_words)
10
    dp = [[0] * (n + 1) for _ in range(m + 1)]
11
    for i in range(1, m + 1):
12
        for j in range(1, n + 1):
13
            dp[i][j] = dp[i-1][j-1] + 1 if a_words[i-1] == b_words[j-1] else max(dp[i-1][j], dp[i][j-1])
14
    return dp[m][n]
15
 
16
def rouge_l_score(reference: str, hypothesis: str) -> float:
17
    lcs = lcs_length(reference.lower(), hypothesis.lower())
18
    precision = lcs / len(hypothesis.split()) if hypothesis.split() else 0
19
    recall = lcs / len(reference.split()) if reference.split() else 0
20
    if precision + recall == 0:
21
        return 0.0
22
    return 2 * precision * recall / (precision + recall)
23
 
24
def llm_judge_score(question: str, reference: str, actual: str) -> dict:
25
    prompt = f"""Score the ACTUAL answer for correctness vs REFERENCE. Return JSON: {{"score": <1-5>, "reason": "<brief>"}}
26
 
27
Q: {question}
28
Reference: {reference}
29
Actual: {actual}"""
30
    resp = client.chat.completions.create(
31
        model="gpt-6.1-sol",
32
        messages=[{"role": "user", "content": prompt}],
33
        response_format={"type": "json_object"},
34
    )
35
    return json.loads(resp.choices[0].message.content or "{}")
36
 
37
DATASET = [
38
    {"q": "What is backpropagation?", "ref": "Algorithm computing gradients via chain rule to update neural network weights."},
39
    {"q": "What is attention in transformers?", "ref": "Mechanism that weights input tokens by relevance for each output position."},
40
]
41
 
42
records = []
43
for item in DATASET:
44
    resp = client.chat.completions.create(
45
        model="gpt-6-luna", reasoning_effort="none",
46
        messages=[{"role": "user", "content": item["q"]}],
47
    )
48
    actual = resp.choices[0].message.content or ""
49
    rouge = rouge_l_score(item["ref"], actual)
50
    judge = llm_judge_score(item["q"], item["ref"], actual)
51
    records.append({"question": item["q"], "rouge_l": round(rouge, 3), "llm_score": judge.get("score"), "reason": judge.get("reason")})
52
 
53
df = pd.DataFrame(records).sort_values("llm_score", ascending=False)
54
print(df.to_string(index=False))
55