1import json, openai
2from concurrent.futures import ThreadPoolExecutor
3
4GOLDENS = json.load(open("goldens.json")) # [{ id, input, expected }]
5
6JUDGE_PROMPT = """Score the answer on faithfulness (0-5).
7Question: {q}
8Expected: {e}
9Got: {a}
10Return JSON: {{"score": int, "reason": str}}"""
11
12def run_one(g):
13 # 1. Generate candidate answer
14 out = openai.chat.completions.create(
15 model="gpt-4o-mini",
16 messages=[{"role":"system","content":"Be concise."},
17 {"role":"user","content":g["input"]}],
18 ).choices[0].message.content
19
20 # 2. Deterministic checks
21 valid_format = out.strip().endswith(".") and len(out) < 500
22
23 # 3. LLM-as-judge
24 judgment = openai.chat.completions.create(
25 model="gpt-4o", response_format={"type":"json_object"},
26 messages=[{"role":"user","content": JUDGE_PROMPT.format(q=g["input"], e=g["expected"], a=out)}],
27 ).choices[0].message.content
28 score = json.loads(judgment)["score"]
29
30 return {"id": g["id"], "out": out, "valid": valid_format, "score": score}
31
32# 4. Run goldens in parallel
33with ThreadPoolExecutor(max_workers=20) as pool:
34 results = list(pool.map(run_one, GOLDENS))
35
36# 5. Aggregate + gate
37avg = sum(r["score"] for r in results) / len(results)
38print(f"Avg faithfulness: {avg:.2f}")
39assert avg >= 4.0, f"Regression! {avg} < 4.0 baseline"
