Pythonhallucination_detector.py
Copy
1
import os
2
import json
3
from collections import Counter
4
 
5
import openai
6
 
7
client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
8
 
9
def self_consistency_check(question: str, model: str = "gpt-4o-mini", n: int = 5) -> dict:
10
    """
11
    Sample n responses to detect inconsistency (a hallucination signal).
12
    High variance in answers = likely hallucination.
13
    """
14
    responses = []
15
    for _ in range(n):
16
        response = client.chat.completions.create(
17
            model=model,
18
            messages=[{"role": "user", "content": question}],
19
            max_tokens=100,
20
            temperature=0.7,  # Some variance to detect inconsistency
21
        )
22
        responses.append(response.choices[0].message.content.strip())
23
    
24
    # Simple consistency: check if most responses agree
25
    # In practice: use sentence embeddings for semantic similarity
26
    response_counts = Counter(responses)
27
    most_common, count = response_counts.most_common(1)[0]
28
    consistency_score = count / n
29
    
30
    return {
31
        "question": question,
32
        "responses": responses,
33
        "most_common": most_common,
34
        "consistency_score": consistency_score,
35
        "likely_hallucination": consistency_score < 0.6,
36
    }
37
 
38
def faithfulness_check(context: str, question: str, answer: str) -> dict:
39
    """Check if answer is faithful to context (no hallucinated facts)."""
40
    prompt = f"""Given this context and question, evaluate if the answer contains ONLY information from the context.
41
 
42
Context: {context}
43
Question: {question}
44
Answer: {answer}
45
 
46
Output JSON: {{"faithful": true/false, "hallucinated_claims": ["..."], "score": 0.0-1.0}}"""
47
    
48
    response = client.chat.completions.create(
49
        model="gpt-4o-mini",
50
        messages=[{"role": "user", "content": prompt}],
51
        max_tokens=200,
52
        temperature=0,
53
        response_format={"type": "json_object"},
54
    )
55
    return json.loads(response.choices[0].message.content)
56
 
57
# ── Test hallucination-prone questions ─────────────────────────────
58
TRICKY_QUESTIONS = [
59
    "What was the exact date Abraham Lincoln signed the Emancipation Proclamation?",
60
    "What is the population of the Moon?",
61
    "Who invented the telephone, and in what year exactly?",
62
]
63
 
64
for q in TRICKY_QUESTIONS:
65
    result = self_consistency_check(q, n=3)
66
    status = "⚠️ INCONSISTENT" if result["likely_hallucination"] else "✅ Consistent"
67
    print(f"\n{status} (score={result['consistency_score']:.0%})")
68
    print(f"Q: {q}")
69
    print(f"Most common: {result['most_common'][:80]}...")
70
 
71
# ── Faithfulness test ───────────────────────────────────────────────
72
CONTEXT = "The Python programming language was created by Guido van Rossum and first released in 1991."
73
QUESTION = "When was Python created and by whom?"
74
 
75
# Generate an answer
76
answer_response = client.chat.completions.create(
77
    model="gpt-4o-mini",
78
    messages=[
79
        {"role": "system", "content": f"Answer using only this context: {CONTEXT}"},
80
        {"role": "user", "content": QUESTION},
81
    ],
82
    max_tokens=100,
83
    temperature=0,
84
)
85
answer = answer_response.choices[0].message.content
86
 
87
print("\n=== Faithfulness Check ===")
88
faith = faithfulness_check(CONTEXT, QUESTION, answer)
89
print(f"Answer: {answer}")
90
print(f"Faithful: {faith['faithful']}, Score: {faith['score']}")
91