Pythonmeta_prompting.py
Copy
1
import os
2
import json
3
import openai
4
 
5
client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
6
 
7
def generate_prompt(task_description: str) -> str:
8
    """Use LLM to generate an optimized prompt for a task."""
9
    meta_prompt = f"""You are an expert prompt engineer. Generate a clear, effective system prompt for this task:
10
 
11
Task: {task_description}
12
 
13
Requirements:
14
- Be specific about the output format
15
- Include tone/style guidance
16
- Handle edge cases
17
- Be concise (< 200 words)
18
 
19
Output just the system prompt, no explanation."""
20
    
21
    response = client.chat.completions.create(
22
        model="gpt-4o-mini",
23
        messages=[{"role": "user", "content": meta_prompt}],
24
        max_tokens=300, temperature=0.7,
25
    )
26
    return response.choices[0].message.content.strip()
27
 
28
def score_prompt(task: str, prompt: str, test_cases: list[str]) -> dict:
29
    """Score a prompt by running test cases and evaluating quality."""
30
    scores = []
31
    
32
    for test_input in test_cases[:3]:  # Limit for cost
33
        # Run the prompt
34
        response = client.chat.completions.create(
35
            model="gpt-4o-mini",
36
            messages=[
37
                {"role": "system", "content": prompt},
38
                {"role": "user", "content": test_input},
39
            ],
40
            max_tokens=200, temperature=0,
41
        )
42
        output = response.choices[0].message.content
43
        
44
        # Evaluate quality
45
        eval_prompt = f"""Rate this LLM response quality for the task.
46
Task: {task}
47
Input: {test_input}
48
Response: {output}
49
 
50
Score 1-10 and explain. Output JSON: {{"score": <1-10>, "issues": ["..."]}}"""
51
        
52
        eval_response = client.chat.completions.create(
53
            model="gpt-4o-mini",
54
            messages=[{"role": "user", "content": eval_prompt}],
55
            max_tokens=150, temperature=0,
56
            response_format={"type": "json_object"},
57
        )
58
        scores.append(json.loads(eval_response.choices[0].message.content))
59
    
60
    avg_score = sum(s["score"] for s in scores) / len(scores)
61
    all_issues = [issue for s in scores for issue in s.get("issues", [])]
62
    return {"avg_score": avg_score, "issues": list(set(all_issues))[:5]}
63
 
64
def improve_prompt(task: str, prompt: str, issues: list[str]) -> str:
65
    """Improve a prompt based on identified issues."""
66
    improvement_prompt = f"""Improve this prompt to fix the identified issues.
67
 
68
Task: {task}
69
Current prompt: {prompt}
70
Issues to fix:
71
{chr(10).join(f"- {issue}" for issue in issues)}
72
 
73
Output only the improved prompt."""
74
    
75
    response = client.chat.completions.create(
76
        model="gpt-4o-mini",
77
        messages=[{"role": "user", "content": improvement_prompt}],
78
        max_tokens=300, temperature=0.5,
79
    )
80
    return response.choices[0].message.content.strip()
81
 
82
# ── Run optimization loop ───────────────────────────────────────────
83
TASK = "Classify customer support tickets by urgency: low/medium/high/critical"
84
TEST_CASES = [
85
    "My billing is wrong again, third time this month!",
86
    "How do I change my password?",
87
    "Server is down for all enterprise customers RIGHT NOW",
88
    "What are your business hours?",
89
]
90
 
91
print("Generating initial prompt...")
92
prompt = generate_prompt(TASK)
93
print(f"Initial: {prompt[:100]}...
94
")
95
 
96
for iteration in range(2):
97
    result = score_prompt(TASK, prompt, TEST_CASES)
98
    print(f"Iteration {iteration + 1}: score={result['avg_score']:.1f}/10")
99
    print(f"  Issues: {result['issues'][:2]}")
100
    
101
    if result["avg_score"] >= 8.5:
102
        print("Prompt quality sufficient!")
103
        break
104
    
105
    prompt = improve_prompt(TASK, prompt, result["issues"])
106
    print(f"  Improved: {prompt[:100]}...
107
")
108
 
109
print(f"\nFinal prompt:\n{prompt}")
110