Prompt EngineeringAdvanced

Build an Automatic Prompt Optimizer

From a one-off rewrite to refinement loops on failures, candidate search and compiled DSPy programs. How to optimise prompts against a metric without overfitting, and how to review what the optimiser produces.

By Amit UpadhyayPublished 02 Oct 20269 min

Following one scenario: “Improve the triage prompt without moving to a bigger model.”A ticket-triage prompt running on gpt-6-luna, currently 82% accurateWhy it matters →

What you will build

A meta-prompting loop that improves the triage prompt on the production model, verified on a hold-out set.

The same scenario as Read and See. You are done when every line below is true. Tick them off as you go.

Done when

Labelled tickets are split into tuning and hold-out sets
Each round writes candidates with the strong model and scores them on the production model
The loop stops at the round cap or when the hold-out stops improving
The final report shows hold-out accuracy for the old and new prompts

0 of 4 done

0/1 steps

Build Steps

Progress

0/1

◆ Before you start

Prerequisites

Python 3.11+Required
python --version
openaiRequired
pip install openai

Step 1 of 1

Use LLMs to generate and optimize prompts

Apply meta-prompting: use LLMs to write, critique, and iteratively improve prompts.

Tasks

Write a prompt generator that takes a task description

Build a prompt evaluator that scores a prompt on 5 criteria

Run a 3-iteration optimization loop

Compare optimized vs original prompt quality on your eval set

Use few-shot meta-prompting to guide the generator

🎯 What You'll Learn

▸Generate prompts automatically for a task description
▸Use LLM-as-critic to score and improve prompts
▸Implement automatic prompt optimization (APO) loop

Code

Pythonmeta_prompting.py
1 import os
2 import json
3 import openai
4  
5 client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
6  
7 def generate_prompt(task_description: str) -> str:
8     """Use LLM to generate an optimized prompt for a task."""
9     meta_prompt = f"""You are an expert prompt engineer. Generate a clear, effective system prompt for this task:
10  
11 Task: {task_description}
12  
13 Requirements:
14 - Be specific about the output format
15 - Include tone/style guidance
16 - Handle edge cases
17 - Be concise (< 200 words)
18  
19 Output just the system prompt, no explanation."""
20     
21     response = client.chat.completions.create(
22         model="gpt-6-luna", reasoning_effort="none",
23         messages=[{"role": "user", "content": meta_prompt}],
24         max_completion_tokens=300, temperature=0.7,
25     )
26     return response.choices[0].message.content.strip()
27  
28 def score_prompt(task: str, prompt: str, test_cases: list[str]) -> dict:
29     """Score a prompt by running test cases and evaluating quality."""
30     scores = []
31     
32     for test_input in test_cases[:3]:  # Limit for cost
33         # Run the prompt
34         response = client.chat.completions.create(
35             model="gpt-6-luna", reasoning_effort="none",
36             messages=[
37                 {"role": "system", "content": prompt},
38                 {"role": "user", "content": test_input},
39             ],
40             max_completion_tokens=200, temperature=0,
41         )
42         output = response.choices[0].message.content
43         
44         # Evaluate quality
45         eval_prompt = f"""Rate this LLM response quality for the task.
46 Task: {task}
47 Input: {test_input}
48 Response: {output}
49  
50 Score 1-10 and explain. Output JSON: {{"score": <1-10>, "issues": ["..."]}}"""
51         
52         eval_response = client.chat.completions.create(
53             model="gpt-6-luna", reasoning_effort="none",
54             messages=[{"role": "user", "content": eval_prompt}],
55             max_completion_tokens=150, temperature=0,
56             response_format={"type": "json_object"},
57         )
58         scores.append(json.loads(eval_response.choices[0].message.content))
59     
60     avg_score = sum(s["score"] for s in scores) / len(scores)
61     all_issues = [issue for s in scores for issue in s.get("issues", [])]
62     return {"avg_score": avg_score, "issues": list(set(all_issues))[:5]}
63  
64 def improve_prompt(task: str, prompt: str, issues: list[str]) -> str:
65     """Improve a prompt based on identified issues."""
66     improvement_prompt = f"""Improve this prompt to fix the identified issues.
67  
68 Task: {task}
69 Current prompt: {prompt}
70 Issues to fix:
71 {chr(10).join(f"- {issue}" for issue in issues)}
72  
73 Output only the improved prompt."""
74     
75     response = client.chat.completions.create(
76         model="gpt-6-luna", reasoning_effort="none",
77         messages=[{"role": "user", "content": improvement_prompt}],
78         max_completion_tokens=300, temperature=0.5,
79     )
80     return response.choices[0].message.content.strip()
81  
82 # ── Run optimization loop ───────────────────────────────────────────
83 TASK = "Classify customer support tickets by urgency: low/medium/high/critical"
84 TEST_CASES = [
85     "My billing is wrong again, third time this month!",
86     "How do I change my password?",
87     "Server is down for all enterprise customers RIGHT NOW",
88     "What are your business hours?",
89 ]
90  
91 print("Generating initial prompt...")
92 prompt = generate_prompt(TASK)
93 print(f"Initial: {prompt[:100]}...
94 ")
95  
96 for iteration in range(2):
97     result = score_prompt(TASK, prompt, TEST_CASES)
98     print(f"Iteration {iteration + 1}: score={result['avg_score']:.1f}/10")
99     print(f"  Issues: {result['issues'][:2]}")
100     
101     if result["avg_score"] >= 8.5:
102         print("Prompt quality sufficient!")
103         break
104     
105     prompt = improve_prompt(TASK, prompt, result["issues"])
106     print(f"  Improved: {prompt[:100]}...
107 ")
108  
109 print(f"\nFinal prompt:\n{prompt}")
110

💡 Pro Tips

▸Meta-prompting works best when you have a clear eval metric — optimize against it
▸Start with a good base prompt and refine rather than generating from scratch

Evaluate it

Prove it works before you call it done

Working once on your machine is a demo. Measure it against a fixed set of cases.

Run the loop for up to five rounds and record tuning and hold-out scores for every round.

Pass bar

Hold-out accuracy above the original, with a tuning-to-hold-out gap under 5 points

Ship and observe

What changes when it is live

Before it ships, in order

Version the winning prompt like any other code change
Re-run the loop when the ticket mix changes

Put on a dashboard

Hold-out accuracy
Tuning vs hold-out gap
Live sampled accuracy

Break it on purpose

Learn the failure modes while they are cheap

Make one change at a time, predict what will happen, then run it.

ChangeWhat you should see

Let the optimiser see the hold-out ticketsBoth scores rise together, and live accuracy does not.

Run 15 roundsThe tuning score keeps climbing while the hold-out score stalls or drops.

Score candidates on the strong modelThe winner underperforms once it runs on the small model.

What this build costs

A full run costs under $1.

Where this leads

System prompts

Optimised or hand-written, the prompt still needs a clear structure that people can maintain.

Next topic →

meta-prompting prompt optimisation DSPy APE OPRO evaluation

← PreviousRed Teaming LLM Applications: Attack It Before Your Users Do

Continue reading

All Prompt Engineering articles →

Prompt Engineering9 min

Red Teaming LLM Applications: Attack It Before Your Users Do

Threat model first, then manual sessions, then automated scans with garak, promptfoo and PyRIT. How to find jailbreaks, data leaks and unsafe tool calls, fix them in the architecture, and keep them fixed in CI.

Prompt Engineering10 min

Tree-of-Thought: Search Over Reasoning Paths

Propose several next steps, judge the partial states, keep the best and expand them. How to build a Tree-of-Thought search, why the evaluator decides everything, and when a reasoning model makes it unnecessary.

Prompt Engineering13 min

Constrained Generation: Beyond Hope-Based Parsing

Enforce output structure at the token level with grammars and token masks — how it differs from native JSON mode, and when the strictness costs you reasoning quality.

Discussion

A meta-prompting loop that improves the triage prompt on the production model, verified on a hold-out set.

The same scenario as Read and See. You are done when every line below is true. Tick them off as you go.

Done when

Labelled tickets are split into tuning and hold-out sets
Each round writes candidates with the strong model and scores them on the production model
The loop stops at the round cap or when the hold-out stops improving
The final report shows hold-out accuracy for the old and new prompts

0 of 4 done

1 import os 2 import json 3 import openai 4 5 client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"]) 6 7 def generate_prompt(task_description: str) -> str: 8 """Use LLM to generate an optimized prompt for a task.""" 9 meta_prompt = f"""You are an expert prompt engineer. Generate a clear, effective system prompt for this task: 10 11 Task: {task_description} 12 13 Requirements: 14 - Be specific about the output format 15 - Include tone/style guidance 16 - Handle edge cases 17 - Be concise (< 200 words) 18 19 Output just the system prompt, no explanation.""" 20 21 response = client.chat.completions.create( 22 model="gpt-6-luna", reasoning_effort="none", 23 messages=[{"role": "user", "content": meta_prompt}], 24 max_completion_tokens=300, temperature=0.7, 25 ) 26 return response.choices[0].message.content.strip() 27 28 def score_prompt(task: str, prompt: str, test_cases: list[str]) -> dict: 29 """Score a prompt by running test cases and evaluating quality.""" 30 scores = [] 31 32 for test_input in test_cases[:3]: # Limit for cost 33 # Run the prompt 34 response = client.chat.completions.create( 35 model="gpt-6-luna", reasoning_effort="none", 36 messages=[ 37 {"role": "system", "content": prompt}, 38 {"role": "user", "content": test_input}, 39 ], 40 max_completion_tokens=200, temperature=0, 41 ) 42 output = response.choices[0].message.content 43 44 # Evaluate quality 45 eval_prompt = f"""Rate this LLM response quality for the task. 46 Task: {task} 47 Input: {test_input} 48 Response: {output} 49 50 Score 1-10 and explain. Output JSON: {{"score": <1-10>, "issues": ["..."]}}""" 51 52 eval_response = client.chat.completions.create( 53 model="gpt-6-luna", reasoning_effort="none", 54 messages=[{"role": "user", "content": eval_prompt}], 55 max_completion_tokens=150, temperature=0, 56 response_format={"type": "json_object"}, 57 ) 58 scores.append(json.loads(eval_response.choices[0].message.content)) 59 60 avg_score = sum(s["score"] for s in scores) / len(scores) 61 all_issues = [issue for s in scores for issue in s.get("issues", [])] 62 return {"avg_score": avg_score, "issues": list(set(all_issues))[:5]} 63 64 def improve_prompt(task: str, prompt: str, issues: list[str]) -> str: 65 """Improve a prompt based on identified issues.""" 66 improvement_prompt = f"""Improve this prompt to fix the identified issues. 67 68 Task: {task} 69 Current prompt: {prompt} 70 Issues to fix: 71 {chr(10).join(f"- {issue}" for issue in issues)} 72 73 Output only the improved prompt.""" 74 75 response = client.chat.completions.create( 76 model="gpt-6-luna", reasoning_effort="none", 77 messages=[{"role": "user", "content": improvement_prompt}], 78 max_completion_tokens=300, temperature=0.5, 79 ) 80 return response.choices[0].message.content.strip() 81 82 # ── Run optimization loop ─────────────────────────────────────────── 83 TASK = "Classify customer support tickets by urgency: low/medium/high/critical" 84 TEST_CASES = [ 85 "My billing is wrong again, third time this month!", 86 "How do I change my password?", 87 "Server is down for all enterprise customers RIGHT NOW", 88 "What are your business hours?", 89 ] 90 91 print("Generating initial prompt...") 92 prompt = generate_prompt(TASK) 93 print(f"Initial: {prompt[:100]}... 94 ") 95 96 for iteration in range(2): 97 result = score_prompt(TASK, prompt, TEST_CASES) 98 print(f"Iteration {iteration + 1}: score={result['avg_score']:.1f}/10") 99 print(f" Issues: {result['issues'][:2]}") 100 101 if result["avg_score"] >= 8.5: 102 print("Prompt quality sufficient!") 103 break 104 105 prompt = improve_prompt(TASK, prompt, result["issues"]) 106 print(f" Improved: {prompt[:100]}... 107 ") 108 109 print(f"\nFinal prompt:\n{prompt}") 110

Learn the failure modes while they are cheap

Make one change at a time, predict what will happen, then run it.

ChangeWhat you should see

Let the optimiser see the hold-out ticketsBoth scores rise together, and live accuracy does not.

Run 15 roundsThe tuning score keeps climbing while the hold-out score stalls or drops.

Score candidates on the strong modelThe winner underperforms once it runs on the small model.

1	import os
2	import json
3	import openai
4
5	client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
6
7	def generate_prompt(task_description: str) -> str:
8	"""Use LLM to generate an optimized prompt for a task."""
9	meta_prompt = f"""You are an expert prompt engineer. Generate a clear, effective system prompt for this task:
10
11	Task: {task_description}
12
13	Requirements:
14	- Be specific about the output format
15	- Include tone/style guidance
16	- Handle edge cases
17	- Be concise (< 200 words)
18
19	Output just the system prompt, no explanation."""
20
21	response = client.chat.completions.create(
22	model="gpt-6-luna", reasoning_effort="none",
23	messages=[{"role": "user", "content": meta_prompt}],
24	max_completion_tokens=300, temperature=0.7,
25	)
26	return response.choices[0].message.content.strip()
27
28	def score_prompt(task: str, prompt: str, test_cases: list[str]) -> dict:
29	"""Score a prompt by running test cases and evaluating quality."""
30	scores = []
31
32	for test_input in test_cases[:3]: # Limit for cost
33	# Run the prompt
34	response = client.chat.completions.create(
35	model="gpt-6-luna", reasoning_effort="none",
36	messages=[
37	{"role": "system", "content": prompt},
38	{"role": "user", "content": test_input},
39	],
40	max_completion_tokens=200, temperature=0,
41	)
42	output = response.choices[0].message.content
43
44	# Evaluate quality
45	eval_prompt = f"""Rate this LLM response quality for the task.
46	Task: {task}
47	Input: {test_input}
48	Response: {output}
49
50	Score 1-10 and explain. Output JSON: {{"score": <1-10>, "issues": ["..."]}}"""
51
52	eval_response = client.chat.completions.create(
53	model="gpt-6-luna", reasoning_effort="none",
54	messages=[{"role": "user", "content": eval_prompt}],
55	max_completion_tokens=150, temperature=0,
56	response_format={"type": "json_object"},
57	)
58	scores.append(json.loads(eval_response.choices[0].message.content))
59
60	avg_score = sum(s["score"] for s in scores) / len(scores)
61	all_issues = [issue for s in scores for issue in s.get("issues", [])]
62	return {"avg_score": avg_score, "issues": list(set(all_issues))[:5]}
63
64	def improve_prompt(task: str, prompt: str, issues: list[str]) -> str:
65	"""Improve a prompt based on identified issues."""
66	improvement_prompt = f"""Improve this prompt to fix the identified issues.
67
68	Task: {task}
69	Current prompt: {prompt}
70	Issues to fix:
71	{chr(10).join(f"- {issue}" for issue in issues)}
72
73	Output only the improved prompt."""
74
75	response = client.chat.completions.create(
76	model="gpt-6-luna", reasoning_effort="none",
77	messages=[{"role": "user", "content": improvement_prompt}],
78	max_completion_tokens=300, temperature=0.5,
79	)
80	return response.choices[0].message.content.strip()
81
82	# ── Run optimization loop ───────────────────────────────────────────
83	TASK = "Classify customer support tickets by urgency: low/medium/high/critical"
84	TEST_CASES = [
85	"My billing is wrong again, third time this month!",
86	"How do I change my password?",
87	"Server is down for all enterprise customers RIGHT NOW",
88	"What are your business hours?",
89	]
90
91	print("Generating initial prompt...")
92	prompt = generate_prompt(TASK)
93	print(f"Initial: {prompt[:100]}...
94	")
95
96	for iteration in range(2):
97	result = score_prompt(TASK, prompt, TEST_CASES)
98	print(f"Iteration {iteration + 1}: score={result['avg_score']:.1f}/10")
99	print(f" Issues: {result['issues'][:2]}")
100
101	if result["avg_score"] >= 8.5:
102	print("Prompt quality sufficient!")
103	break
104
105	prompt = improve_prompt(TASK, prompt, result["issues"])
106	print(f" Improved: {prompt[:100]}...
107	")
108
109	print(f"\nFinal prompt:\n{prompt}")
110