Pythonprovider_benchmark.py
Copy
1
import os
2
import time
3
import statistics
4
from dataclasses import dataclass
5
 
6
import openai
7
import anthropic
8
 
9
@dataclass
10
class BenchmarkResult:
11
    provider: str
12
    model: str
13
    latency_ms: float
14
    input_tokens: int
15
    output_tokens: int
16
    cost_usd: float
17
    response: str
18
 
19
# Pricing per 1M tokens (as of 2024 — check provider docs for current)
20
PRICING = {
21
    "gpt-4o": {"input": 2.50, "output": 10.00},
22
    "gpt-4o-mini": {"input": 0.15, "output": 0.60},
23
    "claude-3-5-sonnet": {"input": 3.00, "output": 15.00},
24
    "claude-3-haiku": {"input": 0.25, "output": 1.25},
25
}
26
 
27
def calc_cost(model: str, input_tokens: int, output_tokens: int) -> float:
28
    if model not in PRICING:
29
        return 0.0
30
    p = PRICING[model]
31
    return (input_tokens * p["input"] + output_tokens * p["output"]) / 1_000_000
32
 
33
def benchmark_openai(prompt: str, model: str = "gpt-4o-mini") -> BenchmarkResult:
34
    client = openai.OpenAI(api_key=os.environ["OPENAI_API_KEY"])
35
    start = time.perf_counter()
36
    response = client.chat.completions.create(
37
        model=model,
38
        messages=[{"role": "user", "content": prompt}],
39
        max_tokens=200,
40
    )
41
    latency = (time.perf_counter() - start) * 1000
42
    usage = response.usage
43
    return BenchmarkResult(
44
        provider="openai",
45
        model=model,
46
        latency_ms=latency,
47
        input_tokens=usage.prompt_tokens,
48
        output_tokens=usage.completion_tokens,
49
        cost_usd=calc_cost(model, usage.prompt_tokens, usage.completion_tokens),
50
        response=response.choices[0].message.content,
51
    )
52
 
53
def benchmark_anthropic(prompt: str, model: str = "claude-3-haiku-20240307") -> BenchmarkResult:
54
    client = anthropic.Anthropic(api_key=os.environ["ANTHROPIC_API_KEY"])
55
    start = time.perf_counter()
56
    response = client.messages.create(
57
        model=model,
58
        max_tokens=200,
59
        messages=[{"role": "user", "content": prompt}],
60
    )
61
    latency = (time.perf_counter() - start) * 1000
62
    return BenchmarkResult(
63
        provider="anthropic",
64
        model=model,
65
        latency_ms=latency,
66
        input_tokens=response.usage.input_tokens,
67
        output_tokens=response.usage.output_tokens,
68
        cost_usd=calc_cost("claude-3-haiku", response.usage.input_tokens, response.usage.output_tokens),
69
        response=response.content[0].text,
70
    )
71
 
72
# Run benchmark
73
PROMPT = "Explain what a Merkle tree is in 3 sentences."
74
results = []
75
 
76
for run in range(3):
77
    results.append(benchmark_openai(PROMPT))
78
    results.append(benchmark_anthropic(PROMPT))
79
 
80
# Summary
81
for provider in ["openai", "anthropic"]:
82
    provider_results = [r for r in results if r.provider == provider]
83
    avg_latency = statistics.mean(r.latency_ms for r in provider_results)
84
    avg_cost = statistics.mean(r.cost_usd for r in provider_results)
85
    print(f"{provider}: avg_latency={avg_latency:.0f}ms, avg_cost=${avg_cost:.6f}")
86