Prompt Engineering Fundamentals

# test_cases.yaml content (loaded as Python dict for demo)
test_config = {
    "prompt_template": "Classify the sentiment of this review: {text}\nRespond with exactly one word: POSITIVE, NEGATIVE, or NEUTRAL",
    "model": "gpt-4o",
    "temperature": 0.0,
    "test_cases": [
        {
            "input": {"text": "This product is amazing! Best purchase ever."},
            "expected": "POSITIVE",
            "tags": ["clear_positive"]
        },
        {
            "input": {"text": "Terrible quality. Broke after one day."},
            "expected": "NEGATIVE",
            "tags": ["clear_negative"]
        },
        {
            "input": {"text": "It's okay. Does what it says."},
            "expected": "NEUTRAL",
            "tags": ["subtle"]
        },
        {
            "input": {"text": "Not bad, but not great either. Shipping was fast though."},
            "expected": "NEUTRAL",
            "tags": ["subtle", "mixed_signals"]
        }
    ]
}

import openai
import yaml
import json
from dataclasses import dataclass
from typing import Optional
from pathlib import Path

client = openai.OpenAI()

@dataclass
class TestResult:
    input_data: dict
    expected: str
    actual: str
    passed: bool
    latency_ms: float
    tags: list[str]

def run_prompt_tests(config_path: str) -> list[TestResult]:
    """Run all test cases from a YAML config file."""
    import time
    
    with open(config_path) as f:
        config = yaml.safe_load(f)
    
    results = []
    
    for case in config["test_cases"]:
        # Format the prompt with test inputs
        prompt = config["prompt_template"].format(**case["input"])
        
        start = time.perf_counter()
        response = client.chat.completions.create(
            model=config["model"],
            messages=[{"role": "user", "content": prompt}],
            temperature=config["temperature"],
            max_tokens=50
        )
        latency = (time.perf_counter() - start) * 1000
        
        actual = response.choices[0].message.content.strip()
        passed = actual.upper() == case["expected"].upper()
        
        results.append(TestResult(
            input_data=case["input"],
            expected=case["expected"],
            actual=actual,
            passed=passed,
            latency_ms=latency,
            tags=case.get("tags", [])
        ))
    
    return results

def print_report(results: list[TestResult]) -> None:
    """Print a formatted test report."""
    total = len(results)
    passed = sum(1 for r in results if r.passed)
    
    print(f"\n{'='*60}")
    print(f"PROMPT TEST REPORT")
    print(f"{'='*60}")
    print(f"Total: {total} | Passed: {passed} | Failed: {total - passed}")
    print(f"Pass Rate: {passed/total*100:.1f}%")
    print(f"Avg Latency: {sum(r.latency_ms for r in results)/total:.0f}ms")
    print(f"{'-'*60}")
    
    for i, r in enumerate(results, 1):
        status = "✅" if r.passed else "❌"
        print(f"{status} Test {i}: expected={r.expected}, got={r.actual}")
        if not r.passed:
            print(f"   Input: {r.input_data}")
            print(f"   Tags: {r.tags}")
    
    # Tag-level analysis
    print(f"\n{'─'*60}")
    print("RESULTS BY TAG:")
    tag_results: dict[str, list[bool]] = {}
    for r in results:
        for tag in r.tags:
            tag_results.setdefault(tag, []).append(r.passed)
    
    for tag, passes in sorted(tag_results.items()):
        rate = sum(passes) / len(passes) * 100
        print(f"  [{tag}]: {rate:.0f}% ({sum(passes)}/{len(passes)})")

# Run it
if __name__ == "__main__":
    results = run_prompt_tests("test_cases.yaml")
    print_report(results)