Prompt Engineering: Techniques & Patterns

// Prompt engineering in TypeScript: pattern catalog, role/context/instruction
// composition, multi-provider request formatters, simulated LLM dispatch with
// deterministic scoring. Mirrors code/prompt_engineering.py.
// Sources:
//   https://platform.openai.com/docs/guides/text-generation
//   https://docs.anthropic.com/en/docs/build-with-claude/prompt-engineering
//   https://ai.google.dev/gemini-api/docs/text-generation

import { createHash } from "node:crypto";

type PatternName =
  | "persona"
  | "few_shot"
  | "chain_of_thought"
  | "template_fill"
  | "critique"
  | "guardrail"
  | "decomposition"
  | "audience_adapt"
  | "boundary";

type Pattern = {
  readonly name: string;
  readonly template: string;
  readonly variables: readonly string[];
  readonly temperature: number;
  readonly description: string;
};

const PROMPT_PATTERNS: Readonly<Record<PatternName, Pattern>> = {
  persona: {
    name: "Persona Pattern",
    template:
      "You are {role} with {experience}.\nYour communication style is {style}.\nYou prioritize {priority}.\n\n{task}",
    variables: ["role", "experience", "style", "priority", "task"],
    temperature: 0.7,
    description: "Activates a specific expert distribution in the training data",
  },
  few_shot: {
    name: "Few-Shot Pattern",
    template: "Here are examples of the expected input/output format:\n\n{examples}\n\nNow process this input:\n{input}",
    variables: ["examples", "input"],
    temperature: 0.0,
    description: "Anchors output format with concrete examples",
  },
  chain_of_thought: {
    name: "Chain-of-Thought Pattern",
    template:
      "Think through this step by step.\n\nProblem: {problem}\n\nSteps:\n1. Identify the key components\n2. Analyze each component\n3. Synthesize your findings\n4. State your conclusion\n\nShow your reasoning before the final answer.",
    variables: ["problem"],
    temperature: 0.3,
    description: "Forces explicit reasoning before the final answer",
  },
  template_fill: {
    name: "Template Fill Pattern",
    template:
      "Extract information from the following text and fill in the template.\n\nText: {text}\n\nTemplate:\n{template_structure}\n\nFill every field. If unknown, write 'N/A'.",
    variables: ["text", "template_structure"],
    temperature: 0.0,
    description: "Constrains output to named fields",
  },
  critique: {
    name: "Critique Pattern",
    template:
      "Task: {task}\n\nStep 1: Generate an initial response.\nStep 2: Critique it for accuracy, completeness, and clarity.\nStep 3: Produce an improved final version.\n\nLabel each step clearly.",
    variables: ["task"],
    temperature: 0.5,
    description: "Self-refinement through explicit critique",
  },
  guardrail: {
    name: "Guardrail Pattern",
    template:
      "You are a {role}.\n\nRules:\n- ONLY answer questions about {domain}\n- If outside {domain}, say: 'This is outside my scope.'\n- NEVER make up information. If unsure, say 'I don't know.'\n- {additional_rules}\n\nUser question: {question}",
    variables: ["role", "domain", "additional_rules", "question"],
    temperature: 0.3,
    description: "Constrains to a domain with explicit boundaries",
  },
  decomposition: {
    name: "Decomposition Pattern",
    template:
      "Problem: {problem}\n\nBreak this into sub-problems:\n1. List each sub-problem\n2. Solve each independently\n3. Combine sub-solutions into a final answer\n4. Verify the final answer against the original problem",
    variables: ["problem"],
    temperature: 0.3,
    description: "Breaks complex problems into manageable pieces",
  },
  audience_adapt: {
    name: "Audience Adaptation Pattern",
    template:
      "Explain {concept} for the following audience: {audience}.\n\nConstraints:\n- Vocabulary appropriate for {audience}\n- Length: {length}\n- Include {include}\n- Exclude {exclude}",
    variables: ["concept", "audience", "length", "include", "exclude"],
    temperature: 0.5,
    description: "Adapts explanation to the target audience",
  },
  boundary: {
    name: "Boundary Pattern",
    template:
      "You are an assistant that ONLY handles {scope}.\n\nIf the request is in scope, help fully.\nIf out of scope, respond exactly with:\n'{refusal_message}'\n\nDo not attempt to answer out-of-scope questions.\n\nUser: {user_input}",
    variables: ["scope", "refusal_message", "user_input"],
    temperature: 0.0,
    description: "Hard boundary on what the model responds to",
  },
} as const;

type Provider = "openai" | "anthropic" | "google";

type ModelConfig = {
  readonly provider: Provider;
  readonly model: string;
  readonly maxTokens: number;
  readonly contextWindow: number;
};

const MODEL_CONFIGS: Readonly<Record<string, ModelConfig>> = {
  "gpt-4o": { provider: "openai", model: "gpt-4o", maxTokens: 2048, contextWindow: 128_000 },
  "claude-3.5-sonnet": { provider: "anthropic", model: "claude-3-5-sonnet-20241022", maxTokens: 2048, contextWindow: 200_000 },
  "gemini-1.5-pro": { provider: "google", model: "gemini-1.5-pro", maxTokens: 2048, contextWindow: 2_000_000 },
};

type BuiltPrompt = {
  readonly system: string;
  readonly user: string;
  readonly temperature: number;
  readonly pattern: PatternName;
  readonly metadata: { description: string; variablesUsed: readonly string[] };
};

function renderTemplate(template: string, vars: Readonly<Record<string, string>>): string {
  return template.replace(/\{(\w+)\}/g, (_, name: string) => {
    const value = vars[name];
    if (value === undefined) throw new Error("Missing template variable: " + name);
    return value;
  });
}

function buildPrompt(
  patternName: PatternName,
  variables: Readonly<Record<string, string>>,
  systemOverride?: string,
): BuiltPrompt {
  const pattern = PROMPT_PATTERNS[patternName];
  const missing = pattern.variables.filter((v) => !(v in variables));
  if (missing.length > 0) {
    throw new Error("Missing variables for " + patternName + ": " + missing.join(","));
  }
  const rendered = renderTemplate(pattern.template, variables);
  const system = systemOverride ?? "You are an AI assistant using the " + pattern.name + ".";
  return {
    system,
    user: rendered,
    temperature: pattern.temperature,
    pattern: patternName,
    metadata: { description: pattern.description, variablesUsed: Object.keys(variables) },
  };
}

type OpenAIRequest = {
  model: string;
  messages: ReadonlyArray<{ role: "system" | "user"; content: string }>;
  temperature: number;
  max_tokens: number;
};

type AnthropicRequest = {
  model: string;
  system: string;
  messages: ReadonlyArray<{ role: "user"; content: string }>;
  temperature: number;
  max_tokens: number;
};

type GoogleRequest = {
  model: string;
  contents: ReadonlyArray<{ role: "user"; parts: ReadonlyArray<{ text: string }> }>;
  generationConfig: { temperature: number; maxOutputTokens: number };
};

type ProviderRequest = OpenAIRequest | AnthropicRequest | GoogleRequest;

function formatOpenAI(p: BuiltPrompt, cfg: ModelConfig): OpenAIRequest {
  return {
    model: cfg.model,
    messages: [
      { role: "system", content: p.system },
      { role: "user", content: p.user },
    ],
    temperature: p.temperature,
    max_tokens: cfg.maxTokens,
  };
}

function formatAnthropic(p: BuiltPrompt, cfg: ModelConfig): AnthropicRequest {
  return {
    model: cfg.model,
    system: p.system,
    messages: [{ role: "user", content: p.user }],
    temperature: p.temperature,
    max_tokens: cfg.maxTokens,
  };
}

function formatGoogle(p: BuiltPrompt, cfg: ModelConfig): GoogleRequest {
  return {
    model: cfg.model,
    contents: [{ role: "user", parts: [{ text: p.system + "\n\n" + p.user }] }],
    generationConfig: { temperature: p.temperature, maxOutputTokens: cfg.maxTokens },
  };
}

const FORMATTERS: Readonly<Record<Provider, (p: BuiltPrompt, c: ModelConfig) => ProviderRequest>> = {
  openai: formatOpenAI,
  anthropic: formatAnthropic,
  google: formatGoogle,
};

type SimulatedResponse = {
  response: string;
  tokensUsed: { prompt: number; completion: number; total: number };
  latencyMs: number;
  finishReason: string;
};

function simulateLlmCall(modelName: string, request: ProviderRequest): SimulatedResponse {
  const promptHash = createHash("md5").update(JSON.stringify(request)).digest("hex").slice(0, 8);
  const responses: Record<string, SimulatedResponse> = {
    "gpt-4o": {
      response: "[GPT-4o " + promptHash + "] Simulated response. Thorough and well-structured.",
      tokensUsed: { prompt: 150, completion: 45, total: 195 },
      latencyMs: 850,
      finishReason: "stop",
    },
    "claude-3.5-sonnet": {
      response: "[Claude 3.5 Sonnet " + promptHash + "] Simulated response. Direct and precise.",
      tokensUsed: { prompt: 145, completion: 40, total: 185 },
      latencyMs: 720,
      finishReason: "end_turn",
    },
    "gemini-1.5-pro": {
      response: "[Gemini 1.5 Pro " + promptHash + "] Simulated response. Comprehensive grounding.",
      tokensUsed: { prompt: 155, completion: 42, total: 197 },
      latencyMs: 900,
      finishReason: "STOP",
    },
  };
  return responses[modelName] ?? {
    response: "Unknown model",
    tokensUsed: { prompt: 0, completion: 0, total: 0 },
    latencyMs: 0,
    finishReason: "unknown",
  };
}

type Criteria = {
  maxWords?: number;
  requiredKeywords?: readonly string[];
  forbiddenPhrases?: readonly string[];
  expectedFormat?: "json" | "bullet_points" | "numbered_list";
};

type Score = {
  wordCount?: number;
  lengthCompliant?: boolean;
  keywordsFound?: readonly string[];
  keywordCoverage?: number;
  forbiddenViolations?: readonly string[];
  noViolations?: boolean;
  formatValid?: boolean;
  compositeScore: number;
};

function scoreResponse(text: string, criteria: Criteria): Score {
  const lower = text.toLowerCase();
  const score: Mutable<Score> = { compositeScore: 0 };
  const components: number[] = [];

  if (criteria.maxWords !== undefined) {
    const wc = text.trim().split(/\s+/).length;
    score.wordCount = wc;
    score.lengthCompliant = wc <= criteria.maxWords;
    components.push(score.lengthCompliant ? 1 : 0);
  }
  if (criteria.requiredKeywords) {
    const found = criteria.requiredKeywords.filter((kw) => lower.includes(kw.toLowerCase()));
    score.keywordsFound = found;
    score.keywordCoverage = criteria.requiredKeywords.length === 0 ? 1 : found.length / criteria.requiredKeywords.length;
    components.push(score.keywordCoverage);
  }
  if (criteria.forbiddenPhrases) {
    const violations = criteria.forbiddenPhrases.filter((p) => lower.includes(p.toLowerCase()));
    score.forbiddenViolations = violations;
    score.noViolations = violations.length === 0;
    components.push(score.noViolations ? 1 : 0);
  }
  if (criteria.expectedFormat) {
    if (criteria.expectedFormat === "json") {
      try {
        JSON.parse(text);
        score.formatValid = true;
      } catch {
        score.formatValid = false;
      }
    } else if (criteria.expectedFormat === "bullet_points") {
      const lines = text.split("\n").map((l) => l.trim()).filter((l) => l.length > 0);
      const bullets = lines.filter((l) => /^\s*[-*+•]\s+/.test(l));
      score.formatValid = bullets.length >= lines.length * 0.5;
    } else {
      score.formatValid = /^\d+\./m.test(text);
    }
    components.push(score.formatValid ? 1 : 0);
  }

  score.compositeScore = components.length === 0 ? 0 : components.reduce((a, b) => a + b, 0) / components.length;
  return score;
}

type Mutable<T> = { -readonly [K in keyof T]: T[K] };

type ModelResult = {
  response: string;
  tokens: SimulatedResponse["tokensUsed"];
  apiLatencyMs: number;
  wallTimeMs: number;
  finishReason: string;
  requestPayload: ProviderRequest;
};

function runPromptTest(prompt: BuiltPrompt, models: readonly string[] = Object.keys(MODEL_CONFIGS)): Record<string, ModelResult> {
  const out: Record<string, ModelResult> = {};
  for (const name of models) {
    const cfg = MODEL_CONFIGS[name];
    if (!cfg) {
      throw new Error("Unknown model: " + name + ". Available models: " + Object.keys(MODEL_CONFIGS).join(", "));
    }
    const request = FORMATTERS[cfg.provider](prompt, cfg);
    const start = Date.now();
    const response = simulateLlmCall(name, request);
    out[name] = {
      response: response.response,
      tokens: response.tokensUsed,
      apiLatencyMs: response.latencyMs,
      wallTimeMs: Date.now() - start,
      finishReason: response.finishReason,
      requestPayload: request,
    };
  }
  return out;
}

function compareModels(results: Record<string, ModelResult>, criteria: Criteria): Array<{ model: string; score: number; tokens: number; latency: number }> {
  const ranked = Object.entries(results).map(([model, r]) => ({
    model,
    score: scoreResponse(r.response, criteria).compositeScore,
    tokens: r.tokens.total,
    latency: r.apiLatencyMs,
  }));
  ranked.sort((a, b) => b.score - a.score);
  return ranked;
}

function main(): void {
  console.log("=".repeat(60));
  console.log("  PROMPT PATTERN CATALOG");
  console.log("=".repeat(60));
  for (const [name, pattern] of Object.entries(PROMPT_PATTERNS)) {
    console.log("\n  [" + name + "] " + pattern.name);
    console.log("    " + pattern.description);
    console.log("    Variables: " + pattern.variables.join(", "));
    console.log("    Recommended temp: " + pattern.temperature);
  }

  console.log("\n" + "=".repeat(60));
  console.log("  SINGLE PROMPT BUILD + TEST");
  console.log("=".repeat(60));

  const prompt = buildPrompt("persona", {
    role: "a senior DevOps engineer at Netflix",
    experience: "8 years of infrastructure automation",
    style: "direct and practical",
    priority: "reliability over speed",
    task: "Explain why container orchestration matters for microservices.",
  });
  console.log("\n  System: " + prompt.system);
  console.log("  Temperature: " + prompt.temperature);

  const results = runPromptTest(prompt);
  for (const [model, r] of Object.entries(results)) {
    console.log("\n  [" + model + "]");
    console.log("    Response: " + r.response.slice(0, 100));
    console.log("    Tokens: " + JSON.stringify(r.tokens));
    console.log("    Latency: " + r.apiLatencyMs + "ms");
  }

  type TestCase = { name: string; pattern: PatternName; variables: Record<string, string>; criteria: Criteria };
  const suite: readonly TestCase[] = [
    {
      name: "Persona: Technical Writer",
      pattern: "persona",
      variables: {
        role: "a senior technical writer at Stripe",
        experience: "10 years of API documentation",
        style: "precise and example-driven",
        priority: "clarity over comprehensiveness",
        task: "Explain what an API rate limit is and why it exists.",
      },
      criteria: { maxWords: 200, requiredKeywords: ["Simulated"], forbiddenPhrases: ["in conclusion"] },
    },
    {
      name: "Chain-of-Thought: Math",
      pattern: "chain_of_thought",
      variables: { problem: "20% discount on $85 vs $10 coupon. Which order saves more?" },
      criteria: { requiredKeywords: ["Simulated"], maxWords: 300 },
    },
    {
      name: "Guardrail: Scoped Assistant",
      pattern: "guardrail",
      variables: {
        role: "Python programming tutor",
        domain: "Python programming",
        additional_rules: "Do not write complete solutions.",
        question: "How do I sort a list of dictionaries by a key?",
      },
      criteria: { requiredKeywords: ["Simulated"] },
    },
  ];

  console.log("\n" + "=".repeat(60));
  console.log("  TEST SUITE");
  console.log("=".repeat(60));
  for (const test of suite) {
    const p = buildPrompt(test.pattern, test.variables);
    const rs = runPromptTest(p);
    const ranked = compareModels(rs, test.criteria);
    console.log("\n  Test: " + test.name);
    console.log("  Pattern: " + test.pattern);
    for (const r of ranked) {
      console.log("    " + r.model.padEnd(20) + " score=" + r.score.toFixed(3) + " tokens=" + r.tokens + " latency=" + r.latency + "ms");
    }
  }
}

main();

import json
import time
import hashlib
import re


PROMPT_PATTERNS = {
    "persona": {
        "name": "Persona Pattern",
        "template": (
            "You are {role} with {experience}.\n"
            "Your communication style is {style}.\n"
            "You prioritize {priority}.\n\n"
            "{task}"
        ),
        "variables": ["role", "experience", "style", "priority", "task"],
        "temperature": 0.7,
        "description": "Activates a specific expert distribution in the model's training data",
    },
    "few_shot": {
        "name": "Few-Shot Pattern",
        "template": (
            "Here are examples of the expected input/output format:\n\n"
            "{examples}\n\n"
            "Now process this input:\n{input}"
        ),
        "variables": ["examples", "input"],
        "temperature": 0.0,
        "description": "Provides concrete examples to anchor the output format and style",
    },
    "chain_of_thought": {
        "name": "Chain-of-Thought Pattern",
        "template": (
            "Think through this step by step.\n\n"
            "Problem: {problem}\n\n"
            "Steps:\n"
            "1. Identify the key components\n"
            "2. Analyze each component\n"
            "3. Synthesize your findings\n"
            "4. State your conclusion\n\n"
            "Show your reasoning before giving the final answer."
        ),
        "variables": ["problem"],
        "temperature": 0.3,
        "description": "Forces explicit reasoning steps before the final answer",
    },
    "template_fill": {
        "name": "Template Fill Pattern",
        "template": (
            "Extract information from the following text and fill in the template.\n\n"
            "Text: {text}\n\n"
            "Template:\n{template_structure}\n\n"
            "Fill in every field. If information is not available, write 'N/A'."
        ),
        "variables": ["text", "template_structure"],
        "temperature": 0.0,
        "description": "Constrains output to a specific structure with named fields",
    },
    "critique": {
        "name": "Critique Pattern",
        "template": (
            "Task: {task}\n\n"
            "Step 1: Generate an initial response.\n"
            "Step 2: Critique your response for accuracy, completeness, and clarity.\n"
            "Step 3: Produce an improved final version.\n\n"
            "Label each step clearly."
        ),
        "variables": ["task"],
        "temperature": 0.5,
        "description": "Self-refinement through explicit critique before final output",
    },
    "guardrail": {
        "name": "Guardrail Pattern",
        "template": (
            "You are a {role}.\n\n"
            "Rules:\n"
            "- ONLY answer questions about {domain}\n"
            "- If the question is outside {domain}, say: 'This is outside my scope.'\n"
            "- NEVER make up information. If unsure, say 'I don't know.'\n"
            "- {additional_rules}\n\n"
            "User question: {question}"
        ),
        "variables": ["role", "domain", "additional_rules", "question"],
        "temperature": 0.3,
        "description": "Constrains the model to a specific domain with explicit boundaries",
    },
    "meta_prompt": {
        "name": "Meta-Prompt Pattern",
        "template": (
            "Write a prompt for an LLM that will {objective}.\n\n"
            "The prompt should include:\n"
            "- A specific role/persona\n"
            "- Clear constraints and output format\n"
            "- 2-3 few-shot examples\n"
            "- Edge case handling\n\n"
            "Optimize the prompt for {metric}.\n"
            "Target model: {model}."
        ),
        "variables": ["objective", "metric", "model"],
        "temperature": 0.7,
        "description": "Uses the LLM to generate optimized prompts for other tasks",
    },
    "decomposition": {
        "name": "Decomposition Pattern",
        "template": (
            "Problem: {problem}\n\n"
            "Break this into sub-problems:\n"
            "1. List each sub-problem\n"
            "2. Solve each independently\n"
            "3. Combine sub-solutions into a final answer\n"
            "4. Verify the final answer against the original problem"
        ),
        "variables": ["problem"],
        "temperature": 0.3,
        "description": "Breaks complex problems into manageable pieces",
    },
    "audience_adapt": {
        "name": "Audience Adaptation Pattern",
        "template": (
            "Explain {concept} for the following audience: {audience}.\n\n"
            "Constraints:\n"
            "- Use vocabulary appropriate for {audience}\n"
            "- Length: {length}\n"
            "- Include {include}\n"
            "- Exclude {exclude}"
        ),
        "variables": ["concept", "audience", "length", "include", "exclude"],
        "temperature": 0.5,
        "description": "Adapts explanation complexity to the target audience",
    },
    "boundary": {
        "name": "Boundary Pattern",
        "template": (
            "You are an assistant that ONLY handles {scope}.\n\n"
            "If the user's request is within scope, help them fully.\n"
            "If the user's request is outside scope, respond exactly with:\n"
            "'{refusal_message}'\n\n"
            "Do not attempt to answer out-of-scope questions.\n\n"
            "User: {user_input}"
        ),
        "variables": ["scope", "refusal_message", "user_input"],
        "temperature": 0.0,
        "description": "Hard boundary on what the model will and will not respond to",
    },
}


MODEL_CONFIGS = {
    "gpt-4o": {
        "provider": "openai",
        "model": "gpt-4o",
        "max_tokens": 2048,
        "context_window": 128_000,
    },
    "claude-3.5-sonnet": {
        "provider": "anthropic",
        "model": "claude-3-5-sonnet-20241022",
        "max_tokens": 2048,
        "context_window": 200_000,
    },
    "gemini-1.5-pro": {
        "provider": "google",
        "model": "gemini-1.5-pro",
        "max_tokens": 2048,
        "context_window": 2_000_000,
    },
}


def build_prompt(pattern_name, variables, system_override=None):
    pattern = PROMPT_PATTERNS.get(pattern_name)
    if not pattern:
        raise ValueError(f"Unknown pattern: {pattern_name}. Available: {list(PROMPT_PATTERNS.keys())}")

    missing = [v for v in pattern["variables"] if v not in variables]
    if missing:
        raise ValueError(f"Missing variables for {pattern_name}: {missing}")

    rendered = pattern["template"].format(**variables)
    system = system_override or f"You are an AI assistant using the {pattern['name']}."

    return {
        "system": system,
        "user": rendered,
        "temperature": pattern["temperature"],
        "pattern": pattern_name,
        "metadata": {
            "description": pattern["description"],
            "variables_used": list(variables.keys()),
        },
    }


def build_multi_turn(pattern_name, turns, system_override=None):
    pattern = PROMPT_PATTERNS.get(pattern_name)
    if not pattern:
        raise ValueError(f"Unknown pattern: {pattern_name}")

    system = system_override or f"You are an AI assistant using the {pattern['name']}."
    messages = [{"role": "system", "content": system}]
    for role, content in turns:
        messages.append({"role": role, "content": content})

    return {
        "messages": messages,
        "temperature": pattern["temperature"],
        "pattern": pattern_name,
    }


def format_openai_request(prompt):
    return {
        "model": MODEL_CONFIGS["gpt-4o"]["model"],
        "messages": [
            {"role": "system", "content": prompt["system"]},
            {"role": "user", "content": prompt["user"]},
        ],
        "temperature": prompt["temperature"],
        "max_tokens": MODEL_CONFIGS["gpt-4o"]["max_tokens"],
    }


def format_anthropic_request(prompt):
    return {
        "model": MODEL_CONFIGS["claude-3.5-sonnet"]["model"],
        "system": prompt["system"],
        "messages": [
            {"role": "user", "content": prompt["user"]},
        ],
        "temperature": prompt["temperature"],
        "max_tokens": MODEL_CONFIGS["claude-3.5-sonnet"]["max_tokens"],
    }


def format_google_request(prompt):
    return {
        "model": MODEL_CONFIGS["gemini-1.5-pro"]["model"],
        "contents": [
            {"role": "user", "parts": [{"text": f"{prompt['system']}\n\n{prompt['user']}"}]},
        ],
        "generationConfig": {
            "temperature": prompt["temperature"],
            "maxOutputTokens": MODEL_CONFIGS["gemini-1.5-pro"]["max_tokens"],
        },
    }


FORMATTERS = {
    "openai": format_openai_request,
    "anthropic": format_anthropic_request,
    "google": format_google_request,
}


def simulate_llm_call(model_name, request):
    time.sleep(0.01)
    prompt_hash = hashlib.md5(json.dumps(request, sort_keys=True).encode()).hexdigest()[:8]

    simulated_responses = {
        "gpt-4o": {
            "response": (
                f"[GPT-4o response {prompt_hash}] This is a simulated response. "
                "GPT-4o tends to be thorough and well-structured with strong instruction following."
            ),
            "tokens_used": {"prompt": 150, "completion": 45, "total": 195},
            "latency_ms": 850,
            "finish_reason": "stop",
        },
        "claude-3.5-sonnet": {
            "response": (
                f"[Claude 3.5 Sonnet response {prompt_hash}] This is a simulated response. "
                "Claude tends to be direct, precise, and follows system instructions closely."
            ),
            "tokens_used": {"prompt": 145, "completion": 40, "total": 185},
            "latency_ms": 720,
            "finish_reason": "end_turn",
        },
        "gemini-1.5-pro": {
            "response": (
                f"[Gemini 1.5 Pro response {prompt_hash}] This is a simulated response. "
                "Gemini tends to be comprehensive with strong factual grounding."
            ),
            "tokens_used": {"prompt": 155, "completion": 42, "total": 197},
            "latency_ms": 900,
            "finish_reason": "STOP",
        },
    }

    return simulated_responses.get(
        model_name,
        {"response": "Unknown model", "tokens_used": {}, "latency_ms": 0},
    )


def run_prompt_test(prompt, models=None):
    if models is None:
        models = list(MODEL_CONFIGS.keys())

    results = {}
    for model_name in models:
        config = MODEL_CONFIGS[model_name]
        formatter = FORMATTERS[config["provider"]]
        request = formatter(prompt)

        start = time.time()
        response = simulate_llm_call(model_name, request)
        wall_time = (time.time() - start) * 1000

        results[model_name] = {
            "response": response["response"],
            "tokens": response["tokens_used"],
            "api_latency_ms": response["latency_ms"],
            "wall_time_ms": round(wall_time, 1),
            "finish_reason": response.get("finish_reason"),
            "request_payload": request,
        }

    return results


def score_response(response_text, criteria):
    scores = {}

    if "max_words" in criteria:
        word_count = len(response_text.split())
        scores["word_count"] = word_count
        scores["length_compliant"] = word_count <= criteria["max_words"]

    if "required_keywords" in criteria:
        found = [kw for kw in criteria["required_keywords"] if kw.lower() in response_text.lower()]
        scores["keywords_found"] = found
        scores["keyword_coverage"] = (
            len(found) / len(criteria["required_keywords"])
            if criteria["required_keywords"]
            else 1.0
        )

    if "forbidden_phrases" in criteria:
        violations = [fp for fp in criteria["forbidden_phrases"] if fp.lower() in response_text.lower()]
        scores["forbidden_violations"] = violations
        scores["no_violations"] = len(violations) == 0

    if "expected_format" in criteria:
        fmt = criteria["expected_format"]
        if fmt == "json":
            try:
                json.loads(response_text)
                scores["format_valid"] = True
            except (json.JSONDecodeError, TypeError):
                scores["format_valid"] = False
        elif fmt == "bullet_points":
            lines = [line.strip() for line in response_text.split("\n") if line.strip()]
            bullet_lines = [line for line in lines if line.startswith(("-", "*", "1"))]
            scores["format_valid"] = len(bullet_lines) >= len(lines) * 0.5
        elif fmt == "numbered_list":
            numbered = re.findall(r"^\d+\.", response_text, re.MULTILINE)
            scores["format_valid"] = len(numbered) >= 2
        else:
            scores["format_valid"] = True

    total = 0
    count = 0
    for key, value in scores.items():
        if isinstance(value, bool):
            total += 1.0 if value else 0.0
            count += 1
        elif isinstance(value, float) and 0 <= value <= 1:
            total += value
            count += 1

    scores["composite_score"] = round(total / count, 3) if count > 0 else 0.0
    return scores


def compare_models(test_results, criteria):
    comparison = {}
    for model_name, result in test_results.items():
        scores = score_response(result["response"], criteria)
        comparison[model_name] = {
            "scores": scores,
            "tokens": result["tokens"],
            "latency_ms": result["api_latency_ms"],
        }

    ranked = sorted(
        comparison.items(),
        key=lambda x: x[1]["scores"]["composite_score"],
        reverse=True,
    )
    return comparison, ranked


TEST_SUITE = [
    {
        "name": "Persona: Technical Writer",
        "pattern": "persona",
        "variables": {
            "role": "a senior technical writer at Stripe",
            "experience": "10 years of API documentation experience",
            "style": "precise, concise, and example-driven",
            "priority": "clarity over comprehensiveness",
            "task": "Explain what an API rate limit is and why it exists.",
        },
        "criteria": {
            "max_words": 200,
            "required_keywords": ["rate limit", "API", "requests"],
            "forbidden_phrases": ["in conclusion", "it is important to note"],
        },
    },
    {
        "name": "Few-Shot: Sentiment Analysis",
        "pattern": "few_shot",
        "variables": {
            "examples": (
                'Input: "The food was amazing but service was slow"\n'
                'Output: {"sentiment": "mixed", "food": "positive", "service": "negative"}\n\n'
                'Input: "Terrible experience, never coming back"\n'
                'Output: {"sentiment": "negative", "food": null, "service": "negative"}'
            ),
            "input": "Great ambiance and the pasta was perfect, though a bit pricey",
        },
        "criteria": {
            "expected_format": "json",
            "required_keywords": ["sentiment"],
        },
    },
    {
        "name": "Chain-of-Thought: Math Problem",
        "pattern": "chain_of_thought",
        "variables": {
            "problem": (
                "A store offers 20% off all items. An item originally costs $85. "
                "There is also a $10 coupon. Which saves more: applying the discount "
                "first then the coupon, or the coupon first then the discount?"
            ),
        },
        "criteria": {
            "required_keywords": ["discount", "coupon", "$"],
            "max_words": 300,
        },
    },
    {
        "name": "Template Fill: Resume Extraction",
        "pattern": "template_fill",
        "variables": {
            "text": (
                "John Smith is a software engineer at Google with 5 years of experience. "
                "He graduated from MIT with a BS in Computer Science in 2019. "
                "He specializes in distributed systems and Go programming."
            ),
            "template_structure": (
                "Name: [full name]\n"
                "Company: [current employer]\n"
                "Years of Experience: [number]\n"
                "Education: [degree, school, year]\n"
                "Specialties: [comma-separated list]"
            ),
        },
        "criteria": {
            "required_keywords": ["John Smith", "Google", "MIT"],
        },
    },
    {
        "name": "Guardrail: Scoped Assistant",
        "pattern": "guardrail",
        "variables": {
            "role": "Python programming tutor",
            "domain": "Python programming",
            "additional_rules": "Do not write complete solutions. Guide the student with hints.",
            "question": "How do I sort a list of dictionaries by a specific key?",
        },
        "criteria": {
            "required_keywords": ["sorted", "key", "lambda"],
            "forbidden_phrases": ["here is the complete solution"],
        },
    },
]


def run_test_suite():
    print("=" * 70)
    print("  PROMPT ENGINEERING TEST SUITE")
    print("=" * 70)

    all_results = []

    for test in TEST_SUITE:
        print(f"\n{'=' * 60}")
        print(f"  Test: {test['name']}")
        print(f"  Pattern: {test['pattern']}")
        print(f"{'=' * 60}")

        prompt = build_prompt(test["pattern"], test["variables"])
        print(f"\n  System: {prompt['system'][:80]}...")
        print(f"  User prompt: {prompt['user'][:120]}...")
        print(f"  Temperature: {prompt['temperature']}")

        results = run_prompt_test(prompt)
        comparison, ranked = compare_models(results, test["criteria"])

        print(f"\n  {'Model':<25} {'Score':>8} {'Tokens':>8} {'Latency':>10}")
        print(f"  {'-' * 55}")
        for model_name, data in ranked:
            score = data["scores"]["composite_score"]
            tokens = data["tokens"].get("total", 0)
            latency = data["latency_ms"]
            print(f"  {model_name:<25} {score:>8.3f} {tokens:>8} {latency:>8}ms")

        all_results.append({
            "test": test["name"],
            "pattern": test["pattern"],
            "rankings": [(name, data["scores"]["composite_score"]) for name, data in ranked],
        })

    print(f"\n\n{'=' * 70}")
    print("  SUMMARY: MODEL RANKINGS ACROSS ALL TESTS")
    print(f"{'=' * 70}")

    model_wins = {}
    for result in all_results:
        if result["rankings"]:
            winner = result["rankings"][0][0]
            model_wins[winner] = model_wins.get(winner, 0) + 1

    for model, wins in sorted(model_wins.items(), key=lambda x: x[1], reverse=True):
        print(f"  {model}: {wins} wins out of {len(all_results)} tests")

    return all_results


def run_pattern_catalog_demo():
    print("=" * 70)
    print("  PROMPT PATTERN CATALOG")
    print("=" * 70)

    for name, pattern in PROMPT_PATTERNS.items():
        print(f"\n  [{name}] {pattern['name']}")
        print(f"    {pattern['description']}")
        print(f"    Variables: {', '.join(pattern['variables'])}")
        print(f"    Recommended temp: {pattern['temperature']}")


def run_single_prompt_demo():
    print(f"\n{'=' * 70}")
    print("  SINGLE PROMPT BUILD + TEST")
    print("=" * 70)

    prompt = build_prompt("persona", {
        "role": "a senior DevOps engineer at Netflix",
        "experience": "8 years of infrastructure automation",
        "style": "direct and practical",
        "priority": "reliability over speed",
        "task": "Explain why container orchestration matters for microservices.",
    })

    print(f"\n  System message:\n    {prompt['system']}")
    print(f"\n  User message:\n    {prompt['user'][:200]}...")
    print(f"\n  Temperature: {prompt['temperature']}")
    print(f"\n  Pattern metadata: {json.dumps(prompt['metadata'], indent=4)}")

    results = run_prompt_test(prompt)
    for model, result in results.items():
        print(f"\n  [{model}]")
        print(f"    Response: {result['response'][:100]}...")
        print(f"    Tokens: {result['tokens']}")
        print(f"    Latency: {result['api_latency_ms']}ms")


if __name__ == "__main__":
    run_pattern_catalog_demo()
    run_single_prompt_demo()
    run_test_suite()