Multi-Agent AI Systems: Moving From Demos to Production

// Sequential Pipeline: Planner → Coder → Reviewer → Tester
interface AgentResult {
  content: string;
  metadata: Record<string, unknown>;
  status: 'success' | 'failure' | 'needs_revision';
}

interface PipelineAgent {
  name: string;
  process(input: AgentResult): Promise<AgentResult>;
}

class SequentialPipeline {
  private agents: PipelineAgent[];

  constructor(agents: PipelineAgent[]) {
    this.agents = agents;
  }

  async execute(initialInput: string): Promise<AgentResult> {
    let result: AgentResult = {
      content: initialInput,
      metadata: {},
      status: 'success',
    };

    for (const agent of this.agents) {
      console.log(`  Pipeline stage: ${agent.name}`);
      result = await agent.process(result);

      if (result.status === 'failure') {
        console.error(`  Pipeline failed at ${agent.name}`);
        break;
      }
    }

    return result;
  }
}

// Usage
const pipeline = new SequentialPipeline([
  new PlannerAgent(),   // Decomposes the task into steps
  new CoderAgent(),     // Writes the implementation
  new ReviewerAgent(),  // Reviews for bugs and style
  new TesterAgent(),    // Generates and runs tests
]);

const result = await pipeline.execute(
  "Add a rate limiter middleware to the Express API"
);

// Supervisor/Worker: One orchestrator delegates to specialists
interface WorkerAgent {
  name: string;
  capabilities: string[];
  execute(task: string, context: SharedContext): Promise<AgentResult>;
}

interface SharedContext {
  goal: string;
  history: Array<{ agent: string; result: AgentResult }>;
  artifacts: Map<string, string>;
}

class SupervisorAgent {
  private workers: WorkerAgent[];
  private llm: LLMClient;

  constructor(workers: WorkerAgent[], llm: LLMClient) {
    this.workers = workers;
    this.llm = llm;
  }

  async solve(goal: string, maxRounds: number = 10): Promise<string> {
    const context: SharedContext = {
      goal,
      history: [],
      artifacts: new Map(),
    };

    for (let round = 0; round < maxRounds; round++) {
      // Ask the supervisor LLM which worker to invoke next
      const decision = await this.llm.decide({
        goal,
        availableWorkers: this.workers.map(w => ({
          name: w.name,
          capabilities: w.capabilities,
        })),
        history: context.history,
      });

      if (decision.action === 'finish') {
        return decision.finalAnswer;
      }

      // Dispatch to the chosen worker
      const worker = this.workers.find(w => w.name === decision.workerName);
      if (!worker) throw new Error(`Unknown worker: ${decision.workerName}`);

      console.log(`  Supervisor delegating to: ${worker.name}`);
      const result = await worker.execute(decision.task, context);
      context.history.push({ agent: worker.name, result });

      // Store any artifacts the worker produced
      if (result.metadata.artifact) {
        context.artifacts.set(
          result.metadata.artifactName as string,
          result.content
        );
      }
    }

    throw new Error('Supervisor exceeded max rounds');
  }
}

# Peer-to-Peer: Agents negotiate as equals
from dataclasses import dataclass, field
from typing import Callable
import asyncio

@dataclass
class Message:
    sender: str
    receiver: str
    content: str
    msg_type: str  # "request", "response", "broadcast"

class PeerAgent:
    def __init__(self, name: str, capabilities: list[str], handler: Callable):
        self.name = name
        self.capabilities = capabilities
        self.handler = handler
        self.inbox: asyncio.Queue[Message] = asyncio.Queue()
        self.peers: dict[str, "PeerAgent"] = {}

    def register_peer(self, peer: "PeerAgent"):
        self.peers[peer.name] = peer

    async def send(self, receiver: str, content: str, msg_type: str = "request"):
        msg = Message(self.name, receiver, content, msg_type)
        await self.peers[receiver].inbox.put(msg)

    async def broadcast(self, content: str):
        for peer_name, peer in self.peers.items():
            msg = Message(self.name, peer_name, content, "broadcast")
            await peer.inbox.put(msg)

    async def run(self):
        """Main loop: process messages from inbox."""
        while True:
            msg = await self.inbox.get()
            response = await self.handler(self, msg)
            if response and msg.msg_type == "request":
                await self.send(msg.sender, response, "response")

# Example: design review with peer agents
async def architect_handler(agent: PeerAgent, msg: Message) -> str:
    if "review architecture" in msg.content:
        return "Architecture looks solid. Consider adding a cache layer."
    return None

async def security_handler(agent: PeerAgent, msg: Message) -> str:
    if "security review" in msg.content:
        return "Found 2 issues: SQL injection risk in query builder, missing rate limit."
    return None

architect = PeerAgent("architect", ["design", "architecture"], architect_handler)
security = PeerAgent("security", ["security", "compliance"], security_handler)
architect.register_peer(security)
security.register_peer(architect)

// Hierarchical: Multi-level agent trees
interface HierarchicalAgent {
  name: string;
  level: number;
  children: HierarchicalAgent[];
  execute(task: string): Promise<AgentResult>;
}

class ManagerAgent implements HierarchicalAgent {
  name: string;
  level: number;
  children: HierarchicalAgent[];
  private llm: LLMClient;

  constructor(name: string, level: number, children: HierarchicalAgent[]) {
    this.name = name;
    this.level = level;
    this.children = children;
  }

  async execute(task: string): Promise<AgentResult> {
    // Decompose task into subtasks for children
    const subtasks = await this.llm.decompose(task, this.children.map(c => c.name));

    const results: AgentResult[] = [];
    for (const subtask of subtasks) {
      const child = this.children.find(c => c.name === subtask.assignee);
      console.log(`  [${this.name}] Delegating to ${child.name}: ${subtask.description}`);
      const result = await child.execute(subtask.description);
      results.push(result);
    }

    // Synthesize child results
    const synthesis = await this.llm.synthesize(task, results);
    return { content: synthesis, metadata: { level: this.level }, status: 'success' };
  }
}

// Build the hierarchy
const system = new ManagerAgent("CTO", 0, [
  new ManagerAgent("Backend Lead", 1, [
    new LeafAgent("API Developer"),
    new LeafAgent("Database Engineer"),
  ]),
  new ManagerAgent("Frontend Lead", 1, [
    new LeafAgent("UI Developer"),
    new LeafAgent("UX Specialist"),
  ]),
  new ManagerAgent("QA Lead", 1, [
    new LeafAgent("Test Engineer"),
    new LeafAgent("Performance Tester"),
  ]),
]);

await system.execute("Build a user dashboard with real-time analytics");

// Define clear agent contracts
interface AgentDefinition {
  name: string;
  role: string;
  systemPrompt: string;
  tools: ToolDefinition[];
  inputSchema: z.ZodSchema;
  outputSchema: z.ZodSchema;
}

const agents: AgentDefinition[] = [
  {
    name: "diff-analyzer",
    role: "Analyze the PR diff and categorize changes",
    systemPrompt: `You are a code diff analyst. Your job is to:
      1. Parse the git diff
      2. Categorize each change (new feature, bug fix, refactor, config)
      3. Identify files with the highest risk
      4. Output a structured analysis`,
    tools: [readFileTool, gitDiffTool],
    inputSchema: z.object({ prNumber: z.number() }),
    outputSchema: z.object({
      files: z.array(z.object({
        path: z.string(),
        changeType: z.enum(["added", "modified", "deleted"]),
        riskLevel: z.enum(["low", "medium", "high"]),
        summary: z.string(),
      })),
    }),
  },
  {
    name: "bug-detector",
    role: "Find potential bugs and logic errors",
    systemPrompt: `You are a bug detection specialist. Analyze code changes for:
      - Null/undefined handling issues
      - Off-by-one errors
      - Race conditions
      - Missing error handling
      - Type mismatches
      Report each issue with severity, file, line, and explanation.`,
    tools: [readFileTool, searchCodeTool],
    inputSchema: z.object({ files: z.array(z.string()), diff: z.string() }),
    outputSchema: z.object({
      issues: z.array(z.object({
        severity: z.enum(["critical", "warning", "info"]),
        file: z.string(),
        line: z.number(),
        description: z.string(),
        suggestion: z.string(),
      })),
    }),
  },
  {
    name: "security-scanner",
    role: "Identify security vulnerabilities",
    systemPrompt: `You are a security specialist. Scan for:
      - SQL injection, XSS, CSRF vulnerabilities
      - Hardcoded secrets or credentials
      - Insecure dependencies
      - Missing input validation
      - Authentication/authorization gaps`,
    tools: [readFileTool, dependencyCheckTool],
    inputSchema: z.object({ files: z.array(z.string()) }),
    outputSchema: z.object({
      vulnerabilities: z.array(z.object({
        severity: z.enum(["critical", "high", "medium", "low"]),
        type: z.string(),
        file: z.string(),
        description: z.string(),
        remediation: z.string(),
      })),
    }),
  },
  {
    name: "review-synthesizer",
    role: "Combine all findings into a final review",
    systemPrompt: `You are a senior engineer writing a PR review. Synthesize
      findings from the analysis, bug detection, and security scan into a
      clear, actionable review. Prioritize critical issues first.`,
    tools: [],
    inputSchema: z.object({
      analysis: z.unknown(),
      bugs: z.unknown(),
      security: z.unknown(),
    }),
    outputSchema: z.object({
      summary: z.string(),
      verdict: z.enum(["approve", "request_changes", "comment"]),
      comments: z.array(z.object({
        file: z.string(),
        line: z.number(),
        body: z.string(),
      })),
    }),
  },
];

// Structured inter-agent communication
interface AgentMessage {
  id: string;
  from: string;
  to: string;
  type: 'task' | 'result' | 'error' | 'clarification';
  payload: unknown;
  timestamp: number;
  traceId: string;  // For distributed tracing
}

class MessageBus {
  private handlers: Map<string, (msg: AgentMessage) => Promise<void>> = new Map();
  private messageLog: AgentMessage[] = [];

  register(agentName: string, handler: (msg: AgentMessage) => Promise<void>) {
    this.handlers.set(agentName, handler);
  }

  async send(message: AgentMessage): Promise<void> {
    this.messageLog.push(message);
    console.log(`  [${message.from} → ${message.to}] ${message.type}`);

    const handler = this.handlers.get(message.to);
    if (!handler) throw new Error(`No handler for agent: ${message.to}`);
    await handler(message);
  }

  getTrace(traceId: string): AgentMessage[] {
    return this.messageLog.filter(m => m.traceId === traceId);
  }
}

// Production orchestrator with retries and timeouts
import Anthropic from "@anthropic-ai/sdk";

interface OrchestratorConfig {
  maxRetries: number;
  timeoutMs: number;
  tokenBudget: number;
}

class ReviewOrchestrator {
  private client: Anthropic;
  private config: OrchestratorConfig;
  private tokenUsage: number = 0;

  constructor(config: OrchestratorConfig) {
    this.client = new Anthropic();
    this.config = config;
  }

  async reviewPR(prNumber: number): Promise<ReviewResult> {
    const traceId = crypto.randomUUID();
    console.log(`Starting review for PR #${prNumber} [trace: ${traceId}]`);

    // Stage 1: Analyze the diff
    const analysis = await this.runAgent("diff-analyzer", {
      prNumber,
    }, traceId);

    // Stage 2: Run bug detection and security scan in parallel
    const [bugs, security] = await Promise.all([
      this.runAgent("bug-detector", {
        files: analysis.files.map(f => f.path),
        diff: analysis.rawDiff,
      }, traceId),
      this.runAgent("security-scanner", {
        files: analysis.files.map(f => f.path),
      }, traceId),
    ]);

    // Stage 3: Synthesize into final review
    const review = await this.runAgent("review-synthesizer", {
      analysis,
      bugs,
      security,
    }, traceId);

    console.log(`Review complete. Tokens used: ${this.tokenUsage}`);
    return review;
  }

  private async runAgent(
    agentName: string,
    input: unknown,
    traceId: string,
    attempt: number = 0
  ): Promise<unknown> {
    if (this.tokenUsage > this.config.tokenBudget) {
      throw new Error(`Token budget exceeded: ${this.tokenUsage}/${this.config.tokenBudget}`);
    }

    try {
      const result = await Promise.race([
        this.executeAgent(agentName, input, traceId),
        this.timeout(this.config.timeoutMs),
      ]);
      return result;
    } catch (error) {
      if (attempt < this.config.maxRetries) {
        console.warn(`  Retrying ${agentName} (attempt ${attempt + 1})`);
        return this.runAgent(agentName, input, traceId, attempt + 1);
      }
      throw error;
    }
  }

  private timeout(ms: number): Promise<never> {
    return new Promise((_, reject) =>
      setTimeout(() => reject(new Error("Agent timeout")), ms)
    );
  }
}

// Shared state management for multi-agent workflows
interface WorkflowState {
  id: string;
  status: 'running' | 'completed' | 'failed';
  agents: Record<string, AgentState>;
  artifacts: Map<string, Artifact>;
  context: Record<string, unknown>;
}

interface AgentState {
  status: 'pending' | 'running' | 'completed' | 'failed';
  startedAt?: number;
  completedAt?: number;
  tokensUsed: number;
  result?: unknown;
  error?: string;
}

interface Artifact {
  key: string;
  producedBy: string;
  content: string;
  contentType: string;
  createdAt: number;
}

class WorkflowMemory {
  private state: WorkflowState;

  constructor(workflowId: string) {
    this.state = {
      id: workflowId,
      status: 'running',
      agents: {},
      artifacts: new Map(),
      context: {},
    };
  }

  // Agents write artifacts for other agents to consume
  setArtifact(agentName: string, key: string, content: string, contentType: string) {
    this.state.artifacts.set(key, {
      key,
      producedBy: agentName,
      content,
      contentType,
      createdAt: Date.now(),
    });
  }

  getArtifact(key: string): Artifact | undefined {
    return this.state.artifacts.get(key);
  }

  // Shared context that any agent can read or update
  setContext(key: string, value: unknown) {
    this.state.context[key] = value;
  }

  getContext(key: string): unknown {
    return this.state.context[key];
  }

  // Track agent lifecycle
  markAgentStarted(name: string) {
    this.state.agents[name] = {
      status: 'running',
      startedAt: Date.now(),
      tokensUsed: 0,
    };
  }

  markAgentCompleted(name: string, result: unknown, tokensUsed: number) {
    this.state.agents[name] = {
      ...this.state.agents[name],
      status: 'completed',
      completedAt: Date.now(),
      tokensUsed,
      result,
    };
  }

  // Snapshot for observability
  getSnapshot(): WorkflowState {
    return structuredClone(this.state);
  }
}

// Production error handling for multi-agent systems
class CircuitBreaker {
  private failures: number = 0;
  private lastFailure: number = 0;
  private state: 'closed' | 'open' | 'half-open' = 'closed';

  constructor(
    private threshold: number = 3,
    private resetTimeMs: number = 60_000
  ) {}

  async execute<T>(fn: () => Promise<T>): Promise<T> {
    if (this.state === 'open') {
      if (Date.now() - this.lastFailure > this.resetTimeMs) {
        this.state = 'half-open';
      } else {
        throw new Error('Circuit breaker is open');
      }
    }

    try {
      const result = await fn();
      this.onSuccess();
      return result;
    } catch (error) {
      this.onFailure();
      throw error;
    }
  }

  private onSuccess() {
    this.failures = 0;
    this.state = 'closed';
  }

  private onFailure() {
    this.failures++;
    this.lastFailure = Date.now();
    if (this.failures >= this.threshold) {
      this.state = 'open';
    }
  }
}

// Fallback strategy: if the primary agent fails, use a simpler fallback
class AgentWithFallback {
  constructor(
    private primary: AgentDefinition,
    private fallback: AgentDefinition,
    private breaker: CircuitBreaker = new CircuitBreaker()
  ) {}

  async execute(input: unknown): Promise<AgentResult> {
    try {
      return await this.breaker.execute(() =>
        runAgent(this.primary, input)
      );
    } catch (error) {
      console.warn(
        `Primary agent ${this.primary.name} failed, using fallback ${this.fallback.name}`
      );
      return runAgent(this.fallback, input);
    }
  }
}

# LangGraph: Multi-agent workflow as a graph
from langgraph.graph import StateGraph, START, END
from typing import TypedDict, Annotated
import operator

class ReviewState(TypedDict):
    pr_diff: str
    analysis: str
    bugs: list[str]
    security_issues: list[str]
    final_review: str

def analyze_diff(state: ReviewState) -> ReviewState:
    """Agent 1: Analyze the diff and categorize changes."""
    analysis = llm.invoke(
        f"Analyze this PR diff and categorize the changes:\n{state['pr_diff']}"
    )
    return {"analysis": analysis.content}

def detect_bugs(state: ReviewState) -> ReviewState:
    """Agent 2: Find potential bugs."""
    bugs = llm.invoke(
        f"Find bugs in this code analysis:\n{state['analysis']}"
    )
    return {"bugs": [bugs.content]}

def scan_security(state: ReviewState) -> ReviewState:
    """Agent 3: Security scan."""
    issues = llm.invoke(
        f"Find security vulnerabilities:\n{state['analysis']}"
    )
    return {"security_issues": [issues.content]}

def synthesize_review(state: ReviewState) -> ReviewState:
    """Agent 4: Combine into final review."""
    review = llm.invoke(
        f"""Synthesize a PR review from:
        Analysis: {state['analysis']}
        Bugs: {state['bugs']}
        Security: {state['security_issues']}"""
    )
    return {"final_review": review.content}

def route_after_analysis(state: ReviewState) -> list[str]:
    """Run bug detection and security scan in parallel."""
    return ["detect_bugs", "scan_security"]

# Build the graph
graph = StateGraph(ReviewState)
graph.add_node("analyze_diff", analyze_diff)
graph.add_node("detect_bugs", detect_bugs)
graph.add_node("scan_security", scan_security)
graph.add_node("synthesize_review", synthesize_review)

graph.add_edge(START, "analyze_diff")
graph.add_conditional_edges("analyze_diff", route_after_analysis)
graph.add_edge("detect_bugs", "synthesize_review")
graph.add_edge("scan_security", "synthesize_review")
graph.add_edge("synthesize_review", END)

app = graph.compile()
result = app.invoke({"pr_diff": diff_content})

# CrewAI: Agents as team members
from crewai import Agent, Task, Crew, Process

analyst = Agent(
    role="Senior Code Analyst",
    goal="Thoroughly analyze code changes and identify patterns",
    backstory="You have 15 years of experience reviewing code at top tech companies.",
    verbose=True,
    llm="claude-sonnet-4-20250514",
)

security_expert = Agent(
    role="Application Security Engineer",
    goal="Find security vulnerabilities before they reach production",
    backstory="Former penetration tester turned security engineer. You have seen every attack vector.",
    verbose=True,
    llm="claude-sonnet-4-20250514",
)

reviewer = Agent(
    role="Tech Lead",
    goal="Make the final review decision based on all findings",
    backstory="Staff engineer who balances velocity with quality. You know when to block and when to approve.",
    verbose=True,
    llm="claude-sonnet-4-20250514",
)

analyze_task = Task(
    description="Analyze the PR diff: {pr_diff}",
    expected_output="Structured analysis of all changes with risk levels",
    agent=analyst,
)

security_task = Task(
    description="Scan for security vulnerabilities in the changes",
    expected_output="List of security issues with severity and remediation",
    agent=security_expert,
)

review_task = Task(
    description="Write the final PR review combining all findings",
    expected_output="Complete PR review with verdict and inline comments",
    agent=reviewer,
)

crew = Crew(
    agents=[analyst, security_expert, reviewer],
    tasks=[analyze_task, security_task, review_task],
    process=Process.sequential,
    verbose=True,
)

result = crew.kickoff(inputs={"pr_diff": diff_content})

# AutoGen: Conversational multi-agent collaboration
from autogen import AssistantAgent, UserProxyAgent, GroupChat, GroupChatManager

coder = AssistantAgent(
    name="Coder",
    system_message="""You are an expert programmer. Write clean, well-tested code.
    When given a task, implement it and explain your design decisions.""",
    llm_config={"model": "claude-sonnet-4-20250514"},
)

reviewer = AssistantAgent(
    name="Reviewer",
    system_message="""You are a code reviewer. Examine code for bugs, style issues,
    and architectural problems. Be thorough but constructive.""",
    llm_config={"model": "claude-sonnet-4-20250514"},
)

tester = AssistantAgent(
    name="Tester",
    system_message="""You write comprehensive test cases. Cover edge cases,
    error conditions, and performance scenarios.""",
    llm_config={"model": "claude-sonnet-4-20250514"},
)

executor = UserProxyAgent(
    name="Executor",
    human_input_mode="NEVER",
    code_execution_config={"work_dir": "workspace"},
)

group_chat = GroupChat(
    agents=[coder, reviewer, tester, executor],
    messages=[],
    max_round=12,
)

manager = GroupChatManager(groupchat=group_chat)
executor.initiate_chat(
    manager,
    message="Build a rate limiter middleware for Express.js with sliding window algorithm"
)

// Claude Agent SDK: Production-focused agent building
import Anthropic from "@anthropic-ai/sdk";

const anthropic = new Anthropic();

// Define a specialized agent with scoped tools
async function createReviewAgent(
  role: string,
  systemPrompt: string,
  tools: Anthropic.Tool[]
) {
  return async function execute(task: string): Promise<string> {
    const messages: Anthropic.MessageParam[] = [
      { role: "user", content: task },
    ];

    let totalTokens = 0;

    while (true) {
      const response = await anthropic.messages.create({
        model: "claude-sonnet-4-20250514",
        max_tokens: 4096,
        system: systemPrompt,
        tools,
        messages,
      });

      totalTokens += response.usage.input_tokens + response.usage.output_tokens;

      if (response.stop_reason === "end_turn") {
        const text = response.content.find(b => b.type === "text");
        console.log(`  [${role}] completed (${totalTokens} tokens)`);
        return text?.text ?? "";
      }

      if (response.stop_reason === "tool_use") {
        const toolUses = response.content.filter(b => b.type === "tool_use");
        messages.push({ role: "assistant", content: response.content });

        const toolResults = await Promise.all(
          toolUses.map(async (tu) => ({
            type: "tool_result" as const,
            tool_use_id: tu.id,
            content: await executeTool(tu.name, tu.input),
          }))
        );

        messages.push({ role: "user", content: toolResults });
      }
    }
  };
}

// Orchestrate multiple agents
async function multiAgentReview(prDiff: string) {
  const analyzer = await createReviewAgent(
    "Analyzer",
    "You analyze code diffs and categorize changes by risk level.",
    [readFileTool, gitLogTool]
  );

  const bugHunter = await createReviewAgent(
    "Bug Hunter",
    "You find bugs, logic errors, and edge cases in code.",
    [readFileTool, searchTool]
  );

  const analysis = await analyzer(`Analyze this diff:\n${prDiff}`);

  const bugs = await bugHunter(
    `Given this analysis:\n${analysis}\nFind all potential bugs.`
  );

  return { analysis, bugs };
}

// Model routing: match model capability to task complexity
const MODEL_TIERS = {
  simple: "claude-haiku-4-20250514",       // Fast, cheap — formatting, routing, classification
  standard: "claude-sonnet-4-20250514",   // Balanced — most agent tasks
  complex: "claude-opus-4-20250514",     // Most capable — complex reasoning, synthesis
} as const;

interface AgentConfig {
  name: string;
  modelTier: keyof typeof MODEL_TIERS;
  maxTokens: number;
  tokenBudget: number;
}

const agentConfigs: AgentConfig[] = [
  { name: "router",           modelTier: "simple",   maxTokens: 256,  tokenBudget: 500 },
  { name: "diff-analyzer",    modelTier: "simple",   maxTokens: 1024, tokenBudget: 2000 },
  { name: "bug-detector",     modelTier: "standard", maxTokens: 2048, tokenBudget: 5000 },
  { name: "security-scanner", modelTier: "standard", maxTokens: 2048, tokenBudget: 5000 },
  { name: "synthesizer",      modelTier: "complex",  maxTokens: 4096, tokenBudget: 8000 },
];

function getModelForAgent(agentName: string): string {
  const config = agentConfigs.find(c => c.name === agentName);
  return MODEL_TIERS[config?.modelTier ?? "standard"];
}

// Semantic caching for agent results
import { createHash } from "crypto";

class AgentCache {
  private cache: Map<string, { result: unknown; timestamp: number }> = new Map();
  private ttlMs: number;

  constructor(ttlMs: number = 3600_000) {  // 1 hour default
    this.ttlMs = ttlMs;
  }

  private hashInput(agentName: string, input: unknown): string {
    const content = JSON.stringify({ agentName, input });
    return createHash("sha256").update(content).digest("hex");
  }

  get(agentName: string, input: unknown): unknown | null {
    const hash = this.hashInput(agentName, input);
    const entry = this.cache.get(hash);

    if (!entry) return null;
    if (Date.now() - entry.timestamp > this.ttlMs) {
      this.cache.delete(hash);
      return null;
    }

    console.log(`  Cache hit for ${agentName}`);
    return entry.result;
  }

  set(agentName: string, input: unknown, result: unknown): void {
    const hash = this.hashInput(agentName, input);
    this.cache.set(hash, { result, timestamp: Date.now() });
  }
}

// Multi-agent observability layer
interface AgentSpan {
  traceId: string;
  spanId: string;
  parentSpanId?: string;
  agentName: string;
  startTime: number;
  endTime?: number;
  input: unknown;
  output?: unknown;
  tokensUsed: number;
  model: string;
  toolCalls: ToolCallRecord[];
  status: 'success' | 'failure' | 'timeout';
  error?: string;
}

interface ToolCallRecord {
  tool: string;
  input: unknown;
  output: unknown;
  durationMs: number;
}

class AgentTracer {
  private spans: AgentSpan[] = [];

  startSpan(traceId: string, agentName: string, input: unknown, parentSpanId?: string): string {
    const spanId = crypto.randomUUID();
    this.spans.push({
      traceId,
      spanId,
      parentSpanId,
      agentName,
      startTime: Date.now(),
      input,
      tokensUsed: 0,
      model: '',
      toolCalls: [],
      status: 'success',
    });
    return spanId;
  }

  endSpan(spanId: string, output: unknown, tokensUsed: number, status: 'success' | 'failure') {
    const span = this.spans.find(s => s.spanId === spanId);
    if (span) {
      span.endTime = Date.now();
      span.output = output;
      span.tokensUsed = tokensUsed;
      span.status = status;
    }
  }

  // Generate a dashboard-ready summary
  getTraceSummary(traceId: string) {
    const traceSpans = this.spans.filter(s => s.traceId === traceId);
    return {
      totalDuration: Math.max(...traceSpans.map(s => s.endTime ?? 0)) -
                     Math.min(...traceSpans.map(s => s.startTime)),
      totalTokens: traceSpans.reduce((sum, s) => sum + s.tokensUsed, 0),
      agentCount: traceSpans.length,
      failedAgents: traceSpans.filter(s => s.status === 'failure').map(s => s.agentName),
      timeline: traceSpans.map(s => ({
        agent: s.agentName,
        duration: (s.endTime ?? Date.now()) - s.startTime,
        tokens: s.tokensUsed,
        status: s.status,
      })),
    };
  }
}