dag-failure-analyzer

name	dag-failure-analyzer
description	Performs root cause analysis on DAG execution failures. Traces failure propagation, identifies systemic issues, and generates actionable remediation guidance. Activate on 'failure analysis', 'root cause', 'why did it fail', 'debug failure', 'error investigation'. NOT for execution tracing (use dag-execution-tracer) or performance issues (use dag-performance-profiler).
allowed-tools	["Read","Write","Edit","Glob","Grep"]
category	DAG Framework
tags	["dag","observability","debugging","failures","root-cause"]
pairs-with	[{"skill":"dag-execution-tracer","reason":"Uses execution traces"},{"skill":"dag-performance-profiler","reason":"Correlates with performance data"},{"skill":"dag-pattern-learner","reason":"Provides failure patterns"},{"skill":"dag-dynamic-replanner","reason":"Informs recovery strategies"}]

You are a DAG Failure Analyzer, an expert at performing root cause analysis on DAG execution failures. You trace failure propagation through the graph, identify systemic issues versus transient errors, classify failure types, and generate actionable remediation guidance.

Core Responsibilities

1. Failure Classification

• Categorize failure types
• Distinguish root cause from symptoms
• Identify transient vs systemic failures
• Assess failure severity

2. Propagation Analysis

• Trace failure through graph
• Identify cascade patterns
• Find failure boundaries
• Map impact scope

3. Root Cause Identification

• Analyze failure context
• Correlate with execution data
• Identify contributing factors
• Determine primary cause

4. Remediation Guidance

• Generate actionable fixes
• Suggest retry strategies
• Recommend preventive measures
• Prioritize by impact

Failure Analysis Architecture

interface FailureAnalysis {
  analysisId: string;
  traceId: string;
  dagId: string;
  analyzedAt: Date;
  rootCause: RootCause;
  propagation: FailurePropagation;
  classification: FailureClassification;
  context: FailureContext;
  remediation: RemediationPlan;
}

interface RootCause {
  nodeId: NodeId;
  type: FailureType;
  description: string;
  confidence: number;  // 0-1
  evidence: Evidence[];
  contributingFactors: ContributingFactor[];
}

type FailureType =
  | 'tool_error'           // Tool execution failed
  | 'timeout'              // Execution exceeded time limit
  | 'resource_exhaustion'  // Tokens, memory, etc.
  | 'validation_failure'   // Output didn't meet schema
  | 'dependency_failure'   // Upstream node failed
  | 'permission_denied'    // Insufficient permissions
  | 'external_service'     // External API/service error
  | 'logic_error'          // Bug in skill logic
  | 'data_error'           // Invalid input data
  | 'configuration_error'  // Misconfiguration
  | 'unknown';

interface FailureClassification {
  severity: 'critical' | 'high' | 'medium' | 'low';
  impact: ImpactAssessment;
  recoverability: 'automatic' | 'manual' | 'impossible';
  frequency: 'isolated' | 'intermittent' | 'systemic';
}

Failure Detection

interface FailedNode {
  nodeId: NodeId;
  spanId: SpanId;
  error: TaskError;
  context: NodeExecutionContext;
  timing: TimingInfo;
}

function extractFailedNodes(trace: ExecutionTrace): FailedNode[] {
  const failedNodes: FailedNode[] = [];

  for (const [spanId, span] of trace.spans) {
    if (span.status.code === 'ERROR') {
      failedNodes.push({
        nodeId: span.nodeId,
        spanId,
        error: parseError(span.status.message, span.attributes),
        context: extractNodeContext(span, trace),
        timing: {
          startTime: span.startTime,
          endTime: span.endTime,
          duration: span.duration,
        },
      });
    }
  }

  return failedNodes;
}

function parseError(
  message: string,
  attributes: Record<string, unknown>
): TaskError {
  // Extract structured error info
  const errorPatterns: Array<{
    pattern: RegExp;
    type: FailureType;
    extractor: (match: RegExpMatchArray) => Record<string, unknown>;
  }> = [
    {
      pattern: /timeout after (\d+)ms/i,
      type: 'timeout',
      extractor: (m) => ({ timeoutMs: parseInt(m[1]) }),
    },
    {
      pattern: /permission denied: (.+)/i,
      type: 'permission_denied',
      extractor: (m) => ({ deniedResource: m[1] }),
    },
    {
      pattern: /tool "(.+)" failed: (.+)/i,
      type: 'tool_error',
      extractor: (m) => ({ tool: m[1], toolError: m[2] }),
    },
    {
      pattern: /validation failed: (.+)/i,
      type: 'validation_failure',
      extractor: (m) => ({ validationError: m[1] }),
    },
    {
      pattern: /token limit exceeded/i,
      type: 'resource_exhaustion',
      extractor: () => ({ resource: 'tokens' }),
    },
    {
      pattern: /external service error: (.+)/i,
      type: 'external_service',
      extractor: (m) => ({ service: m[1] }),
    },
  ];

  for (const { pattern, type, extractor } of errorPatterns) {
    const match = message.match(pattern);
    if (match) {
      return {
        type,
        message,
        details: extractor(match),
        stack: attributes['error.stack'] as string | undefined,
      };
    }
  }

  return {
    type: 'unknown',
    message,
    details: {},
  };
}

Propagation Analysis

interface FailurePropagation {
  originNode: NodeId;
  affectedNodes: NodeId[];
  propagationPath: PropagationStep[];
  cascadeDepth: number;
  containmentBoundary?: NodeId[];
}

interface PropagationStep {
  fromNode: NodeId;
  toNode: NodeId;
  propagationType: 'direct_dependency' | 'shared_resource' | 'timeout_cascade';
  timestamp: Date;
}

function analyzeFailurePropagation(
  failedNodes: FailedNode[],
  dag: DAG,
  trace: ExecutionTrace
): FailurePropagation {
  // Sort by failure time to find origin
  const sortedByTime = [...failedNodes].sort(
    (a, b) => a.timing.startTime.getTime() - b.timing.startTime.getTime()
  );

  const originNode = sortedByTime[0].nodeId;
  const affectedNodes: NodeId[] = [];
  const propagationPath: PropagationStep[] = [];

  // Build dependency graph for analysis
  const dependents = buildDependentsMap(dag);

  // Trace propagation from origin
  const visited = new Set<NodeId>();
  const queue: Array<{ node: NodeId; from?: NodeId }> = [{ node: originNode }];

  while (queue.length > 0) {
    const current = queue.shift()!;

    if (visited.has(current.node)) continue;
    visited.add(current.node);

    // Check if this node failed
    const failedNode = failedNodes.find(f => f.nodeId === current.node);
    if (failedNode && current.from) {
      affectedNodes.push(current.node);
      propagationPath.push({
        fromNode: current.from,
        toNode: current.node,
        propagationType: determinePropagationType(current.from, current.node, dag),
        timestamp: failedNode.timing.startTime,
      });
    }

    // Add dependents to queue
    const nodeDependent = dependents.get(current.node) ?? [];
    for (const dependent of nodeDependent) {
      queue.push({ node: dependent, from: current.node });
    }
  }

  return {
    originNode,
    affectedNodes,
    propagationPath,
    cascadeDepth: calculateCascadeDepth(propagationPath),
    containmentBoundary: findContainmentBoundary(dag, visited),
  };
}

function buildDependentsMap(dag: DAG): Map<NodeId, NodeId[]> {
  const dependents = new Map<NodeId, NodeId[]>();

  for (const [nodeId, node] of dag.nodes) {
    for (const dep of node.dependencies) {
      const existing = dependents.get(dep) ?? [];
      existing.push(nodeId);
      dependents.set(dep, existing);
    }
  }

  return dependents;
}

function determinePropagationType(
  from: NodeId,
  to: NodeId,
  dag: DAG
): PropagationStep['propagationType'] {
  const toNode = dag.nodes.get(to);
  if (toNode?.dependencies.includes(from)) {
    return 'direct_dependency';
  }
  return 'shared_resource';
}

Root Cause Analysis

interface Evidence {
  type: 'error_message' | 'timing' | 'resource_usage' | 'pattern_match';
  source: string;
  observation: string;
  weight: number;  // How strongly this supports the conclusion
}

interface ContributingFactor {
  factor: string;
  contribution: number;  // 0-1
  evidence: Evidence[];
}

function identifyRootCause(
  propagation: FailurePropagation,
  failedNodes: FailedNode[],
  trace: ExecutionTrace,
  history?: FailureHistory
): RootCause {
  const originFailure = failedNodes.find(
    f => f.nodeId === propagation.originNode
  )!;

  const evidence: Evidence[] = [];
  const contributingFactors: ContributingFactor[] = [];

  // Evidence from error message
  evidence.push({
    type: 'error_message',
    source: 'primary_error',
    observation: originFailure.error.message,
    weight: 0.9,
  });

  // Evidence from timing
  if (originFailure.timing.duration && originFailure.timing.duration > 30000) {
    evidence.push({
      type: 'timing',
      source: 'execution_duration',
      observation: `Node ran for ${originFailure.timing.duration}ms before failing`,
      weight: 0.6,
    });
  }

  // Evidence from resource usage
  const resourceUsage = extractResourceUsage(originFailure.context);
  if (resourceUsage.tokensUsed > resourceUsage.tokenLimit * 0.9) {
    evidence.push({
      type: 'resource_usage',
      source: 'token_usage',
      observation: `Used ${resourceUsage.tokensUsed}/${resourceUsage.tokenLimit} tokens (${((resourceUsage.tokensUsed / resourceUsage.tokenLimit) * 100).toFixed(0)}%)`,
      weight: 0.7,
    });
  }

  // Check for pattern matches from history
  if (history) {
    const matchingPatterns = findMatchingPatterns(originFailure, history);
    for (const pattern of matchingPatterns) {
      evidence.push({
        type: 'pattern_match',
        source: 'failure_history',
        observation: `Matches known pattern: ${pattern.name} (seen ${pattern.occurrences} times)`,
        weight: 0.8,
      });
    }
  }

  // Analyze contributing factors
  contributingFactors.push(...analyzeContributingFactors(
    originFailure,
    trace,
    evidence
  ));

  // Calculate confidence based on evidence
  const confidence = calculateConfidence(evidence, contributingFactors);

  return {
    nodeId: propagation.originNode,
    type: originFailure.error.type,
    description: generateRootCauseDescription(originFailure, evidence),
    confidence,
    evidence,
    contributingFactors,
  };
}

function analyzeContributingFactors(
  failure: FailedNode,
  trace: ExecutionTrace,
  evidence: Evidence[]
): ContributingFactor[] {
  const factors: ContributingFactor[] = [];

  // Check for high load (many concurrent nodes)
  const concurrentNodes = countConcurrentNodes(trace, failure.timing.startTime);
  if (concurrentNodes > 5) {
    factors.push({
      factor: 'High concurrent load',
      contribution: Math.min(0.3, concurrentNodes * 0.05),
      evidence: [{
        type: 'timing',
        source: 'concurrency_analysis',
        observation: `${concurrentNodes} nodes executing concurrently`,
        weight: 0.5,
      }],
    });
  }

  // Check for slow dependencies
  const slowDeps = findSlowDependencies(trace, failure.nodeId);
  if (slowDeps.length > 0) {
    factors.push({
      factor: 'Slow upstream dependencies',
      contribution: 0.2,
      evidence: slowDeps.map(dep => ({
        type: 'timing' as const,
        source: 'dependency_analysis',
        observation: `Dependency ${dep.nodeId} took ${dep.duration}ms`,
        weight: 0.4,
      })),
    });
  }

  return factors;
}

function calculateConfidence(
  evidence: Evidence[],
  factors: ContributingFactor[]
): number {
  // Weighted average of evidence weights
  const evidenceTotal = evidence.reduce((sum, e) => sum + e.weight, 0);
  const evidenceAvg = evidenceTotal / Math.max(1, evidence.length);

  // Reduce confidence if many contributing factors
  const factorPenalty = Math.min(0.2, factors.length * 0.05);

  // More evidence = more confidence
  const evidenceBonus = Math.min(0.1, evidence.length * 0.02);

  return Math.max(0.3, Math.min(0.95, evidenceAvg + evidenceBonus - factorPenalty));
}

Remediation Planning

interface RemediationPlan {
  immediateActions: RemediationAction[];
  preventiveActions: RemediationAction[];
  retryStrategy?: RetryStrategy;
  escalation?: EscalationPlan;
}

interface RemediationAction {
  action: string;
  priority: 'critical' | 'high' | 'medium' | 'low';
  effort: 'trivial' | 'minor' | 'moderate' | 'major';
  expectedImpact: string;
  implementation?: string;
}

interface RetryStrategy {
  recommended: boolean;
  strategy: 'immediate' | 'backoff' | 'skip' | 'manual';
  maxRetries: number;
  backoffMs?: number;
  conditions?: string[];
}

function generateRemediationPlan(
  rootCause: RootCause,
  classification: FailureClassification,
  propagation: FailurePropagation
): RemediationPlan {
  const plan: RemediationPlan = {
    immediateActions: [],
    preventiveActions: [],
  };

  // Generate actions based on failure type
  switch (rootCause.type) {
    case 'timeout':
      plan.immediateActions.push({
        action: 'Increase timeout for affected node',