The Incident Report

The Incident Report: a nightly batch dies at 3am. The on-call's first three questions - what ran? what broke? what do I resume? - answered from the journal replay, formatted for the incident channel. Nobody greps logs at 3am if the journal can already speak.

source on github

a real captured run

INCIDENT REPORT - nightly batch
====================================================

impact:
  3/6 tasks completed before the stop
  ROOT CAUSE: load:warehouse - Agentic::Errors::LlmAuthenticationError
              "warehouse credentials expired"

completed (do NOT re-run - outputs are journaled):
  + extract:refunds        41ms
  + extract:orders         51ms
  + transform:ledger       71ms

never started (blocked behind the failure):
  . verify:totals
  . notify:finance

resume plan:
  1. rotate the warehouse credentials (error is LlmAuthenticationError:
     retryable? => false; retrying without fixing creds is theater)
  2. re-run the batch - completed?(description) will skip the
     3 journaled tasks; only 3 run
  3. budget: ~162ms of work already banked, don't pay twice

source

# frozen_string_literal: true

# The Incident Report: a nightly batch dies at 3am. The on-call's
# first three questions - what ran? what broke? what do I resume? -
# answered from the journal replay, formatted for the incident
# channel. Nobody greps logs at 3am if the journal can already speak.
#
#   bundle exec ruby examples/incident_report.rb
#
# Runs offline; the outage is scripted.

require class="s">"bundler/setup"
require class="s">"agentic"
require class="s">"tmpdir"

Agentic.logger.level = class="y">:fatal

JOURNAL = File.join(Dir.tmpdir, class="s">"agentic_incident.journal.jsonl")
File.delete(JOURNAL) if File.exist?(JOURNAL)

journal = Agentic:class="y">:ExecutionJournal.new(path: JOURNAL)

NIGHTLY = {
  class="s">"extract:orders" => {time: 0.05},
  class="s">"extract:refunds" => {time: 0.04},
  class="s">"transform:ledger" => {time: 0.07, deps: %w[extract:orders extract:refunds]},
  class="s">"load:warehouse" => {time: 0.03, deps: %w[transform:ledger],
                       error: Agentic:class="y">:Errors:class="y">:LlmAuthenticationError.new(class="s">"warehouse credentials expired")},
  class="s">"verify:totals" => {time: 0.02, deps: %w[load:warehouse]},
  class="s">"notify:finance" => {time: 0.01, deps: %w[verify:totals]}
}.freeze

orchestrator = nil
hooks = journal.lifecycle_hooks(
  after_task_failure: ->(task_id:, task:, failure:, duration:) { orchestrator.cancel_plan }
)
orchestrator = Agentic:class="y">:PlanOrchestrator.new(
  concurrency_limit: 2, lifecycle_hooks: hooks,
  retry_policy: {max_retries: 0, retryable_errors: []}
)

tasks = {}
NIGHTLY.each do |name, spec|
  tasks[name] = Agentic:class="y">:Task.new(description: name,
    agent_spec: {class="s">"name" => name, class="s">"instructions" => class="s">"run"}, payload: spec)
  orchestrator.add_task(tasks[name], (spec[class="y">:deps] || []).map { |d| tasks.fetch(d) }, agent: ->(t) {
    sleep(t.payload[class="y">:time])
    raise t.payload[class="y">:error] if t.payload[class="y">:error]

    class="y">:ok
  })
end
orchestrator.execute_plan

# --- the report: everything below reads ONLY the journal ---------------------
state = Agentic:class="y">:ExecutionJournal.replay(path: JOURNAL)
all_tasks = NIGHTLY.keys
completed = state.completed_descriptions
failed = state.events.select { |e| e[class="y">:event] == class="s">"task_failed" }
never_ran = all_tasks - completed - failed.map { |f| f[class="y">:description] }

puts class="s">"INCIDENT REPORT - nightly batch"
puts class="s">"=" * 52
puts
puts class="s">"impact:"
puts class="s">"  #{completed.size}/#{all_tasks.size} tasks completed before the stop"
failed.each do |f|
  puts class="s">"  ROOT CAUSE: #{f[class="y">:description]} - #{f[class="y">:error_type]}"
  puts class="s">"              \"#{f[class="y">:error]}\class="s">""
end
puts
puts class="s">"completed (do NOT re-run - outputs are journaled):"
completed.each { |d| puts format(class="s">"  + %-20s %4.0fms", d, (state.durations[d] || 0) * 1000) }
puts
puts class="s">"never started (blocked behind the failure):"
never_ran.each { |d| puts class="s">"  . #{d}" }
puts
puts class="s">"resume plan:"
puts class="s">"  1. rotate the warehouse credentials (error is LlmAuthenticationError:"
puts class="s">"     retryable? => false; retrying without fixing creds is theater)"
puts class="s">"  2. re-run the batch - completed?(description) will skip the"
puts class="s">"     #{completed.size} journaled tasks; only #{all_tasks.size - completed.size} run"
puts format(class="s">"  3. budget: ~%.0fms of work already banked, don't pay twice",
  state.durations.values.sum * 1000)