The Dead Letter Office

The Dead Letter Office: three days of journaled runs, every failure collected and triaged by what the errors said about themselves - retryable failures go on the requeue manifest, non-retryable ones get parked with a reason. Nobody re-sends a letter addressed to a revoked mailbox.

source on github

a real captured run

DEAD LETTER OFFICE (3 journaled runs)

  REQUEUE MANIFEST (transient failures - safe to retry):
    sync:billing     LlmRateLimitError: 429, still angry        2 failed attempt(s) on record
    sync:tickets     LlmServerError: 502 from upstream          1 failed attempt(s) on record

  PARKED (retrying will not help - a human must act):
    sync:warehouse   LlmAuthenticationError: 401 key revoked

  recovered on their own (failed once, succeeded later - NOT dead):
    sync:crm

  the office triages by MOST RECENT attempt: sync:crm's old timeout
  doesn't page anyone, and sync:tickets' early success doesn't
  excuse its newer 502. a dead-letter queue that forgets recoveries
  pages people for ghosts; one that forgets relapses buries real mail.
  and each verdict above came from the journal itself - retryability
  was recorded when the error was fresh, not reconstructed later.

source

# frozen_string_literal: true

# The Dead Letter Office: three days of journaled runs, every failure
# collected and triaged by what the errors said about themselves -
# retryable failures go on the requeue manifest, non-retryable ones
# get parked with a reason. Nobody re-sends a letter addressed to a
# revoked mailbox.
#
#   bundle exec ruby examples/dead_letter_office.rb
#
# Runs offline; failures are scripted across runs.

require class="s">"bundler/setup"
require class="s">"agentic"
require class="s">"tmpdir"

Agentic.logger.level = class="y">:fatal

JOURNAL = File.join(Dir.tmpdir, class="s">"agentic_dlo.journal.jsonl")
File.delete(JOURNAL) if File.exist?(JOURNAL)
journal = Agentic:class="y">:ExecutionJournal.new(path: JOURNAL)

RUNS = [
  {class="s">"sync:crm" => Agentic:class="y">:Errors:class="y">:LlmTimeoutError.new(class="s">"read timeout"),
   class="s">"sync:billing" => Agentic:class="y">:Errors:class="y">:LlmRateLimitError.new(class="s">"429, slow down"),
   class="s">"sync:tickets" => nil},
  {class="s">"sync:crm" => nil, # recovers on its own
   class="s">"sync:billing" => Agentic:class="y">:Errors:class="y">:LlmRateLimitError.new(class="s">"429, still angry"),
   class="s">"sync:warehouse" => Agentic:class="y">:Errors:class="y">:LlmAuthenticationError.new(class="s">"401 key revoked")},
  {class="s">"sync:tickets" => Agentic:class="y">:Errors:class="y">:LlmServerError.new(class="s">"502 from upstream"),
   class="s">"sync:warehouse" => Agentic:class="y">:Errors:class="y">:LlmAuthenticationError.new(class="s">"401 key revoked")}
].freeze

RUNS.each do |jobs|
  orchestrator = Agentic:class="y">:PlanOrchestrator.new(
    concurrency_limit: 2, lifecycle_hooks: journal.lifecycle_hooks,
    retry_policy: {max_retries: 0, retryable_errors: []}
  )
  jobs.each do |name, error|
    orchestrator.add_task(Agentic:class="y">:Task.new(
      description: name, agent_spec: {class="s">"name" => name, class="s">"instructions" => class="s">"sync"},
      payload: error
    ), agent: ->(t) {
      sleep(0.005)
      raise t.payload if t.payload

      class="y">:ok
    })
  end
  orchestrator.execute_plan
end

# --- the office: triage from the journal --------------------------------------
state = Agentic:class="y">:ExecutionJournal.replay(path: JOURNAL)

# A letter is DEAD only if its most recent attempt failed
latest = {}
state.events.each do |event|
  next unless %w[task_succeeded task_failed].include?(event[class="y">:event])

  latest[event[class="y">:description]] = event
end
dead = latest.values.select { |e| e[class="y">:event] == class="s">"task_failed" }

# Each failure's retryability was journaled AT THE MOMENT it happened
# (from the error's own retryable? verdict) - no read-time table to
# drift out of date when the taxonomy renames
requeue, parked = dead.partition { |e| e[class="y">:retryable] }
attempts = state.events.select { |e| e[class="y">:event] == class="s">"task_failed" }.group_by { |e| e[class="y">:description] }

puts class="s">"DEAD LETTER OFFICE (#{RUNS.size} journaled runs)"
puts
puts class="s">"  REQUEUE MANIFEST (transient failures - safe to retry):"
requeue.each do |letter|
  puts format(class="s">"    %-16s %-42s %d failed attempt(s) on record",
    letter[class="y">:description], class="s">"#{letter[class="y">:error_type].split("::class="s">").last}: #{letter[class="y">:error]}",
    attempts[letter[class="y">:description]].size)
end
puts
puts class="s">"  PARKED (retrying will not help - a human must act):"
parked.each do |letter|
  puts format(class="s">"    %-16s %s", letter[class="y">:description], class="s">"#{letter[class="y">:error_type].split("::class="s">").last}: #{letter[class="y">:error]}")
end
puts
recovered = latest.values.select { |e| e[class="y">:event] == class="s">"task_succeeded" && attempts.key?(e[class="y">:description]) }
puts class="s">"  recovered on their own (failed once, succeeded later - NOT dead):"
recovered.each { |e| puts class="s">"    #{e[class="y">:description]}" }
puts
puts class="s">"  the office triages by MOST RECENT attempt: sync:crm's old timeout"
puts class="s">"  doesn't page anyone, and sync:tickets' early success doesn't"
puts class="s">"  excuse its newer 502. a dead-letter queue that forgets recoveries"
puts class="s">"  pages people for ghosts; one that forgets relapses buries real mail."
puts class="s">"  and each verdict above came from the journal itself - retryability"
puts class="s">"  was recorded when the error was fresh, not reconstructed later."