Backoff Conformance

Backoff Conformance: every strategy x jitter combination, a thousand draws each through an injected seeded RNG, checked against the documented bounds. Retry timing is a contract like any other - and now that rng: is injectable, the contract is testable without stubbing a single method.

source on github

a real captured run

BACKOFF CONFORMANCE (seed 20260707, 1000 draws per combination)

  strategy      jitter   expected               observed               verdict
  constant      false    [ 1.000,  1.000]      [ 1.000,  1.000]      conforms
  constant      true     [ 0.750,  1.250]      [ 0.750,  1.249]      conforms
  constant      :full    [ 0.000,  1.000]      [ 0.001,  0.999]      conforms
  linear        false    [ 3.000,  3.000]      [ 3.000,  3.000]      conforms
  linear        true     [ 2.250,  3.750]      [ 2.251,  3.748]      conforms
  linear        :full    [ 0.000,  3.000]      [ 0.003,  2.996]      conforms
  exponential   false    [ 4.000,  4.000]      [ 4.000,  4.000]      conforms
  exponential   true     [ 3.000,  5.000]      [ 3.002,  4.998]      conforms
  exponential   :full    [ 0.000,  4.000]      [ 0.003,  3.995]      conforms

  every combination stayed inside its documented envelope AND
  explored at least 80% of it. the timing contract holds.

source

# frozen_string_literal: true

# Backoff Conformance: every strategy x jitter combination, a thousand
# draws each through an injected seeded RNG, checked against the
# documented bounds. Retry timing is a contract like any other - and
# now that rng: is injectable, the contract is testable without
# stubbing a single method.
#
#   bundle exec ruby examples/backoff_conformance.rb [seed]
#
# Runs offline and deterministically. Exit 1 on any bound violation.

require class="s">"bundler/setup"
require class="s">"agentic"

seed = (ARGV.first || 20260707).to_i
DRAWS = 1_000
BASE = 1.0
RETRY_COUNT = 3

# The documented bounds for each (strategy, jitter) combination
def expected_bounds(strategy, jitter, retry_count)
  nominal = case strategy
  when class="y">:constant then BASE
  when class="y">:linear then retry_count * BASE
  when class="y">:exponential then BASE * (2**(retry_count - 1))
  end

  case jitter
  when false then [nominal, nominal]
  when true then [nominal * 0.75, nominal * 1.25]
  when class="y">:full then [0.0, nominal]
  end
end

failures = []
puts class="s">"BACKOFF CONFORMANCE (seed #{seed}, #{DRAWS} draws per combination)"
puts
puts format(class="s">"  %-13s %-8s %-22s %-22s %s", class="s">"strategy", class="s">"jitter", class="s">"expected", class="s">"observed", class="s">"verdict")

%i[constant linear exponential].each do |strategy|
  [false, true, class="y">:full].each do |jitter|
    rng = Random.new(seed)
    orchestrator = Agentic:class="y">:PlanOrchestrator.new(retry_policy: {
      backoff_strategy: strategy,
      backoff_constant: BASE,
      backoff_base: BASE,
      backoff_jitter: jitter,
      rng: rng
    })

    observed = []
    orchestrator.define_singleton_method(class="y">:sleep) { |delay| observed << delay }

    task = Agentic:class="y">:Task.new(description: class="s">"t", agent_spec: {class="s">"name" => class="s">"t", class="s">"instructions" => class="s">"t"})
    task.retry_count = RETRY_COUNT
    DRAWS.times { orchestrator.apply_retry_backoff(task: task) }

    low, high = expected_bounds(strategy, jitter, RETRY_COUNT)
    epsilon = 1e-9
    in_bounds = observed.all? { |d| d >= low - epsilon && d <= high + epsilon }
    spans = jitter == false || (observed.max - observed.min) > (high - low) * 0.8

    verdict = if !in_bounds
      failures << [strategy, jitter, class="y">:bounds]
      class="s">"OUT OF BOUNDS (#{observed.min.round(3)}..#{observed.max.round(3)})"
    elsif !spans
      failures << [strategy, jitter, class="y">:coverage]
      class="s">"poor coverage"
    else
      class="s">"conforms"
    end

    puts format(class="s">"  %-13s %-8s [%6.3f, %6.3f]      [%6.3f, %6.3f]      %s",
      strategy, jitter.inspect, low, high, observed.min, observed.max, verdict)
  end
end

puts
if failures.empty?
  puts class="s">"  every combination stayed inside its documented envelope AND"
  puts class="s">"  explored at least 80% of it. the timing contract holds."
else
  puts class="s">"  CONTRACT VIOLATIONS: #{failures.inspect}"
  exit 1
end