The Perf Diff

The Perf Diff: run the plan before and after a change, diff per-task durations, and flag regressions - with the one qualifier that decides whether anyone should care: is the regressed task ON the critical path? Off-path regressions are trivia; on-path regressions are the release note nobody wrote.

source on github

a real captured run

PERF DIFF (noise floor 15ms)

  task                  before     after     delta
  fetch:prices           101ms     160ms      +59ms  SLOWER + ON CRITICAL PATH
  fetch:inventory         60ms      60ms       +0ms
  reprice:catalog         80ms      30ms      -50ms  faster
  index:search            50ms      50ms       +0ms
  warm:cache              40ms      41ms       +1ms

  wall clock: 231ms -> 241ms (+9ms)

  VERDICT: don't ship. fetch:prices regressed
  on the critical path - the repricing win is real and the users
  will never feel it, because the wall clock got worse anyway.

source

# frozen_string_literal: true

# The Perf Diff: run the plan before and after a change, diff per-task
# durations, and flag regressions - with the one qualifier that decides
# whether anyone should care: is the regressed task ON the critical
# path? Off-path regressions are trivia; on-path regressions are the
# release note nobody wrote.
#
#   bundle exec ruby examples/perf_diff.rb
#
# Runs offline; the "change" speeds one task up and quietly breaks
# another, as changes do.

require class="s">"bundler/setup"
require class="s">"agentic"

BASELINE = {
  class="s">"fetch:prices" => {sleep: 0.10, deps: []},
  class="s">"fetch:inventory" => {sleep: 0.06, deps: []},
  class="s">"reprice:catalog" => {sleep: 0.08, deps: [class="s">"fetch:prices", class="s">"fetch:inventory"]},
  class="s">"index:search" => {sleep: 0.05, deps: [class="s">"reprice:catalog"]},
  class="s">"warm:cache" => {sleep: 0.04, deps: [class="s">"reprice:catalog"]}
}.freeze

# The optimization sped up repricing... and the same PR made
# fetch:prices slower. Ship it? Let's find out.
AFTER_THE_PR = BASELINE.merge(
  class="s">"reprice:catalog" => BASELINE[class="s">"reprice:catalog"].merge(sleep: 0.03),
  class="s">"fetch:prices" => BASELINE[class="s">"fetch:prices"].merge(sleep: 0.16)
).freeze

def measure(work)
  durations = {}
  orchestrator = Agentic:class="y">:PlanOrchestrator.new(
    concurrency_limit: 4,
    lifecycle_hooks: {
      after_task_success: ->(task_id:, task:, result:, duration:) { durations[task.description] = duration }
    }
  )
  tasks = {}
  work.each do |name, spec|
    tasks[name] = Agentic:class="y">:Task.new(description: name,
      agent_spec: {class="s">"name" => name, class="s">"instructions" => class="s">"work"}, payload: spec[class="y">:sleep])
    orchestrator.add_task(tasks[name], spec[class="y">:deps].map { |d| tasks.fetch(d) },
      agent: ->(t) { sleep(t.payload) || class="y">:ok })
  end
  result = orchestrator.execute_plan
  [durations, result.execution_time, orchestrator.graph]
end

def critical_path_names(graph, durations)
  names = graph[class="y">:tasks].transform_values(&class="y">:description)
  memo = {}
  walk = lambda do |task_id|
    memo[task_id] ||= begin
      best = graph[class="y">:dependencies][task_id].map { |d| walk.call(d) }.max_by { |p| p[class="y">:cost] } || {cost: 0.0, path: []}
      {cost: best[class="y">:cost] + durations[names[task_id]], path: best[class="y">:path] + [names[task_id]]}
    end
  end
  graph[class="y">:order].map { |id| walk.call(id) }.max_by { |p| p[class="y">:cost] }[class="y">:path]
end

NOISE_MS = 15

before, wall_before, = measure(BASELINE)
after, wall_after, graph = measure(AFTER_THE_PR)
path_after = critical_path_names(graph, after)

puts class="s">"PERF DIFF (noise floor #{NOISE_MS}ms)"
puts
puts format(class="s">"  %-18s %9s %9s %9s  %s", class="s">"task", class="s">"before", class="s">"after", class="s">"delta", class="s">"")
regressions = []
BASELINE.each_key do |name|
  delta_ms = (after[name] - before[name]) * 1000
  marker =
    if delta_ms.abs < NOISE_MS then class="s">""
    elsif delta_ms.negative? then class="s">"faster"
    else
      on_path = path_after.include?(name)
      regressions << {name: name, on_path: on_path}
      on_path ? class="s">"SLOWER + ON CRITICAL PATH" : class="s">"slower (off-path)"
    end
  puts format(class="s">"  %-18s %7dms %7dms %+8dms  %s",
    name, before[name] * 1000, after[name] * 1000, delta_ms, marker)
end

puts
puts format(class="s">"  wall clock: %dms -> %dms (%+dms)",
  wall_before * 1000, wall_after * 1000, (wall_after - wall_before) * 1000)
puts
blocking = regressions.select { |r| r[class="y">:on_path] }
if blocking.any?
  puts class="s">"  VERDICT: don't ship. #{blocking.map { |r| r[class="y">:name] }.join(", class="s">")} regressed"
  puts class="s">"  on the critical path - the repricing win is real and the users"
  puts class="s">"  will never feel it, because the wall clock got worse anyway."
  exit 1
else
  puts class="s">"  VERDICT: ship it."
end