Perf History

Perf History: last release's run left a journal; this release's run is compared against it. No synthetic baseline, no same-process rerun - the baseline is what production actually did, keyed by task description, straight from ExecutionJournal durations.

source on github

a real captured run

PERF HISTORY (baseline: agentic_perf_history.jsonl, noise floor 15ms)

  task                 last release   this one     delta
  resize:images               61ms        62ms       +2ms
  transcode:audio            110ms       110ms       +0ms
  generate:captions           80ms       140ms      +60ms  REGRESSED
  package:episode             50ms        23ms      -27ms  faster

  generate:captions regressed against the journal of the
  last shipped release. the baseline wasn't a benchmark rig - it
  was what actually ran, fsynced as it happened.

source

# frozen_string_literal: true

# Perf History: last release's run left a journal; this release's run
# is compared against it. No synthetic baseline, no same-process
# rerun - the baseline is what production actually did, keyed by task
# description, straight from ExecutionJournal durations.
#
#   bundle exec ruby examples/perf_history.rb
#
# Runs offline; "last release" is journaled first, then "this release"
# runs against its recorded history.

require class="s">"bundler/setup"
require class="s">"agentic"
require class="s">"tmpdir"

BASELINE_JOURNAL = File.join(Dir.tmpdir, class="s">"agentic_perf_history.jsonl")
File.delete(BASELINE_JOURNAL) if File.exist?(BASELINE_JOURNAL)

RELEASE_1 = {
  class="s">"resize:images" => 0.06,
  class="s">"transcode:audio" => 0.11,
  class="s">"generate:captions" => 0.08,
  class="s">"package:episode" => 0.05
}.freeze

# This release: captions got slower (new model), packaging got faster
RELEASE_2 = RELEASE_1.merge(
  class="s">"generate:captions" => 0.14,
  class="s">"package:episode" => 0.02
).freeze

def run_release(work, journal: nil, collect: nil)
  hooks = {}
  hooks = journal.lifecycle_hooks if journal
  if collect
    hooks = (journal ? journal.lifecycle_hooks : {}).merge(
      after_task_success: ->(task_id:, task:, result:, duration:) { collect[task.description] = duration }
    )
  end

  orchestrator = Agentic:class="y">:PlanOrchestrator.new(concurrency_limit: 4, lifecycle_hooks: hooks)
  previous = nil
  work.each do |name, latency|
    task = Agentic:class="y">:Task.new(description: name,
      agent_spec: {class="s">"name" => name, class="s">"instructions" => class="s">"work"}, payload: latency)
    orchestrator.add_task(task, previous ? [previous] : [], agent: ->(t) { sleep(t.payload) || class="y">:ok })
    previous = task
  end
  orchestrator.execute_plan
end

# --- release 1 ships; its journal IS the baseline ----------------------------
run_release(RELEASE_1, journal: Agentic:class="y">:ExecutionJournal.new(path: BASELINE_JOURNAL))
baseline = Agentic:class="y">:ExecutionJournal.replay(path: BASELINE_JOURNAL).durations

# --- release 2 runs; the journal from last time judges it --------------------
current = {}
run_release(RELEASE_2, collect: current)

NOISE_MS = 15

puts class="s">"PERF HISTORY (baseline: #{BASELINE_JOURNAL.split("/class="s">").last}, noise floor #{NOISE_MS}ms)"
puts
puts format(class="s">"  %-20s %10s %10s %9s", class="s">"task", class="s">"last release", class="s">"this one", class="s">"delta")
regressions = []
current.each do |name, duration|
  recorded = baseline[name]
  next unless recorded

  delta_ms = (duration - recorded) * 1000
  verdict =
    if delta_ms.abs < NOISE_MS then class="s">""
    elsif delta_ms.negative? then class="s">"faster"
    else
      regressions << name
      class="s">"REGRESSED"
    end
  puts format(class="s">"  %-20s %9.0fms %9.0fms %+8.0fms  %s",
    name, recorded * 1000, duration * 1000, delta_ms, verdict)
end

puts
if regressions.any?
  puts class="s">"  #{regressions.join(", class="s">")} regressed against the journal of the"
  puts class="s">"  last shipped release. the baseline wasn't a benchmark rig - it"
  puts class="s">"  was what actually ran, fsynced as it happened."
  exit 1
else
  puts class="s">"  no regressions against recorded history. ship."
end