Capability Evals

Capability Evals: golden test cases run against registered capabilities, scored, and gated. When you swap a lambda for an LLM (or one model for another), the eval suite is what tells you whether quality moved - contracts check shape, evals check SUBSTANCE.

source on github

a real captured run

CAPABILITY EVALS (pass threshold 90%)

  classify_sentiment   4/4 (100%)
  extract_amount       1/3 ( 33%) BELOW THRESHOLD
    FAIL "you owe $12.50 by friday"
         expected {:cents=>1250}, got {:cents=>1200}
    FAIL "refund of $0.99 issued"
         expected {:cents=>99}, got {:cents=>0}

  suite score: 67%
  the gate holds: extract_amount drops cents - a shape-valid,
  substance-wrong answer that no contract could catch. contracts
  check types; evals check truth. you need both.

source

# frozen_string_literal: true

# Capability Evals: golden test cases run against registered
# capabilities, scored, and gated. When you swap a lambda for an LLM
# (or one model for another), the eval suite is what tells you whether
# quality moved - contracts check shape, evals check SUBSTANCE.
#
#   bundle exec ruby examples/capability_evals.rb
#
# Runs offline; one capability has a bug the evals catch.

require class="s">"bundler/setup"
require class="s">"agentic"

def capability(name, inputs:, outputs:, &impl)
  spec = Agentic:class="y">:CapabilitySpecification.new(
    name: name, description: name, version: class="s">"1.0.0", inputs: inputs, outputs: outputs
  )
  Agentic.register_capability(
    spec, Agentic:class="y">:CapabilityProvider.new(capability: spec, implementation: impl)
  )
end

capability(class="s">"classify_sentiment",
  inputs: {text: {type: class="s">"string", required: true}},
  outputs: {label: {type: class="s">"string", required: true, enum: %w[positive negative neutral]}}) do |i|
  text = i[class="y">:text].downcase
  label = if text.match?(/love|great|excellent|happy/)
    class="s">"positive"
  elsif text.match?(/hate|terrible|awful|angry/)
    class="s">"negative"
  else
    class="s">"neutral"
  end
  {label: label}
end

capability(class="s">"extract_amount",
  inputs: {text: {type: class="s">"string", required: true}},
  outputs: {cents: {type: class="s">"number", required: true}}) do |i|
  # The bug: parses dollars but drops the cents
  dollars = i[class="y">:text][/\$(\d+)/, 1].to_i
  {cents: dollars * 100}
end

EVALS = {
  class="s">"classify_sentiment" => [
    {input: {text: class="s">"I love this gem"}, expect: {label: class="s">"positive"}},
    {input: {text: class="s">"This is terrible"}, expect: {label: class="s">"negative"}},
    {input: {text: class="s">"It runs on Ruby 3.3"}, expect: {label: class="s">"neutral"}},
    {input: {text: class="s">"absolutely EXCELLENT work"}, expect: {label: class="s">"positive"}}
  ],
  class="s">"extract_amount" => [
    {input: {text: class="s">"invoice total $45"}, expect: {cents: 4500}},
    {input: {text: class="s">"you owe $12.50 by friday"}, expect: {cents: 1250}},
    {input: {text: class="s">"refund of $0.99 issued"}, expect: {cents: 99}}
  ]
}.freeze

registry = Agentic:class="y">:AgentCapabilityRegistry.instance
THRESHOLD = 0.9

puts class="s">"CAPABILITY EVALS (pass threshold #{(THRESHOLD * 100).round}%)"
puts

overall = []
EVALS.each do |name, cases|
  provider = registry.get_provider(name)
  results = cases.map do |eval_case|
    actual = provider.execute(eval_case[class="y">:input])
    passed = eval_case[class="y">:expect].all? { |key, expected| actual[key] == expected }
    [eval_case, actual, passed]
  end

  passed = results.count { |_, _, ok| ok }
  rate = passed.to_f / cases.size
  overall << rate
  puts format(class="s">"  %-20s %d/%d (%3.0f%%) %s", name, passed, cases.size, rate * 100,
    (rate >= THRESHOLD) ? class="s">"" : class="s">"BELOW THRESHOLD")

  results.reject { |_, _, ok| ok }.each do |eval_case, actual, _|
    puts class="s">"    FAIL #{eval_case[class="y">:input][class="y">:text].inspect}"
    puts class="s">"         expected #{eval_case[class="y">:expect].inspect}, got #{actual.inspect}"
  end
end

puts
score = overall.sum / overall.size
puts format(class="s">"  suite score: %.0f%%", score * 100)
if score < THRESHOLD
  puts class="s">"  the gate holds: extract_amount drops cents - a shape-valid,"
  puts class="s">"  substance-wrong answer that no contract could catch. contracts"
  puts class="s">"  check types; evals check truth. you need both."
  exit 1
end