Eval Scorers

Eval Scorers: the same eval set scored four ways - exact match, keyword containment, numeric tolerance, and a judge rubric. Exact scoring drowns one real failure in wording noise; the right scorer per field reports exactly the failure that matters. The harness shape never changes - only the scorer column does.

source on github

a real captured run

EVAL SCORERS: one eval set, four ways to say "good enough"

  case 1: "My package arrived broken and I want my money back"
    exact      on summary   FAIL  (0.00)
    contains   on summary   PASS  (1.00)
    tolerance  on priority  PASS  (1.00)
    judge      on summary   PASS  (1.00)

  case 2: "How do I change my email address?"
    exact      on summary   PASS  (1.00)
    contains   on summary   PASS  (1.00)
    tolerance  on priority  PASS  (1.00)

  case 3: "The app crashes every time I open settings and I lost work"
    exact      on summary   FAIL  (0.00)
    contains   on summary   FAIL  (0.00)
    tolerance  on priority  FAIL  (0.00)
    judge      on summary   FAIL  (0.00)

  scoreboard:
    exact      1/3 pass
    contains   2/3 pass
    tolerance  2/3 pass
    judge      1/2 pass

  exact flagged 2/3 cases, but most of that is wording noise.
  the field-appropriate scorers flagged only case 3 - the crash
  ticket - and that failure is REAL: the capability has no rule for
  crashes, so a data-loss ticket scores priority 0.3. same harness,
  same cases; the scorer column is what makes a failure mean something.

source

# frozen_string_literal: true

# Eval Scorers: the same eval set scored four ways - exact match,
# keyword containment, numeric tolerance, and a judge rubric. Exact
# scoring drowns one real failure in wording noise; the right scorer
# per field reports exactly the failure that matters. The harness
# shape never changes - only the scorer column does.
#
#   bundle exec ruby examples/eval_scorers.rb
#
# Runs offline; exits 1 because one capability blind spot is real.

require class="s">"bundler/setup"
require class="s">"agentic"

SEVERITY = {
  class="s">"damaged item" => 0.9,
  class="s">"refund requested" => 0.8,
  class="s">"account email update" => 0.2
}.freeze

spec = Agentic:class="y">:CapabilitySpecification.new(
  name: class="s">"summarize_ticket", description: class="s">"Summarize a support ticket", version: class="s">"1.0.0",
  inputs: {text: {type: class="s">"string", required: true}},
  outputs: {
    summary: {type: class="s">"string", required: true},
    priority: {type: class="s">"number", required: true, min: 0, max: 1}
  }
)
Agentic.register_capability(spec, Agentic:class="y">:CapabilityProvider.new(capability: spec, implementation: ->(i) {
  text = i[class="y">:text].downcase
  fragments = []
  fragments << class="s">"damaged item" if text.match?(/broken|damaged/)
  fragments << class="s">"refund requested" if text.match?(/refund|money back/)
  fragments << class="s">"account email update" if text.match?(/email/)
  # the blind spot: no rule for crashes - those tickets read as general inquiries
  summary = fragments.empty? ? class="s">"general inquiry" : class="s">"customer reports #{fragments.join(", class="s">")}"
  {summary: summary, priority: fragments.map { |f| SEVERITY[f] }.max || 0.3}
}))

# --- the scorer seam: (expected, actual) -> score in 0.0..1.0 ------------------
SCORERS = {
  exact: ->(expected, actual) { (expected == actual) ? 1.0 : 0.0 },
  contains: ->(keywords, actual) { keywords.count { |k| actual.to_s.downcase.include?(k) }.fdiv(keywords.size) },
  tolerance: ->(spec, actual) { ((spec[class="y">:value] - actual).abs <= spec[class="y">:within]) ? 1.0 : 0.0 },
  judge: ->(rubric, actual) { rubric.call(actual) }
}.freeze
PASS_AT = 0.99 # judge scorers may grade partially; everything else is 0-or-1

NAMES_A_PROBLEM = ->(summary) { (summary.include?(class="s">"general inquiry") ? 0.0 : 0.6) }
NAMES_AN_ACTION = ->(summary) { summary.match?(/request|update/) ? 0.4 : 0.0 }
RUBRIC = ->(summary) { NAMES_A_PROBLEM.call(summary) + NAMES_AN_ACTION.call(summary) }

CASES = [
  {ticket: class="s">"My package arrived broken and I want my money back",
   checks: [
     {field: class="y">:summary, scorer: class="y">:exact, expected: class="s">"Damaged item; refund requested"},
     {field: class="y">:summary, scorer: class="y">:contains, expected: %w[damaged refund]},
     {field: class="y">:priority, scorer: class="y">:tolerance, expected: {value: 0.9, within: 0.15}},
     {field: class="y">:summary, scorer: class="y">:judge, expected: RUBRIC}
   ]},
  {ticket: class="s">"How do I change my email address?",
   checks: [
     {field: class="y">:summary, scorer: class="y">:exact, expected: class="s">"customer reports account email update"},
     {field: class="y">:summary, scorer: class="y">:contains, expected: %w[email]},
     {field: class="y">:priority, scorer: class="y">:tolerance, expected: {value: 0.2, within: 0.1}}
   ]},
  {ticket: class="s">"The app crashes every time I open settings and I lost work",
   checks: [
     {field: class="y">:summary, scorer: class="y">:exact, expected: class="s">"Crash in settings; data loss"},
     {field: class="y">:summary, scorer: class="y">:contains, expected: %w[crash settings]},
     {field: class="y">:priority, scorer: class="y">:tolerance, expected: {value: 0.95, within: 0.1}},
     {field: class="y">:summary, scorer: class="y">:judge, expected: RUBRIC}
   ]}
].freeze

provider = Agentic:class="y">:AgentCapabilityRegistry.instance.get_provider(class="s">"summarize_ticket")
results = CASES.flat_map.with_index(1) do |kase, number|
  output = provider.execute(text: kase[class="y">:ticket])
  kase[class="y">:checks].map do |check|
    score = SCORERS.fetch(check[class="y">:scorer]).call(check[class="y">:expected], output[check[class="y">:field]])
    {case: number, scorer: check[class="y">:scorer], field: check[class="y">:field], score: score, pass: score >= PASS_AT}
  end
end

puts class="s">"EVAL SCORERS: one eval set, four ways to say \"good enough\class="s">""
puts
CASES.each_with_index do |kase, index|
  puts class="s">"  case #{index + 1}: #{kase[class="y">:ticket].inspect}"
  results.select { |r| r[class="y">:case] == index + 1 }.each do |r|
    puts format(class="s">"    %-10s on %-9s %s  (%.2f)", r[class="y">:scorer], r[class="y">:field], r[class="y">:pass] ? class="s">"PASS" : class="s">"FAIL", r[class="y">:score])
  end
  puts
end

by_scorer = results.group_by { |r| r[class="y">:scorer] }
puts class="s">"  scoreboard:"
by_scorer.each do |scorer, rows|
  puts format(class="s">"    %-10s %d/%d pass", scorer, rows.count { |r| r[class="y">:pass] }, rows.size)
end

exact_fails = by_scorer[class="y">:exact].count { |r| !r[class="y">:pass] }
real_fails = results.reject { |r| r[class="y">:scorer] == class="y">:exact }.reject { |r| r[class="y">:pass] }.map { |r| r[class="y">:case] }.uniq
puts
puts class="s">"  exact flagged #{exact_fails}/3 cases, but most of that is wording noise."
puts class="s">"  the field-appropriate scorers flagged only case #{real_fails.join(", class="s">")} - the crash"
puts class="s">"  ticket - and that failure is REAL: the capability has no rule for"
puts class="s">"  crashes, so a data-loss ticket scores priority 0.3. same harness,"
puts class="s">"  same cases; the scorer column is what makes a failure mean something."

exit(real_fails.any? ? 1 : 0)