apiVersion: nines.dev/v1alpha1
kind: SLOExtensionBundle
metadata:
  name: example-agent
  version: 1.0.0
  publisher: example.invalid
  description: Example of a community-owned agent evaluation measurement. Instrumentation is illustrative.
spec:
  engine: nines.dev/measurement-v1
  policies:
    - name: evaluation-defaults
      description: Illustrative weekly measurement and burn policies; review before making a paging commitment.
      target: 0.99
      windowSeconds: 604800
      quality:
        minEvents: 100
        maxSampleAgeSeconds: 300
      policies:
        - name: fast-burn
          severity: critical
          shortWindow: 5m
          longWindow: 1h
          burnRate: 14.4
          forSeconds: 120
          minCoverage: 1
          minEvents: 100
        - name: sustained-burn
          severity: warning
          shortWindow: 30m
          longWindow: 6h
          burnRate: 6
          forSeconds: 300
          minCoverage: 1
          minEvents: 100
  evaluations:
    - name: outcome-rubric
      version: 1.0.0
      rubric: support-v1 checks the declared task outcome using independently reviewed assertions. A proposing agent cannot change the rubric or approve its own result.
      sampling: Declare the selected task cohort and evaluator version in bounded labels. Track selected versus completed evaluations separately.
      populationLabels:
        rubric: support-v1
        evaluator: rules-v1
      unknownVerdict: not_good
      missingVerdict: unknown
  recipes:
    - name: evaluation-pass
      category: agent
      sli: evaluation-quality
      title: Agent outcome evaluation
      description: Count passing evaluation verdicts within an explicitly selected cohort.
      eligibleEvents: Every terminal verdict in the selected task evaluation cohort, including fail and unknown.
      goodEvents: Verdict pass under the pinned rubric and evaluator version.
      requirements:
        - Emit one pass, fail or unknown verdict per selected logical task. Measure selected-but-missing verdicts separately.
        - Keep prompt, output, user and task identifiers out of metric labels.
      policy: evaluation-defaults
      evaluation: outcome-rubric
      inputs:
        metric:
          field: metric
          type: string
          description: Observed terminal evaluation counter name.
          required: true
        service:
          field: service
          type: string
          description: Service that owns the agent task outcome.
          required: true
        population:
          field: labels
          type: labels
          description: Exact cohort filters, including the reviewed rubric and evaluator version.
          required: true
        target:
          field: target
          type: number
          description: Reviewed event-success target fraction.
          required: false
          default: 0.99
          minimum: 0.95
          maximum: 0.9999
      manifest:
        apiVersion: nines.dev/v1alpha1
        kind: ServiceLevelObjective
        metadata:
          name: agent-evaluation
          project: production
        spec:
          displayName: Agent outcome evaluation
          service: agent
          source: primary
          lifecycle: experimental
          target: 0.99
          windowSeconds: 604800
          stepSeconds: 60
          measurement:
            type: availability
            metric: agent_evaluations_total
            labels:
              service: agent
            successLabels:
              verdict: pass
            countFrom: counter
            scrapeIntervalSeconds: 15
            observationPoint: Every terminal verdict in the selected evaluation cohort. Fail and unknown verdicts remain in the denominator; missing evaluations require separate coverage evidence.
  tests:
    - name: explicit-cohort
      recipe: evaluation-pass
      inputs:
        metric: support_evaluations_total
        service: support-agent
        population:
          service: support-agent
          environment: production
          rubric: support-v1
          evaluator: rules-v1
      goodSelector: support_evaluations_total{environment="production",evaluator="rules-v1",rubric="support-v1",service="support-agent",verdict="pass"}
      totalSelector: support_evaluations_total{environment="production",evaluator="rules-v1",rubric="support-v1",service="support-agent"}
    - name: missing-metric
      recipe: evaluation-pass
      inputs:
        service: support-agent
        population:
          service: support-agent
      expectError: true
    - name: invalid-target
      recipe: evaluation-pass
      inputs:
        metric: support_evaluations_total
        service: support-agent
        population:
          service: support-agent
        target: 0.5
      expectError: true