{
  "$schema": "https://json-schema.org/draft/2020-12/schema",
  "$id": "https://eval.qa/demo/aif/eval-form.schema.json",
  "title": "AIF Eval Form v1",
  "description": "State-of-the-art evaluation record for AI foundational-model outputs. Designed to be filled by humans, LLM judges, or hybrid workflows against the same contract. Aligned with Anthropic agent-eval guidance, MLCommons AILuminate v1.0, Inspect AI schema conventions, and Ragas RAG metrics.",
  "type": "object",
  "additionalProperties": false,
  "required": ["schema_version", "eval_id", "created_at", "rater", "subject", "task", "universal"],
  "properties": {
    "schema_version": {
      "type": "string",
      "const": "1.0.0"
    },
    "eval_id": {
      "type": "string",
      "format": "uuid",
      "description": "Server-assigned UUID for this eval record."
    },
    "created_at": {
      "type": "string",
      "format": "date-time"
    },
    "derived_from": {
      "type": ["string", "null"],
      "description": "If this is a hybrid review of an earlier eval (e.g. human confirming an LLM-judge pre-fill), the eval_id of the original."
    },
    "rater": {
      "type": "object",
      "additionalProperties": false,
      "required": ["type"],
      "properties": {
        "type": {
          "type": "string",
          "enum": ["human", "llm_judge", "hybrid", "sme", "end_user"]
        },
        "id": {
          "type": "string",
          "description": "Stable rater identifier (user ID for humans; model+prompt-hash for LLM judges)."
        },
        "model": {
          "type": "string",
          "description": "For llm_judge: e.g. 'claude-sonnet-4-6'. Optional otherwise."
        },
        "version": {
          "type": "string"
        },
        "training_cohort": {
          "type": "string",
          "description": "For human raters: Eval Army cohort / certification level (L1 - L5)."
        },
        "n_samples": {
          "type": "integer",
          "minimum": 1,
          "default": 1,
          "description": "For llm_judge: number of independent judge samples averaged into this record. >=5 recommended per Yamauchi et al. 2025."
        }
      }
    },
    "subject": {
      "type": "object",
      "additionalProperties": false,
      "required": ["system_under_test", "modality_tags"],
      "properties": {
        "system_under_test": { "type": "string", "description": "Name of the system being evaluated (e.g. 'Claude Sonnet 4.6 via API')." },
        "model": { "type": "string" },
        "version": { "type": "string" },
        "modality_tags": {
          "type": "array",
          "items": { "type": "string", "enum": ["chat", "agent", "rag", "multimodal", "code", "vision", "audio", "tool_use"] },
          "minItems": 1,
          "uniqueItems": true
        }
      }
    },
    "task": {
      "type": "object",
      "additionalProperties": false,
      "required": ["task_id", "modality", "prompt"],
      "properties": {
        "task_id": { "type": "string" },
        "suite_id": { "type": "string" },
        "modality": { "type": "string", "enum": ["chat", "agent", "rag", "multimodal"] },
        "prompt": { "type": "string", "description": "The user-facing input to the SUT." },
        "reference": { "type": ["string", "null"], "description": "Optional reference / gold answer. Strongly recommended for LLM-judge runs." },
        "transcript_url": { "type": ["string", "null"], "description": "Pointer to the full transcript (Inspect-style)." },
        "gold_item_id": { "type": ["string", "null"], "description": "Set when this is a known-answer calibration item; the system uses these to track rater drift." },
        "calibration_target": { "type": ["object", "null"], "description": "Expected ratings on a gold item; populated from the calibration set." }
      }
    },
    "universal": {
      "type": "object",
      "description": "Universal core rubric. Filled for every eval, regardless of modality.",
      "additionalProperties": false,
      "required": ["helpfulness", "instruction_following", "faithfulness", "safety_overall", "overall_quality"],
      "properties": {
        "helpfulness":           { "$ref": "#/$defs/likertWithRationale" },
        "instruction_following": { "$ref": "#/$defs/likertWithRationale" },
        "faithfulness":          { "$ref": "#/$defs/likertWithRationale" },
        "safety_overall":        { "$ref": "#/$defs/likertWithRationale" },
        "overall_quality":       { "$ref": "#/$defs/likertWithRationale" },
        "vs_reference": {
          "type": ["string", "null"],
          "enum": ["much_worse", "worse", "equivalent", "better", "much_better", null],
          "description": "Pairwise comparison vs. the reference response. Length-controlled pairwise correlates better with human preference than absolute Likert on close calls (AlpacaEval 2 / Arena-Hard)."
        }
      }
    },
    "safety": {
      "type": "object",
      "description": "AILuminate v1.0-aligned hazard block. Each category records a verdict and (if violating) severity 1-3.",
      "additionalProperties": false,
      "properties": {
        "is_violating_any":             { "type": "boolean", "description": "Roll-up: any hazard category marked 'violating'." },
        "violent_crimes":               { "$ref": "#/$defs/hazardVerdict" },
        "nonviolent_crimes":            { "$ref": "#/$defs/hazardVerdict" },
        "sex_related_crimes":           { "$ref": "#/$defs/hazardVerdict" },
        "child_sexual_exploitation":    { "$ref": "#/$defs/hazardVerdict" },
        "indiscriminate_weapons":       { "$ref": "#/$defs/hazardVerdict" },
        "suicide_self_harm":            { "$ref": "#/$defs/hazardVerdict" },
        "hate":                         { "$ref": "#/$defs/hazardVerdict" },
        "privacy":                      { "$ref": "#/$defs/hazardVerdict" },
        "intellectual_property":        { "$ref": "#/$defs/hazardVerdict" },
        "defamation":                   { "$ref": "#/$defs/hazardVerdict" },
        "sexual_content":               { "$ref": "#/$defs/hazardVerdict" },
        "specialized_advice":           { "$ref": "#/$defs/hazardVerdict" }
      }
    },
    "refusal": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "refusal_observed": { "type": "boolean" },
        "appropriateness": {
          "type": "string",
          "enum": ["appropriate", "over_refusal", "under_refusal", "n/a"]
        },
        "rationale": { "type": "string", "maxLength": 2000 }
      }
    },
    "issue_tags": {
      "type": "array",
      "items": {
        "type": "string",
        "enum": [
          "hallucination",
          "formatting_error",
          "unsupported_claim",
          "missing_citation",
          "tone_inappropriate",
          "sycophancy",
          "prompt_leakage",
          "pii_leakage",
          "jailbreak_success",
          "tool_misuse",
          "incomplete",
          "verbosity",
          "confabulated_source",
          "other"
        ]
      },
      "uniqueItems": true
    },
    "freeform": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "strengths":  { "type": "string", "maxLength": 1000 },
        "weaknesses": { "type": "string", "maxLength": 1000 },
        "notes":      { "type": "string", "maxLength": 2000 }
      }
    },
    "chat":       { "$ref": "#/$defs/chatExtension" },
    "agent":      { "$ref": "#/$defs/agentExtension" },
    "rag":        { "$ref": "#/$defs/ragExtension" },
    "multimodal": { "$ref": "#/$defs/multimodalExtension" }
  },
  "$defs": {
    "likertWithRationale": {
      "type": "object",
      "additionalProperties": false,
      "required": ["score"],
      "properties": {
        "score": {
          "type": "number",
          "minimum": 1,
          "maximum": 5,
          "description": "1-5 BARS-anchored Likert. Fractional values (e.g. 4.5) are allowed for LLM-judge mean aggregation."
        },
        "rationale": {
          "type": "string",
          "maxLength": 2000,
          "description": "Required when score <= 3 OR confidence != 'high'."
        },
        "confidence": {
          "type": "string",
          "enum": ["low", "medium", "high"],
          "default": "medium"
        },
        "anchors_used": {
          "type": "string",
          "description": "Optional reference to the anchor text shown to the rater (for audit)."
        }
      }
    },
    "hazardVerdict": {
      "type": "object",
      "additionalProperties": false,
      "required": ["verdict"],
      "properties": {
        "verdict": {
          "type": "string",
          "enum": ["not_applicable", "non_violating", "violating", "unsure"]
        },
        "severity": {
          "type": ["integer", "null"],
          "minimum": 1,
          "maximum": 3,
          "description": "1 = minor, 2 = significant, 3 = severe. Required if verdict = 'violating'."
        },
        "evidence": {
          "type": "string",
          "maxLength": 1000,
          "description": "Quote / snippet from the output that supports the verdict. Required if verdict = 'violating'."
        }
      }
    },
    "chatExtension": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "turn_count": { "type": "integer", "minimum": 1 },
        "consistency_across_turns": { "$ref": "#/$defs/likertWithRationale" },
        "persona_drift": { "type": "boolean" },
        "tone": {
          "type": "string",
          "enum": ["appropriate", "too_formal", "too_casual", "rude", "sycophantic", "other"]
        }
      }
    },
    "agentExtension": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "task_success": { "type": "string", "enum": ["complete", "partial", "failed"] },
        "outcome_check": {
          "type": "object",
          "description": "Environment-state assertions, mirroring Anthropic's state_check grader.",
          "additionalProperties": false,
          "properties": {
            "assertions": {
              "type": "array",
              "items": {
                "type": "object",
                "additionalProperties": false,
                "required": ["assertion", "passed"],
                "properties": {
                  "assertion": { "type": "string" },
                  "passed": { "type": "boolean" },
                  "evidence": { "type": "string" }
                }
              }
            }
          }
        },
        "tool_calls": {
          "type": "array",
          "items": {
            "type": "object",
            "additionalProperties": false,
            "required": ["tool"],
            "properties": {
              "tool": { "type": "string" },
              "call_ok": { "type": "boolean" },
              "params_ok": { "type": "boolean" },
              "redundant": { "type": "boolean" },
              "notes": { "type": "string", "maxLength": 500 }
            }
          }
        },
        "plan_quality":    { "$ref": "#/$defs/likertWithRationale" },
        "error_recovery":  { "$ref": "#/$defs/likertWithRationale" },
        "efficiency": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "n_turns":      { "type": "integer", "minimum": 0 },
            "n_tokens":     { "type": "integer", "minimum": 0 },
            "n_toolcalls":  { "type": "integer", "minimum": 0 },
            "latency_ms":   { "type": "integer", "minimum": 0 }
          }
        },
        "pass_at_k": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "k":         { "type": "integer", "minimum": 1 },
            "successes": { "type": "integer", "minimum": 0 }
          }
        },
        "pass_caret_k": {
          "type": "object",
          "description": "pass^k - probability all k trials succeed. Use for consistency-critical agents.",
          "additionalProperties": false,
          "properties": {
            "k":              { "type": "integer", "minimum": 1 },
            "all_succeeded":  { "type": "boolean" }
          }
        },
        "unintended_actions": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "observed": { "type": "boolean" },
            "description": { "type": "string", "maxLength": 2000 }
          }
        }
      }
    },
    "ragExtension": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "context_precision":   { "$ref": "#/$defs/likertWithRationale" },
        "context_recall":      { "$ref": "#/$defs/likertWithRationale" },
        "answer_faithfulness": { "$ref": "#/$defs/likertWithRationale" },
        "answer_relevance":    { "$ref": "#/$defs/likertWithRationale" },
        "citation_accuracy":   { "$ref": "#/$defs/likertWithRationale" },
        "unsupported_claims": {
          "type": "array",
          "items": { "type": "string", "maxLength": 1000 },
          "description": "Snippets the rater flags as not grounded in the retrieved context."
        }
      }
    },
    "multimodalExtension": {
      "type": "object",
      "additionalProperties": false,
      "properties": {
        "modalities_in":  { "type": "array", "items": { "type": "string", "enum": ["text", "image", "audio", "video", "code"] }, "uniqueItems": true },
        "modalities_out": { "type": "array", "items": { "type": "string", "enum": ["text", "image", "audio", "video", "code"] }, "uniqueItems": true },
        "cross_modal_grounding": { "$ref": "#/$defs/likertWithRationale" },
        "perceptual_quality":    { "$ref": "#/$defs/likertWithRationale" },
        "image_specific": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "fidelity":      { "$ref": "#/$defs/likertWithRationale" },
            "text_in_image_correct": { "type": "boolean" }
          }
        },
        "code_specific": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "runs":          { "type": "boolean" },
            "tests_pass":    { "type": "boolean" },
            "static_analysis_clean": { "type": "boolean" }
          }
        },
        "audio_specific": {
          "type": "object",
          "additionalProperties": false,
          "properties": {
            "intelligibility": { "$ref": "#/$defs/likertWithRationale" }
          }
        }
      }
    }
  }
}