{
  "schemaVersion": 1,
  "title": "Four-fact recall scoring benchmark",
  "published": "2026-09-16",
  "author": "Awesome Patel",
  "question": "How do literal match, normalized match, and a four-fact checklist classify fixed variants of one fictional reference answer?",
  "referenceAnswer": "The copper token goes in the west drawer before the museum opens.",
  "protocol": {
    "sessionType": "Deterministic fixture benchmark; no human participants",
    "literalMatch": "Attempt must equal the reference answer character for character.",
    "normalizedMatch": "Lowercase, remove ASCII punctuation, trim, and collapse whitespace; then compare with the normalized reference.",
    "factChecklist": [
      { "fact": "material", "requiredTerms": ["copper"] },
      { "fact": "object", "requiredTerms": ["token"] },
      { "fact": "location", "requiredTerms": ["west", "drawer"] },
      { "fact": "timing", "requiredTerms": ["before", "museum", "opens"] }
    ],
    "fullChecklistRule": "All four facts must be present. Matching is case-insensitive after ASCII punctuation removal and whitespace collapse."
  },
  "summary": {
    "variantCount": 12,
    "completeFactFixtures": 8,
    "oneFactChangedOrMissingFixtures": 4,
    "literalFullMatches": 1,
    "normalizedFullMatches": 4,
    "checklistFullMatches": 8
  },
  "rows": [
    { "id": "R01", "condition": "reference copied exactly", "attempt": "The copper token goes in the west drawer before the museum opens.", "fixtureFactsComplete": true, "literalMatch": true, "normalizedMatch": true, "factsPresent": 4, "checklistFullMatch": true },
    { "id": "R02", "condition": "uppercase", "attempt": "THE COPPER TOKEN GOES IN THE WEST DRAWER BEFORE THE MUSEUM OPENS.", "fixtureFactsComplete": true, "literalMatch": false, "normalizedMatch": true, "factsPresent": 4, "checklistFullMatch": true },
    { "id": "R03", "condition": "punctuation removed", "attempt": "The copper token goes in the west drawer before the museum opens", "fixtureFactsComplete": true, "literalMatch": false, "normalizedMatch": true, "factsPresent": 4, "checklistFullMatch": true },
    { "id": "R04", "condition": "spacing changed", "attempt": "  The copper token   goes in the west drawer before the museum opens.  ", "fixtureFactsComplete": true, "literalMatch": false, "normalizedMatch": true, "factsPresent": 4, "checklistFullMatch": true },
    { "id": "R05", "condition": "clauses reordered", "attempt": "Before the museum opens, the copper token goes in the west drawer.", "fixtureFactsComplete": true, "literalMatch": false, "normalizedMatch": false, "factsPresent": 4, "checklistFullMatch": true },
    { "id": "R06", "condition": "telegraphic wording", "attempt": "Copper token: west drawer before museum opens.", "fixtureFactsComplete": true, "literalMatch": false, "normalizedMatch": false, "factsPresent": 4, "checklistFullMatch": true },
    { "id": "R07", "condition": "leading context added", "attempt": "Please remember that the copper token goes in the west drawer before the museum opens.", "fixtureFactsComplete": true, "literalMatch": false, "normalizedMatch": false, "factsPresent": 4, "checklistFullMatch": true },
    { "id": "R08", "condition": "articles and verb removed", "attempt": "Copper token in west drawer before museum opens.", "fixtureFactsComplete": true, "literalMatch": false, "normalizedMatch": false, "factsPresent": 4, "checklistFullMatch": true },
    { "id": "R09", "condition": "material omitted", "attempt": "The token goes in the west drawer before the museum opens.", "fixtureFactsComplete": false, "literalMatch": false, "normalizedMatch": false, "factsPresent": 3, "checklistFullMatch": false },
    { "id": "R10", "condition": "location changed", "attempt": "The copper token goes in the east drawer before the museum opens.", "fixtureFactsComplete": false, "literalMatch": false, "normalizedMatch": false, "factsPresent": 3, "checklistFullMatch": false },
    { "id": "R11", "condition": "timing changed", "attempt": "The copper token goes in the west drawer after the museum opens.", "fixtureFactsComplete": false, "literalMatch": false, "normalizedMatch": false, "factsPresent": 3, "checklistFullMatch": false },
    { "id": "R12", "condition": "material changed", "attempt": "The silver token goes in the west drawer before the museum opens.", "fixtureFactsComplete": false, "literalMatch": false, "normalizedMatch": false, "factsPresent": 3, "checklistFullMatch": false }
  ],
  "limitations": [
    "The attempts are authored fixtures, not responses from people.",
    "The checklist recognizes only the published terms and does not understand meaning, synonyms, negation, or factual truth.",
    "The benchmark measures classification behavior for one fictional answer; it does not measure recall, retention, ability, or learning gains."
  ],
  "versionHistory": [
    { "version": "1.0.0", "date": "2026-09-16", "change": "Initial protocol, 12 fixtures, three scoring rules, and raw exports." }
  ]
}