{
  "format": "tacitus-framework-debugging-comparison/2",
  "scope": "Developer-visible debugging after reading original outputs, not held-out or matched-budget evaluation. Original raw results and rubric remain unchanged.",
  "guard_version": "known-failures-v2; narrow heuristics added after observing errors, not an independent quality rubric",
  "original_responses": 36,
  "refined_responses": 12,
  "all_same_source_receipts": true,
  "models": [
    "@cf/meta/llama-3.1-8b-instruct-fp8",
    "@cf/ibm-granite/granite-4.0-h-micro"
  ],
  "conclusion": "Results remain mixed and include material regressions. Additional prose guardrails did not establish reliable factual correctness or efficiency. Typed outputs, source/value validation and domain review are still required.",
  "pairs": [
    {
      "task_id": "gaza-reporting-stages",
      "model": "@cf/meta/llama-3.1-8b-instruct-fp8",
      "original_request_id": "d5f732780cd0b1222f4d",
      "refined_request_id": "8138ceb7bea4071f3f52",
      "source_sha256": "f82a8ccf366fc77829dd2681346c74639aeff5c998c377881e5f2455433ad5a0",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 6,
        "lexical_total": 6,
        "prompt_tokens": 797,
        "output_tokens": 231,
        "latency_ms": 10850,
        "known_failure_guards": [
          {
            "id": "truck-window-retained",
            "description": "Retain 26 October–1 November for the truck figures.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "report-label-and-page-date",
            "description": "Retain report label 5 November and page date 6 November.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "stage-numbers-attached",
            "description": "Keep manifested466 and offloaded397 as distinct stages.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 6,
        "lexical_total": 6,
        "prompt_tokens": 910,
        "output_tokens": 175,
        "latency_ms": 10599,
        "known_failure_guards": [
          {
            "id": "truck-window-retained",
            "description": "Retain 26 October–1 November for the truck figures.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "report-label-and-page-date",
            "description": "Retain report label 5 November and page date 6 November.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "stage-numbers-attached",
            "description": "Keep manifested466 and offloaded397 as distinct stages.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "syria-ceasefire-scope",
      "model": "@cf/meta/llama-3.1-8b-instruct-fp8",
      "original_request_id": "677fd21e67c355a70b1d",
      "refined_request_id": "88339c63eaca8989f147",
      "source_sha256": "b0a9c3d331700dc25562153214efdcd123f52de89feb47dd68b8784b6d04dc74",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 5,
        "lexical_total": 5,
        "prompt_tokens": 947,
        "output_tokens": 236,
        "latency_ms": 15546,
        "known_failure_guards": [
          {
            "id": "issg-support-not-sole-implementer",
            "description": "Do not promote ISSG support/assistance into a sole implementation mandate.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 4,
        "lexical_total": 5,
        "prompt_tokens": 1057,
        "output_tokens": 240,
        "latency_ms": 12332,
        "known_failure_guards": [
          {
            "id": "issg-support-not-sole-implementer",
            "description": "Do not promote ISSG support/assistance into a sole implementation mandate.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "fria-dpia-evidence-reuse",
      "model": "@cf/meta/llama-3.1-8b-instruct-fp8",
      "original_request_id": "1329ecab52535082116d",
      "refined_request_id": "33ddbc5de2b84d3ee245",
      "source_sha256": "e8245c1e8ae042c74b56650d7b82db64eb44505e5c3af495af28b292b5fe6741",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 4,
        "lexical_total": 5,
        "prompt_tokens": 2038,
        "output_tokens": 375,
        "latency_ms": 23350,
        "known_failure_guards": [
          {
            "id": "dpo-review-role",
            "description": "Keep DPO advice distinct from the AI Office template role.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "ai-office-not-reviewer",
            "description": "Do not assign deployment assessment review to the AI Office.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-fria-replacement",
            "description": "Do not present replacing FRIA with DPIA as the recommended action.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-reverse-replacement-heading",
            "description": "Do not recommend replacing DPIA with FRIA either.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "conditional-complement",
            "description": "Retain complement plus a condition instead of a universal replacement rule.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 5,
        "lexical_total": 5,
        "prompt_tokens": 2220,
        "output_tokens": 424,
        "latency_ms": 21792,
        "known_failure_guards": [
          {
            "id": "dpo-review-role",
            "description": "Keep DPO advice distinct from the AI Office template role.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "ai-office-not-reviewer",
            "description": "Do not assign deployment assessment review to the AI Office.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-fria-replacement",
            "description": "Do not present replacing FRIA with DPIA as the recommended action.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-reverse-replacement-heading",
            "description": "Do not recommend replacing DPIA with FRIA either.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "conditional-complement",
            "description": "Retain complement plus a condition instead of a universal replacement rule.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "ai-law-version-dates",
      "model": "@cf/meta/llama-3.1-8b-instruct-fp8",
      "original_request_id": "8a19f6bb8e8cdd68b78d",
      "refined_request_id": "cc40c37832c924c9d72c",
      "source_sha256": "6e29b4b562a0ed22cd5601c7fead96d122253bcb1d3855f14e4b1eeb6d709aa1",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 6,
        "lexical_total": 6,
        "prompt_tokens": 664,
        "output_tokens": 140,
        "latency_ms": 7399,
        "known_failure_guards": [
          {
            "id": "entry-force-not-application",
            "description": "Do not call 2 August 2026 the entry-into-force date.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-unsupported-absolute-entry-date",
            "description": "Do not invent an absolute entry-into-force date when publication timing is not supplied.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "entry-force-publication-rule",
            "description": "State the separate twentieth-day-after-publication entry rule.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 6,
        "lexical_total": 6,
        "prompt_tokens": 805,
        "output_tokens": 127,
        "latency_ms": 6841,
        "known_failure_guards": [
          {
            "id": "entry-force-not-application",
            "description": "Do not call 2 August 2026 the entry-into-force date.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-unsupported-absolute-entry-date",
            "description": "Do not invent an absolute entry-into-force date when publication timing is not supplied.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "entry-force-publication-rule",
            "description": "State the separate twentieth-day-after-publication entry rule.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "georgia-sample-exposure",
      "model": "@cf/meta/llama-3.1-8b-instruct-fp8",
      "original_request_id": "0e759a55803fafda419f",
      "refined_request_id": "5d540d07d1604e72ff45",
      "source_sha256": "f0b2ed2c4a2110554108b9162678e5fa9be6f3724ac1f7fc79a057738a617611",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 4,
        "lexical_total": 5,
        "prompt_tokens": 617,
        "output_tokens": 168,
        "latency_ms": 10519,
        "known_failure_guards": [
          {
            "id": "synthetic-label-retained",
            "description": "Keep airtime values explicitly synthetic.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 4,
        "lexical_total": 5,
        "prompt_tokens": 720,
        "output_tokens": 217,
        "latency_ms": 10107,
        "known_failure_guards": [
          {
            "id": "synthetic-label-retained",
            "description": "Keep airtime values explicitly synthetic.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "ireland-dialogue-framing",
      "model": "@cf/meta/llama-3.1-8b-instruct-fp8",
      "original_request_id": "2fe2001ecdc23d334c41",
      "refined_request_id": "97b347de0d293c0b19f6",
      "source_sha256": "1d223d443aeb85900d1ea0a9353e7bc4b5a92759d9e36ca2bafc9d31eb03873b",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 7,
        "lexical_total": 7,
        "prompt_tokens": 619,
        "output_tokens": 305,
        "latency_ms": 15840,
        "known_failure_guards": [
          {
            "id": "synthetic-label-retained",
            "description": "Keep the invented/synthetic-statement label.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-invented-critics",
            "description": "Do not invent critics or attributed opposing speakers.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 6,
        "lexical_total": 7,
        "prompt_tokens": 725,
        "output_tokens": 185,
        "latency_ms": 8700,
        "known_failure_guards": [
          {
            "id": "synthetic-label-retained",
            "description": "Keep the invented/synthetic-statement label.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-invented-critics",
            "description": "Do not invent critics or attributed opposing speakers.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "gaza-reporting-stages",
      "model": "@cf/ibm-granite/granite-4.0-h-micro",
      "original_request_id": "0e147b5602512a97d773",
      "refined_request_id": "6e54df49bb356a7960d6",
      "source_sha256": "f82a8ccf366fc77829dd2681346c74639aeff5c998c377881e5f2455433ad5a0",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 4,
        "lexical_total": 6,
        "prompt_tokens": 794,
        "output_tokens": 105,
        "latency_ms": 3317,
        "known_failure_guards": [
          {
            "id": "truck-window-retained",
            "description": "Retain 26 October–1 November for the truck figures.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "report-label-and-page-date",
            "description": "Retain report label 5 November and page date 6 November.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "stage-numbers-attached",
            "description": "Keep manifested466 and offloaded397 as distinct stages.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 3,
        "lexical_total": 6,
        "prompt_tokens": 907,
        "output_tokens": 90,
        "latency_ms": 3551,
        "known_failure_guards": [
          {
            "id": "truck-window-retained",
            "description": "Retain 26 October–1 November for the truck figures.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "report-label-and-page-date",
            "description": "Retain report label 5 November and page date 6 November.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "stage-numbers-attached",
            "description": "Keep manifested466 and offloaded397 as distinct stages.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "syria-ceasefire-scope",
      "model": "@cf/ibm-granite/granite-4.0-h-micro",
      "original_request_id": "579e0163d19293206f9a",
      "refined_request_id": "76c015fa24dd5cb28044",
      "source_sha256": "b0a9c3d331700dc25562153214efdcd123f52de89feb47dd68b8784b6d04dc74",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 5,
        "lexical_total": 5,
        "prompt_tokens": 944,
        "output_tokens": 228,
        "latency_ms": 7760,
        "known_failure_guards": [
          {
            "id": "issg-support-not-sole-implementer",
            "description": "Do not promote ISSG support/assistance into a sole implementation mandate.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 4,
        "lexical_total": 5,
        "prompt_tokens": 1054,
        "output_tokens": 230,
        "latency_ms": 6601,
        "known_failure_guards": [
          {
            "id": "issg-support-not-sole-implementer",
            "description": "Do not promote ISSG support/assistance into a sole implementation mandate.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "fria-dpia-evidence-reuse",
      "model": "@cf/ibm-granite/granite-4.0-h-micro",
      "original_request_id": "abbdd10ced286d408972",
      "refined_request_id": "d9eddf4c9011adf5fd27",
      "source_sha256": "e8245c1e8ae042c74b56650d7b82db64eb44505e5c3af495af28b292b5fe6741",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 2,
        "lexical_total": 5,
        "prompt_tokens": 2035,
        "output_tokens": 265,
        "latency_ms": 8072,
        "known_failure_guards": [
          {
            "id": "dpo-review-role",
            "description": "Keep DPO advice distinct from the AI Office template role.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "ai-office-not-reviewer",
            "description": "Do not assign deployment assessment review to the AI Office.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-fria-replacement",
            "description": "Do not present replacing FRIA with DPIA as the recommended action.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-reverse-replacement-heading",
            "description": "Do not recommend replacing DPIA with FRIA either.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "conditional-complement",
            "description": "Retain complement plus a condition instead of a universal replacement rule.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 4,
        "lexical_total": 5,
        "prompt_tokens": 2217,
        "output_tokens": 286,
        "latency_ms": 8655,
        "known_failure_guards": [
          {
            "id": "dpo-review-role",
            "description": "Keep DPO advice distinct from the AI Office template role.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "ai-office-not-reviewer",
            "description": "Do not assign deployment assessment review to the AI Office.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-fria-replacement",
            "description": "Do not present replacing FRIA with DPIA as the recommended action.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-reverse-replacement-heading",
            "description": "Do not recommend replacing DPIA with FRIA either.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "conditional-complement",
            "description": "Retain complement plus a condition instead of a universal replacement rule.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "ai-law-version-dates",
      "model": "@cf/ibm-granite/granite-4.0-h-micro",
      "original_request_id": "284588ae3848ae0ff45d",
      "refined_request_id": "1a621a14f8da6444861e",
      "source_sha256": "6e29b4b562a0ed22cd5601c7fead96d122253bcb1d3855f14e4b1eeb6d709aa1",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 6,
        "lexical_total": 6,
        "prompt_tokens": 661,
        "output_tokens": 130,
        "latency_ms": 4777,
        "known_failure_guards": [
          {
            "id": "entry-force-not-application",
            "description": "Do not call 2 August 2026 the entry-into-force date.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-unsupported-absolute-entry-date",
            "description": "Do not invent an absolute entry-into-force date when publication timing is not supplied.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "entry-force-publication-rule",
            "description": "State the separate twentieth-day-after-publication entry rule.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 2,
        "lexical_total": 6,
        "prompt_tokens": 802,
        "output_tokens": 150,
        "latency_ms": 4569,
        "known_failure_guards": [
          {
            "id": "entry-force-not-application",
            "description": "Do not call 2 August 2026 the entry-into-force date.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-unsupported-absolute-entry-date",
            "description": "Do not invent an absolute entry-into-force date when publication timing is not supplied.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "entry-force-publication-rule",
            "description": "State the separate twentieth-day-after-publication entry rule.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "georgia-sample-exposure",
      "model": "@cf/ibm-granite/granite-4.0-h-micro",
      "original_request_id": "f22b3f5ab8688031c6af",
      "refined_request_id": "003cfc7959d2c43a6289",
      "source_sha256": "f0b2ed2c4a2110554108b9162678e5fa9be6f3724ac1f7fc79a057738a617611",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 5,
        "lexical_total": 5,
        "prompt_tokens": 614,
        "output_tokens": 87,
        "latency_ms": 3139,
        "known_failure_guards": [
          {
            "id": "synthetic-label-retained",
            "description": "Keep airtime values explicitly synthetic.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 5,
        "lexical_total": 5,
        "prompt_tokens": 717,
        "output_tokens": 127,
        "latency_ms": 4066,
        "known_failure_guards": [
          {
            "id": "synthetic-label-retained",
            "description": "Keep airtime values explicitly synthetic.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    },
    {
      "task_id": "ireland-dialogue-framing",
      "model": "@cf/ibm-granite/granite-4.0-h-micro",
      "original_request_id": "30b2de620a7bc3bbc9f5",
      "refined_request_id": "1e0bebd688c2a43add24",
      "source_sha256": "1d223d443aeb85900d1ea0a9353e7bc4b5a92759d9e36ca2bafc9d31eb03873b",
      "same_source_sha256": true,
      "original": {
        "lexical_checks": 5,
        "lexical_total": 7,
        "prompt_tokens": 616,
        "output_tokens": 156,
        "latency_ms": 5364,
        "known_failure_guards": [
          {
            "id": "synthetic-label-retained",
            "description": "Keep the invented/synthetic-statement label.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-invented-critics",
            "description": "Do not invent critics or attributed opposing speakers.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      },
      "refined": {
        "lexical_checks": 5,
        "lexical_total": 7,
        "prompt_tokens": 722,
        "output_tokens": 146,
        "latency_ms": 4674,
        "known_failure_guards": [
          {
            "id": "synthetic-label-retained",
            "description": "Keep the invented/synthetic-statement label.",
            "passed": false,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          },
          {
            "id": "no-invented-critics",
            "description": "Do not invent critics or attributed opposing speakers.",
            "passed": true,
            "scope": "Known-failure lexical guard; human source-level review remains required."
          }
        ]
      }
    }
  ]
}