{ "attestation_id": "ATT-8556DEB4390E", "attestation_json": "out/sample.attestation.json", "bins": 10, "disclaimer": "This is a vendor-neutral attestation of the calibration of the *logged decisions supplied*. It certifies that, on those rows, the model's stated probabilities matched observed outcomes to the degree reported, and that the underlying log is tamper-evident. It is NOT a guarantee of model quality on unseen inputs, NOT a certification of regulatory compliance, and NOT legal advice. SR 26-2 (which supersedes SR 11-7 and SR 21-8, eff. 2026-04-17) is non-binding, is most relevant to banking organizations with >$30B in assets, and places generative / agentic AI outside its scope. Regulatory applicability (SR 26-2, EU AI Act, and any successor guidance) must be confirmed by qualified counsel against the deployer's specific use case. Calibration measured on past logged outcomes does not transfer to a different population or task without revalidation.", "evidence_file": "out/sample.evidence.jsonl", "field_mapping": { "domain": "domain", "model": "model", "outcome": "outcome", "prob": "predicted_probability", "ts": "timestamp" }, "generated_utc": "2026-06-09T21:08:58Z", "input_stats": { "rows_read": 180, "rows_used": 180, "skipped_invalid_probability": 0, "skipped_unresolved_outcome": 0 }, "merkle_tip": "50a5a296c397eaf45e92de86a2da85800e7dba751a10667a37496f6a35e265f4", "models": [ { "model": "model-aurora-v2", "scores": { "auc": 0.7087, "base_rate": 0.6667, "brier": 0.2089, "brier_skill_score": 0.0598, "calibration_quality": "marginally calibrated", "confidence_direction": "over-confident (predicted probabilities exceed observed outcomes)", "ece": 0.1279, "mce": 0.3826, "mean_predicted": 0.7694, "n": 60, "reliability_bins": [ { "bin_hi": 0.1, "bin_lo": 0.0, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.2, "bin_lo": 0.1, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.3, "bin_lo": 0.2, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.4, "bin_lo": 0.3, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.5, "bin_lo": 0.4, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.6, "bin_lo": 0.5, "gap": 0.3826, "mean_pred": 0.5826, "n": 5, "observed_rate": 0.2 }, { "bin_hi": 0.7, "bin_lo": 0.6, "gap": 0.0798, "mean_pred": 0.6512, "n": 14, "observed_rate": 0.5714 }, { "bin_hi": 0.8, "bin_lo": 0.7, "gap": 0.1055, "mean_pred": 0.7526, "n": 17, "observed_rate": 0.6471 }, { "bin_hi": 0.9, "bin_lo": 0.8, "gap": 0.0687, "mean_pred": 0.8404, "n": 11, "observed_rate": 0.9091 }, { "bin_hi": 1.0, "bin_lo": 0.9, "gap": 0.1611, "mean_pred": 0.9303, "n": 13, "observed_rate": 0.7692 } ] } }, { "model": "model-orion-lite", "scores": { "auc": 0.7097, "base_rate": 0.7, "brier": 0.2012, "brier_skill_score": 0.0417, "calibration_quality": "marginally calibrated", "confidence_direction": "under-confident (predicted probabilities trail observed outcomes)", "ece": 0.14, "mce": 0.2023, "mean_predicted": 0.6, "n": 60, "reliability_bins": [ { "bin_hi": 0.1, "bin_lo": 0.0, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.2, "bin_lo": 0.1, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.3, "bin_lo": 0.2, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.4, "bin_lo": 0.3, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.5, "bin_lo": 0.4, "gap": 0.0925, "mean_pred": 0.4771, "n": 13, "observed_rate": 0.3846 }, { "bin_hi": 0.6, "bin_lo": 0.5, "gap": 0.2023, "mean_pred": 0.5346, "n": 19, "observed_rate": 0.7368 }, { "bin_hi": 0.7, "bin_lo": 0.6, "gap": 0.0769, "mean_pred": 0.6504, "n": 11, "observed_rate": 0.7273 }, { "bin_hi": 0.8, "bin_lo": 0.7, "gap": 0.1476, "mean_pred": 0.7347, "n": 17, "observed_rate": 0.8824 }, { "bin_hi": 0.9, "bin_lo": 0.8, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 1.0, "bin_lo": 0.9, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null } ] } }, { "model": "model-vega-fast", "scores": { "auc": 0.6019, "base_rate": 0.7, "brier": 0.2391, "brier_skill_score": -0.1384, "calibration_quality": "poorly calibrated", "confidence_direction": "over-confident (predicted probabilities exceed observed outcomes)", "ece": 0.1844, "mce": 0.295, "mean_predicted": 0.8844, "n": 60, "reliability_bins": [ { "bin_hi": 0.1, "bin_lo": 0.0, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.2, "bin_lo": 0.1, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.3, "bin_lo": 0.2, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.4, "bin_lo": 0.3, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.5, "bin_lo": 0.4, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.6, "bin_lo": 0.5, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.7, "bin_lo": 0.6, "gap": null, "mean_pred": null, "n": 0, "observed_rate": null }, { "bin_hi": 0.8, "bin_lo": 0.7, "gap": 0.295, "mean_pred": 0.795, "n": 6, "observed_rate": 0.5 }, { "bin_hi": 0.9, "bin_lo": 0.8, "gap": 0.2112, "mean_pred": 0.8541, "n": 28, "observed_rate": 0.6429 }, { "bin_hi": 1.0, "bin_lo": 0.9, "gap": 0.13, "mean_pred": 0.9377, "n": 26, "observed_rate": 0.8077 } ] } } ], "schema": "calibration-attestation/v1", "subject": "ILLUSTRATIVE SAMPLE - synthetic data, no real model", "tool": "generate_attestation.py" }