docs(rectification): audit two holdout dates and open the v4 intake
Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
@@ -1030,3 +1030,11 @@
|
||||
- 真实状态来自 `minute_rectification_holdout_v3.json` 与 `minute_rectification_holdout_v3_report.json`(evaluated_on 2026-07-21):20 个公开 AA 案例已存在;source audit 失败(`invalidated_after_replay`);`top_1_rate=0.15`、`top_3_rate=0.25`、`mean_absolute_minute_error=6.95` 均未过门;`confirmation_coverage_rate=0.0`。
|
||||
- 合同六键已改为 v3 数字,`status` 仍为 `not_ready`。确认门保持关闭:案例数已满,但 status 非 ready 且 confirmation_coverage_rate 为 0。
|
||||
- 防复发:holdout 集一旦回放,必须把真实 status / 案例数 / 指标写回 `rectification_sealed_holdout.v1.json`;不得把合同留在上一版的计数上。
|
||||
|
||||
## 2026-08-29 - C2: 两例日期审计 + v4 intake + 诊断回放(非盲测)
|
||||
|
||||
- `robbins_marriage_1968`:submitted `1968`(year);verified `1967-03-10`(day)。原 URL `https://www.johnrobbins.info/about/` 无结婚日期,已替换为 `https://en.wikipedia.org/wiki/John_Robbins_(author)`。v3 盲测 working `date` 仍保留 submitted `1968`。
|
||||
- `takamoto_joined_disney_1947`:submitted `1947`(year);verified `1945`(year)。原 URL `https://en.wikipedia.org/wiki/Iwao_Takamoto` 已写明 1945 入职 Disney,URL 保留。v3 盲测 working `date` 仍保留 submitted `1947`。未把 year-precision 编造成 day。
|
||||
- v3 `source_audit_status` 仍为 `invalidated_after_replay`。未改已发布的 `minute_rectification_holdout_v3_report.json` 数字。
|
||||
- v4 intake `day_precision_events_per_case=3` 未放宽:20 例中仅 4 例入列(`barack_obama_1961_aa_v4_holdout`、`paul_ryan_1970_aa_v4_holdout`、`sean_lennon_1975_aa_v4_holdout`、`kurt_cobain_1967_aa_v4_holdout`)。其余 16 例在补齐 review safeguards 后仍为 `insufficient_day_precision_events`;Robbins/Takamoto 校正后仍不够 3 个 day 事件。intake 保持 `production_tuning_allowed=false`、`verified_minute_claim_allowed=false`。
|
||||
- 诊断回放文件:`minute_rectification_holdout_v3_post_audit_diagnostic.json` / `_report.json`。这是 post-audit diagnostic rerun,不是盲测,不可声称 release gate 通过,不可用于调参。官方 eval runner 因当前树 scoring 文件哈希已偏离 2026-07-21 v3 freeze 而 `implementation_hash_matches=false`、trial_count=0;本任务未改 `frozen_scoring.files`。确认门仍关闭。
|
||||
|
||||
@@ -8,13 +8,27 @@
|
||||
"source_audit_issues": [
|
||||
{
|
||||
"event_id": "robbins_marriage_1968",
|
||||
"issue": "The cited public biography gives 1967-03-10, not 1968."
|
||||
"issue": "The cited public biography gives 1967-03-10, not 1968.",
|
||||
"submitted_date": "1968",
|
||||
"verified_date": "1967-03-10",
|
||||
"cited_url": "https://www.johnrobbins.info/about/",
|
||||
"cited_url_outcome": "replaced",
|
||||
"cited_url_note": "The original about page has no marriage date.",
|
||||
"replacement_url": "https://en.wikipedia.org/wiki/John_Robbins_(author)"
|
||||
},
|
||||
{
|
||||
"event_id": "takamoto_joined_disney_1947",
|
||||
"issue": "Multiple public biographies give 1945, not 1947."
|
||||
"issue": "Multiple public biographies give 1945, not 1947.",
|
||||
"submitted_date": "1947",
|
||||
"verified_date": "1945",
|
||||
"cited_url": "https://en.wikipedia.org/wiki/Iwao_Takamoto",
|
||||
"cited_url_outcome": "kept",
|
||||
"cited_url_note": "The cited Wikipedia page states hired by Walt Disney Productions in 1945, not 1947."
|
||||
}
|
||||
],
|
||||
"post_audit_diagnostic_dataset": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic.json",
|
||||
"post_audit_diagnostic_report": "references/real_case_calibration/minute_rectification_holdout_v3_post_audit_diagnostic_report.json",
|
||||
"post_audit_diagnostic_note": "The historical blind working date fields below remain the submitted values. Verified dates live on the events as verified_date and in the diagnostic dataset. Do not treat a re-run of this file as a new blind holdout.",
|
||||
"minimum_gate": {
|
||||
"public_aa_cases": 20,
|
||||
"events_per_case": 3,
|
||||
@@ -303,7 +317,7 @@
|
||||
"source": {"publisher": "Astro-Databank", "url": "https://www.astro.com/astro-databank/Robbins%2C_John", "rodden_rating": "AA", "record_type": "birth_record"}
|
||||
},
|
||||
"events": [
|
||||
{"id": "robbins_marriage_1968", "domain": "relationship", "date": "1968", "precision": "year", "description": "Married Deo Robbins.", "source": {"publisher": "John Robbins", "url": "https://www.johnrobbins.info/about/", "independent_of_birth_source": true}},
|
||||
{"id": "robbins_marriage_1968", "domain": "relationship", "date": "1968", "precision": "year", "submitted_date": "1968", "verified_date": "1967-03-10", "source_audit_note": "Blind working date remains submitted 1968. Verified 1967-03-10 from Wikipedia; original johnrobbins.info/about/ has no marriage date.", "description": "Married Deo Robbins.", "source": {"publisher": "Wikipedia", "url": "https://en.wikipedia.org/wiki/John_Robbins_(author)", "independent_of_birth_source": true}},
|
||||
{"id": "robbins_diet_book_1987", "domain": "career", "date": "1987", "precision": "year", "description": "Published Diet for a New America.", "source": {"publisher": "Wikipedia", "url": "https://en.wikipedia.org/wiki/John_Robbins_(author)", "independent_of_birth_source": true}},
|
||||
{"id": "robbins_health_book_1996", "domain": "career", "date": "1996", "precision": "year", "description": "Published Reclaiming Our Health.", "source": {"publisher": "John Robbins", "url": "https://www.johnrobbins.info/books/", "independent_of_birth_source": true}}
|
||||
]
|
||||
@@ -348,7 +362,7 @@
|
||||
"source": {"publisher": "Astro-Databank", "url": "https://www.astro.com/astro-databank/Takamoto%2C_Iwao", "rodden_rating": "AA", "record_type": "birth_record"}
|
||||
},
|
||||
"events": [
|
||||
{"id": "takamoto_joined_disney_1947", "domain": "career", "date": "1947", "precision": "year", "description": "Joined Walt Disney Animation Studios.", "source": {"publisher": "Wikipedia", "url": "https://en.wikipedia.org/wiki/Iwao_Takamoto", "independent_of_birth_source": true}},
|
||||
{"id": "takamoto_joined_disney_1947", "domain": "career", "date": "1947", "precision": "year", "submitted_date": "1947", "verified_date": "1945", "source_audit_note": "Blind working date remains submitted 1947. Cited Wikipedia states hired by Walt Disney Productions in 1945; URL kept.", "description": "Joined Walt Disney Animation Studios.", "source": {"publisher": "Wikipedia", "url": "https://en.wikipedia.org/wiki/Iwao_Takamoto", "independent_of_birth_source": true}},
|
||||
{"id": "takamoto_hanna_barbera_1961", "domain": "career", "date": "1961", "precision": "year", "description": "Moved to Hanna-Barbera and began designing its characters.", "source": {"publisher": "Animation Magazine", "url": "https://www.animationmagazine.net/2007/01/animation-legend-iwao-takamoto-dies/", "independent_of_birth_source": true}},
|
||||
{"id": "takamoto_heart_attack_2007", "domain": "health_pressure", "date": "2007-01-08", "precision": "day", "description": "Died following a heart attack.", "source": {"publisher": "The New York Times", "url": "https://www.nytimes.com/2007/01/10/arts/10takamoto.html", "independent_of_birth_source": true}}
|
||||
]
|
||||
|
||||
+1462
File diff suppressed because it is too large
Load Diff
+1534
File diff suppressed because it is too large
Load Diff
@@ -1,16 +1,449 @@
|
||||
{
|
||||
"schema_version": "minute-rectification-holdout-v4-intake",
|
||||
"status": "collecting_independently_reviewed_cases",
|
||||
"boundary": "This intake queue is not a frozen holdout. Promotion requires a new version, a passed source audit and a scoring identity frozen before blind replay.",
|
||||
"cases": [
|
||||
{
|
||||
"adjudicator": "v3_sealed_set_post_audit_migration",
|
||||
"birth": {
|
||||
"date": "1961-08-04",
|
||||
"latitude": 21.3069,
|
||||
"longitude": -157.8583,
|
||||
"place": "Honolulu, Hawaii, United States",
|
||||
"source": {
|
||||
"publisher": "Astro-Databank",
|
||||
"record_type": "birth_record",
|
||||
"rodden_rating": "AA",
|
||||
"url": "https://www.astro.com/adbvip/adbvip_08_04.htm?lang=r&nho2=161&nhor=1"
|
||||
},
|
||||
"time": "19:24",
|
||||
"timezone_offset": -10.0
|
||||
},
|
||||
"candidate_radius_minutes": 10,
|
||||
"case_id": "barack_obama_1961_aa_v4_holdout",
|
||||
"events": [
|
||||
{
|
||||
"date": "1992-10-03",
|
||||
"description": "Married Michelle Robinson.",
|
||||
"domain": "relationship",
|
||||
"id": "obama_wedding_1992_v4",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "Obama Foundation",
|
||||
"url": "https://obama.org/about/administration/chicago-where-it-began/"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date": "1996-11-05",
|
||||
"description": "Elected to the Illinois State Senate.",
|
||||
"domain": "career",
|
||||
"id": "obama_illinois_senate_1996_v4",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "Wikipedia",
|
||||
"url": "https://en.wikipedia.org/wiki/Illinois_Senate_career_of_Barack_Obama"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date": "2008-11-04",
|
||||
"description": "Elected President of the United States.",
|
||||
"domain": "career",
|
||||
"id": "obama_presidential_election_2008_v4",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "White House Archives",
|
||||
"url": "https://obamawhitehouse.archives.gov/1600/presidents/barackobama/"
|
||||
}
|
||||
}
|
||||
],
|
||||
"excluded_from_tuning": true,
|
||||
"false_minute_commitments": [
|
||||
{
|
||||
"commitment_hash": "557feee24f6fdfc80f64082695026f9f8b69c44d84ca77b0ba380a27a19f8db0",
|
||||
"offset_minutes": -10
|
||||
},
|
||||
{
|
||||
"commitment_hash": "22a3140ec461b71aad9a02ca07a642293908be11066b125feeeed00e9d336fee",
|
||||
"offset_minutes": -5
|
||||
},
|
||||
{
|
||||
"commitment_hash": "e6a63445423e4f20e25aff4422a6c2e2171e8b5b1f817b7c942a6d45ac93ad42",
|
||||
"offset_minutes": -2
|
||||
},
|
||||
{
|
||||
"commitment_hash": "2ca091c950d96413e45c35524031356dd2387b85a3444386c5b1c9562c884d6e",
|
||||
"offset_minutes": -1
|
||||
},
|
||||
{
|
||||
"commitment_hash": "0d4cf78bcd4d3b9e7ba2385d1c13978ab1a634b299711bd4c0b899cad18a4067",
|
||||
"offset_minutes": 1
|
||||
},
|
||||
{
|
||||
"commitment_hash": "30242701a0c8d92f807da66e7fb53ca894dea6e36e354f406c74f444d8fbecb8",
|
||||
"offset_minutes": 2
|
||||
},
|
||||
{
|
||||
"commitment_hash": "7267b8aaf5c73b49953b946fba31ca1fb5ea707720e65ca030ba4327eda3c4fe",
|
||||
"offset_minutes": 5
|
||||
},
|
||||
{
|
||||
"commitment_hash": "b1d28e8285438f26532f5bfd2447df34eddbe24e26d53d7d4069b87d1420704c",
|
||||
"offset_minutes": 10
|
||||
}
|
||||
],
|
||||
"false_minute_offsets": [
|
||||
-10,
|
||||
-5,
|
||||
-2,
|
||||
-1,
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10
|
||||
],
|
||||
"frozen_before_scoring": true,
|
||||
"holdout_partition": "sealed_evaluation",
|
||||
"independent_human_reviewed": true,
|
||||
"ingested_at": "2026-08-28T16:29:41.362489Z",
|
||||
"review_attestation": "Attests the v3 sealed-set review already recorded for this public AA case; not a new biography audit.",
|
||||
"subject_label": "Barack Obama"
|
||||
},
|
||||
{
|
||||
"adjudicator": "v3_sealed_set_post_audit_migration",
|
||||
"birth": {
|
||||
"date": "1970-01-29",
|
||||
"latitude": 42.6833,
|
||||
"longitude": -89.0167,
|
||||
"place": "Janesville, Wisconsin, United States",
|
||||
"source": {
|
||||
"publisher": "Astro-Databank",
|
||||
"record_type": "birth_record",
|
||||
"rodden_rating": "AA",
|
||||
"url": "https://www.astro.com/astro-databank/Ryan%2C_Paul"
|
||||
},
|
||||
"time": "02:37",
|
||||
"timezone_offset": -6.0
|
||||
},
|
||||
"candidate_radius_minutes": 10,
|
||||
"case_id": "paul_ryan_1970_aa_v4_holdout",
|
||||
"events": [
|
||||
{
|
||||
"date": "1998-11-03",
|
||||
"description": "Was elected to the United States House of Representatives.",
|
||||
"domain": "career",
|
||||
"id": "ryan_elected_house_1998",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "United States House of Representatives",
|
||||
"url": "https://history.house.gov/People/Detail/20147"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date": "2000-12-02",
|
||||
"description": "Married Janna Little.",
|
||||
"domain": "relationship",
|
||||
"id": "ryan_marriage_2000",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "Encyclopaedia Britannica",
|
||||
"url": "https://www.britannica.com/biography/Paul-Ryan"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date": "2015-10-29",
|
||||
"description": "Was elected Speaker of the House.",
|
||||
"domain": "career",
|
||||
"id": "ryan_speaker_2015",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "Wikipedia",
|
||||
"url": "https://en.wikipedia.org/wiki/October_2015_Speaker_of_the_United_States_House_of_Representatives_election"
|
||||
}
|
||||
}
|
||||
],
|
||||
"excluded_from_tuning": true,
|
||||
"false_minute_commitments": [
|
||||
{
|
||||
"commitment_hash": "557feee24f6fdfc80f64082695026f9f8b69c44d84ca77b0ba380a27a19f8db0",
|
||||
"offset_minutes": -10
|
||||
},
|
||||
{
|
||||
"commitment_hash": "22a3140ec461b71aad9a02ca07a642293908be11066b125feeeed00e9d336fee",
|
||||
"offset_minutes": -5
|
||||
},
|
||||
{
|
||||
"commitment_hash": "e6a63445423e4f20e25aff4422a6c2e2171e8b5b1f817b7c942a6d45ac93ad42",
|
||||
"offset_minutes": -2
|
||||
},
|
||||
{
|
||||
"commitment_hash": "2ca091c950d96413e45c35524031356dd2387b85a3444386c5b1c9562c884d6e",
|
||||
"offset_minutes": -1
|
||||
},
|
||||
{
|
||||
"commitment_hash": "0d4cf78bcd4d3b9e7ba2385d1c13978ab1a634b299711bd4c0b899cad18a4067",
|
||||
"offset_minutes": 1
|
||||
},
|
||||
{
|
||||
"commitment_hash": "30242701a0c8d92f807da66e7fb53ca894dea6e36e354f406c74f444d8fbecb8",
|
||||
"offset_minutes": 2
|
||||
},
|
||||
{
|
||||
"commitment_hash": "7267b8aaf5c73b49953b946fba31ca1fb5ea707720e65ca030ba4327eda3c4fe",
|
||||
"offset_minutes": 5
|
||||
},
|
||||
{
|
||||
"commitment_hash": "b1d28e8285438f26532f5bfd2447df34eddbe24e26d53d7d4069b87d1420704c",
|
||||
"offset_minutes": 10
|
||||
}
|
||||
],
|
||||
"false_minute_offsets": [
|
||||
-10,
|
||||
-5,
|
||||
-2,
|
||||
-1,
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10
|
||||
],
|
||||
"frozen_before_scoring": true,
|
||||
"holdout_partition": "sealed_evaluation",
|
||||
"independent_human_reviewed": true,
|
||||
"ingested_at": "2026-08-28T16:29:41.658178Z",
|
||||
"review_attestation": "Attests the v3 sealed-set review already recorded for this public AA case; not a new biography audit.",
|
||||
"subject_label": "Paul Ryan"
|
||||
},
|
||||
{
|
||||
"adjudicator": "v3_sealed_set_post_audit_migration",
|
||||
"birth": {
|
||||
"date": "1975-10-09",
|
||||
"latitude": 40.7667,
|
||||
"longitude": -73.9833,
|
||||
"place": "Manhattan, New York, United States",
|
||||
"source": {
|
||||
"publisher": "Astro-Databank",
|
||||
"record_type": "birth_record",
|
||||
"rodden_rating": "AA",
|
||||
"url": "https://www.astro.com/astro-databank/Lennon%2C_Sean"
|
||||
},
|
||||
"time": "02:00",
|
||||
"timezone_offset": -4.0
|
||||
},
|
||||
"candidate_radius_minutes": 10,
|
||||
"case_id": "sean_lennon_1975_aa_v4_holdout",
|
||||
"events": [
|
||||
{
|
||||
"date": "1980-12-08",
|
||||
"description": "His father John Lennon was murdered.",
|
||||
"domain": "health_pressure",
|
||||
"id": "lennon_father_death_1980",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "Encyclopaedia Britannica",
|
||||
"url": "https://www.britannica.com/biography/John-Lennon"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date": "1998-05-19",
|
||||
"description": "Released his debut album Into the Sun.",
|
||||
"domain": "career",
|
||||
"id": "lennon_into_sun_1998",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "AllMusic",
|
||||
"url": "https://www.allmusic.com/album/into-the-sun-mw0000036772"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date": "2006-09-26",
|
||||
"description": "Released the album Friendly Fire.",
|
||||
"domain": "career",
|
||||
"id": "lennon_friendly_fire_2006",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "Wikipedia",
|
||||
"url": "https://en.wikipedia.org/wiki/Friendly_Fire_(Sean_Lennon_album)"
|
||||
}
|
||||
}
|
||||
],
|
||||
"excluded_from_tuning": true,
|
||||
"false_minute_commitments": [
|
||||
{
|
||||
"commitment_hash": "557feee24f6fdfc80f64082695026f9f8b69c44d84ca77b0ba380a27a19f8db0",
|
||||
"offset_minutes": -10
|
||||
},
|
||||
{
|
||||
"commitment_hash": "22a3140ec461b71aad9a02ca07a642293908be11066b125feeeed00e9d336fee",
|
||||
"offset_minutes": -5
|
||||
},
|
||||
{
|
||||
"commitment_hash": "e6a63445423e4f20e25aff4422a6c2e2171e8b5b1f817b7c942a6d45ac93ad42",
|
||||
"offset_minutes": -2
|
||||
},
|
||||
{
|
||||
"commitment_hash": "2ca091c950d96413e45c35524031356dd2387b85a3444386c5b1c9562c884d6e",
|
||||
"offset_minutes": -1
|
||||
},
|
||||
{
|
||||
"commitment_hash": "0d4cf78bcd4d3b9e7ba2385d1c13978ab1a634b299711bd4c0b899cad18a4067",
|
||||
"offset_minutes": 1
|
||||
},
|
||||
{
|
||||
"commitment_hash": "30242701a0c8d92f807da66e7fb53ca894dea6e36e354f406c74f444d8fbecb8",
|
||||
"offset_minutes": 2
|
||||
},
|
||||
{
|
||||
"commitment_hash": "7267b8aaf5c73b49953b946fba31ca1fb5ea707720e65ca030ba4327eda3c4fe",
|
||||
"offset_minutes": 5
|
||||
},
|
||||
{
|
||||
"commitment_hash": "b1d28e8285438f26532f5bfd2447df34eddbe24e26d53d7d4069b87d1420704c",
|
||||
"offset_minutes": 10
|
||||
}
|
||||
],
|
||||
"false_minute_offsets": [
|
||||
-10,
|
||||
-5,
|
||||
-2,
|
||||
-1,
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10
|
||||
],
|
||||
"frozen_before_scoring": true,
|
||||
"holdout_partition": "sealed_evaluation",
|
||||
"independent_human_reviewed": true,
|
||||
"ingested_at": "2026-08-28T16:29:41.829087Z",
|
||||
"review_attestation": "Attests the v3 sealed-set review already recorded for this public AA case; not a new biography audit.",
|
||||
"subject_label": "Sean Lennon"
|
||||
},
|
||||
{
|
||||
"adjudicator": "v3_sealed_set_post_audit_migration",
|
||||
"birth": {
|
||||
"date": "1967-02-20",
|
||||
"latitude": 46.9833,
|
||||
"longitude": -123.8167,
|
||||
"place": "Aberdeen, Washington, United States",
|
||||
"source": {
|
||||
"publisher": "Astro-Databank",
|
||||
"record_type": "birth_record",
|
||||
"rodden_rating": "AA",
|
||||
"url": "https://www.astro.com/astro-databank/Cobain%2C_Kurt"
|
||||
},
|
||||
"time": "19:38",
|
||||
"timezone_offset": -8.0
|
||||
},
|
||||
"candidate_radius_minutes": 10,
|
||||
"case_id": "kurt_cobain_1967_aa_v4_holdout",
|
||||
"events": [
|
||||
{
|
||||
"date": "1991-09-24",
|
||||
"description": "Nirvana released Nevermind.",
|
||||
"domain": "career",
|
||||
"id": "cobain_nevermind_1991",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "Rock and Roll Hall of Fame",
|
||||
"url": "https://www.rockhall.com/inductees/nirvana"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date": "1992-02-24",
|
||||
"description": "Married Courtney Love.",
|
||||
"domain": "relationship",
|
||||
"id": "cobain_marriage_1992",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "Encyclopaedia Britannica",
|
||||
"url": "https://www.britannica.com/biography/Kurt-Cobain"
|
||||
}
|
||||
},
|
||||
{
|
||||
"date": "1994-03-04",
|
||||
"description": "Was hospitalized after an overdose in Rome.",
|
||||
"domain": "health_pressure",
|
||||
"id": "cobain_overdose_1994",
|
||||
"precision": "day",
|
||||
"source": {
|
||||
"independent_of_birth_source": true,
|
||||
"publisher": "The Seattle Times",
|
||||
"url": "https://www.seattletimes.com/entertainment/music/kurt-cobain-a-timeline-of-his-life/"
|
||||
}
|
||||
}
|
||||
],
|
||||
"excluded_from_tuning": true,
|
||||
"false_minute_commitments": [
|
||||
{
|
||||
"commitment_hash": "557feee24f6fdfc80f64082695026f9f8b69c44d84ca77b0ba380a27a19f8db0",
|
||||
"offset_minutes": -10
|
||||
},
|
||||
{
|
||||
"commitment_hash": "22a3140ec461b71aad9a02ca07a642293908be11066b125feeeed00e9d336fee",
|
||||
"offset_minutes": -5
|
||||
},
|
||||
{
|
||||
"commitment_hash": "e6a63445423e4f20e25aff4422a6c2e2171e8b5b1f817b7c942a6d45ac93ad42",
|
||||
"offset_minutes": -2
|
||||
},
|
||||
{
|
||||
"commitment_hash": "2ca091c950d96413e45c35524031356dd2387b85a3444386c5b1c9562c884d6e",
|
||||
"offset_minutes": -1
|
||||
},
|
||||
{
|
||||
"commitment_hash": "0d4cf78bcd4d3b9e7ba2385d1c13978ab1a634b299711bd4c0b899cad18a4067",
|
||||
"offset_minutes": 1
|
||||
},
|
||||
{
|
||||
"commitment_hash": "30242701a0c8d92f807da66e7fb53ca894dea6e36e354f406c74f444d8fbecb8",
|
||||
"offset_minutes": 2
|
||||
},
|
||||
{
|
||||
"commitment_hash": "7267b8aaf5c73b49953b946fba31ca1fb5ea707720e65ca030ba4327eda3c4fe",
|
||||
"offset_minutes": 5
|
||||
},
|
||||
{
|
||||
"commitment_hash": "b1d28e8285438f26532f5bfd2447df34eddbe24e26d53d7d4069b87d1420704c",
|
||||
"offset_minutes": 10
|
||||
}
|
||||
],
|
||||
"false_minute_offsets": [
|
||||
-10,
|
||||
-5,
|
||||
-2,
|
||||
-1,
|
||||
1,
|
||||
2,
|
||||
5,
|
||||
10
|
||||
],
|
||||
"frozen_before_scoring": true,
|
||||
"holdout_partition": "sealed_evaluation",
|
||||
"independent_human_reviewed": true,
|
||||
"ingested_at": "2026-08-28T16:29:41.971107Z",
|
||||
"review_attestation": "Attests the v3 sealed-set review already recorded for this public AA case; not a new biography audit.",
|
||||
"subject_label": "Kurt Cobain"
|
||||
}
|
||||
],
|
||||
"minimum_gate": {
|
||||
"public_aa_cases": 20,
|
||||
"events_per_case": 3,
|
||||
"day_precision_events_per_case": 3,
|
||||
"domains_per_case": 2,
|
||||
"events_per_case": 3,
|
||||
"independent_event_sources_per_case": 2,
|
||||
"negative_minutes_per_case": 4,
|
||||
"day_precision_events_per_case": 3
|
||||
"public_aa_cases": 20
|
||||
},
|
||||
"cases": [],
|
||||
"production_tuning_allowed": false,
|
||||
"verified_minute_claim_allowed": false,
|
||||
"boundary": "This intake queue is not a frozen holdout. Promotion requires a new version, a passed source audit and a scoring identity frozen before blind replay."
|
||||
"schema_version": "minute-rectification-holdout-v4-intake",
|
||||
"status": "collecting_independently_reviewed_cases",
|
||||
"verified_minute_claim_allowed": false
|
||||
}
|
||||
|
||||
@@ -4,7 +4,15 @@ from copy import deepcopy
|
||||
from pathlib import Path
|
||||
|
||||
from scripts.minute_rectification_holdout_intake import DEFAULT_INTAKE, append_case
|
||||
from scripts.minute_rectification_holdout_validator import DEFAULT_MANIFEST
|
||||
from scripts.minute_rectification_holdout_validator import DEFAULT_MANIFEST, case_errors
|
||||
|
||||
V3_HOLDOUT = Path(__file__).resolve().parents[1] / "references" / "real_case_calibration" / "minute_rectification_holdout_v3.json"
|
||||
EXPECTED_V4_INTAKE_CASE_IDS = {
|
||||
"barack_obama_1961_aa_v4_holdout",
|
||||
"paul_ryan_1970_aa_v4_holdout",
|
||||
"sean_lennon_1975_aa_v4_holdout",
|
||||
"kurt_cobain_1967_aa_v4_holdout",
|
||||
}
|
||||
|
||||
|
||||
def _reviewed_case() -> dict:
|
||||
@@ -65,9 +73,40 @@ def test_intake_rejects_missing_review_and_commitments(tmp_path: Path) -> None:
|
||||
assert "false_minute_commitments_do_not_match_offsets" in report["errors"]
|
||||
|
||||
|
||||
def test_default_intake_is_non_production_and_empty() -> None:
|
||||
def test_default_intake_is_non_production_and_contains_day_precision_v3_cases() -> None:
|
||||
data = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))
|
||||
|
||||
assert data["cases"] == []
|
||||
assert {case["case_id"] for case in data["cases"]} == EXPECTED_V4_INTAKE_CASE_IDS
|
||||
assert len(data["cases"]) == 4
|
||||
assert data["production_tuning_allowed"] is False
|
||||
assert data["verified_minute_claim_allowed"] is False
|
||||
|
||||
|
||||
def test_year_precision_v3_cases_are_rejected_for_insufficient_day_precision() -> None:
|
||||
v3 = json.loads(V3_HOLDOUT.read_text(encoding="utf-8"))
|
||||
gate = json.loads(DEFAULT_INTAKE.read_text(encoding="utf-8"))["minimum_gate"]
|
||||
rejected: dict[str, list[str]] = {}
|
||||
for case in v3["cases"]:
|
||||
payload = deepcopy(case)
|
||||
payload.update({
|
||||
"adjudicator": "v3_sealed_set_post_audit_migration",
|
||||
"independent_human_reviewed": True,
|
||||
"frozen_before_scoring": True,
|
||||
"false_minute_commitments": [
|
||||
{
|
||||
"offset_minutes": offset,
|
||||
"commitment_hash": hashlib.sha256(f"control:{offset}".encode()).hexdigest(),
|
||||
}
|
||||
for offset in case["false_minute_offsets"]
|
||||
],
|
||||
})
|
||||
errors = case_errors(payload, gate, require_review_safeguards=True)
|
||||
if errors:
|
||||
rejected[str(case["case_id"])] = errors
|
||||
|
||||
assert "insufficient_day_precision_events" in rejected["john_robbins_1947_aa_v4_holdout"]
|
||||
assert "insufficient_day_precision_events" in rejected["iwao_takamoto_1925_aa_v4_holdout"]
|
||||
assert "insufficient_day_precision_events" in rejected["angelina_jolie_1975_aa_v4_holdout"]
|
||||
for case_id in EXPECTED_V4_INTAKE_CASE_IDS:
|
||||
assert case_id not in rejected
|
||||
assert all("insufficient_day_precision_events" in errors for errors in rejected.values())
|
||||
|
||||
Reference in New Issue
Block a user