From ab8dd3ff003d5dcafe81a4dcf3a96283d9ce45d2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 14:30:32 -0700 Subject: [PATCH 01/21] Cite US and UK spec sources by symbol or at a commit, never by a drifting line Rebuild #1146 on current main without its NZ files; #1282 landed the NZ half with its own stricter grammar. Add the shared validator (test_support/microcosm_build/spec_source_citations.py) and an engine-free guard over every country package. It refuses a line citation of a file in this repository, pinned or not, and a line citation of any other source unless its string names the commit or the resource pins the file in a path/commit mapping. Every dotted microcosm. name must resolve. US: port #1146's known-gap and source-stage rewrites that still apply (main dropped the employer-premium gap), and name archived commit 42ed5d45 in the ESI stage note added since. UK: the parity register's 13 citations name its incumbent commit a43e8c5, each re-read there (hmrc_cgt 195 -> 196, hmrc_spi 230 -> 229, frs_only.py -> services.py 137-148); the population contract's citations name registry ref 12a1e028 (30 ported strings, 9 rewritten since #1146 re-read there; compute_esa is at income.py 98-102 at that ref). Re-pin the source_stages.json raw hash and regenerate us/spec/sources.yaml. The digests derived from it follow in the next commit. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 1 + .../build/uk/uk_data_target_parity.json | 26 +- .../build/uk/uk_population_targets.json | 82 +- .../build/uk_runtime/data_target_parity.py | 26 +- .../build/us/ecps_parity_known_gaps.json | 14 +- .../src/microcosm/build/us/source_stages.json | 16 +- .../src/microcosm/build/us/spec/sources.yaml | 156 ++-- .../build/us/target_parity_manifest.json | 2 +- .../shared/test_spec_source_citations.py | 721 ++++++++++++++++++ .../us/test_us_bundle_core_contracts.py | 2 +- .../test_us_financial_assistance_exclusion.py | 13 +- .../us/test_us_survivor_benefits_exclusion.py | 13 +- .../test_us_wic_nutritional_risk_exclusion.py | 13 +- .../microcosm_build/spec_source_citations.py | 285 +++++++ .../microcosm_build/us_spec_bundle.py | 2 +- tools/build_us_target_parity_manifest.py | 3 +- tools/generate_us_bundle_from_constants.py | 2 +- 17 files changed, 1209 insertions(+), 168 deletions(-) create mode 100644 changelog.d/spec-source-citations-us-uk.fixed.md create mode 100644 packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py create mode 100644 test_support/microcosm_build/spec_source_citations.py diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md new file mode 100644 index 0000000000..6467087a62 --- /dev/null +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -0,0 +1 @@ +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, and a line citation of any other source unless its string names the commit (`commit `, `at `, `@` or a `/blob//` link) or the resource pins that file in a `path`/`commit` mapping. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` name archived commit `42ed5d45`. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json b/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json index d48d81dd45..3c8780e8a6 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_data_target_parity.json @@ -178,7 +178,7 @@ "policyengine_uk_data.datasets.imputations.services.services", "policyengine_uk_data.datasets.imputations.consumption" ], - "evidence": "uk-data datasets/imputations/services/services.py:33 and datasets/imputations/consumption.py:790,830; Chronicle#200/#202.", + "evidence": "uk-data at a43e8c5 datasets/imputations/services/services.py:33 and datasets/imputations/consumption.py:790,830; Chronicle#200/#202.", "fence": { "origin": "uk-data post-imputation-rescaled bus subsidy and fares, including London share and quintile rates.", "purpose": "Preserve service-consumption totals and distribution.", @@ -194,7 +194,7 @@ "policyengine_uk_data.datasets.imputations.services.services", "policyengine_uk_data.tests.test_aggregates" ], - "evidence": "uk-data datasets/imputations/frs_only.py and uk-data tests/test_aggregates.py:5; uk-data#199 and uk-data#474.", + "evidence": "uk-data at a43e8c5 datasets/imputations/services/services.py:137-148 (the dfe_education_spending imputation) and tests/test_aggregates.py:5 (its commented-out aggregate); uk-data#199 and uk-data#474.", "fence": { "origin": "uk-data imputed DfE education spending without an active calibration target.", "purpose": "Retain the imputed input while avoiding an unsupported aggregate claim.", @@ -209,7 +209,7 @@ "covers": [ "policyengine_uk_data.datasets.imputations.services.services" ], - "evidence": "uk-data datasets/imputations/services/services.py:18; Chronicle transport facts landed in #200/#202.", + "evidence": "uk-data at a43e8c5 datasets/imputations/services/services.py:18; Chronicle transport facts landed in #200/#202.", "fence": { "origin": "uk-data rescaled rail_usage to a national subsidy total.", "purpose": "Anchor the imputed service-consumption mass.", @@ -877,7 +877,7 @@ "policyengine_uk_data.targets.sources.la_council_tax", "policyengine_uk_data.datasets.local_areas.local_authorities.loss" ], - "evidence": "uk-data targets/sources/la_council_tax.py and datasets/local_areas/local_authorities/loss.py:251-300; Microcosm council_tax/band_a..i contract rows (mhclg, welshgov and scotgov by_area families) bind 2,511 authority cells on the taxbase basis with the 296 English band-H cells and Shetland's band-H cell signed deferred (microcosm#929).", + "evidence": "uk-data at a43e8c5 targets/sources/la_council_tax.py and datasets/local_areas/local_authorities/loss.py:251-300; Microcosm council_tax/band_a..i contract rows (mhclg, welshgov and scotgov by_area families) bind 2,511 authority cells on the taxbase basis with the 296 English band-H cells and Shetland's band-H cell signed deferred (microcosm#929).", "status": "ported_local_declared" }, { @@ -957,7 +957,7 @@ "covers": [ "policyengine_uk_data.targets.sources.voa_council_tax" ], - "evidence": "uk-data targets/sources/voa_council_tax.py:235,275; Microcosm mhclg.council_tax_stock.* (England, composed from the MHCLG taxbase authority rows per region), welshgov.council_tax_stock.* and scotgov.council_tax_stock.* targets on the councils' taxbase basis (microcosm#929).", + "evidence": "uk-data at a43e8c5 targets/sources/voa_council_tax.py:235,275; Microcosm mhclg.council_tax_stock.* (England, composed from the MHCLG taxbase authority rows per region), welshgov.council_tax_stock.* and scotgov.council_tax_stock.* targets on the councils' taxbase basis (microcosm#929).", "status": "ported_national" }, { @@ -1008,7 +1008,7 @@ "covers": [ "policyengine_uk_data.targets.sources.hmrc_cgt" ], - "evidence": "uk-data targets/sources/hmrc_cgt.py:195; Microcosm hmrc_cgt_conditioning_facts.json (vendored HMRC Table 2.1a 2024-25 rows read by the donor stage and the amounts redraw) and uk_population_targets.json CGT rows.", + "evidence": "uk-data at a43e8c5 targets/sources/hmrc_cgt.py:196 (_band_targets); Microcosm hmrc_cgt_conditioning_facts.json (vendored HMRC Table 2.1a 2024-25 rows read by the donor stage and the amounts redraw) and uk_population_targets.json CGT rows.", "status": "ported_national" }, { @@ -1026,7 +1026,7 @@ "covers": [ "policyengine_uk_data.targets.sources.hmrc_spi" ], - "evidence": "uk-data targets/sources/hmrc_spi.py:230; Microcosm HMRC SPI amount/count-by-total-income-band targets and replay fences.", + "evidence": "uk-data at a43e8c5 targets/sources/hmrc_spi.py:229 (get_targets); Microcosm HMRC SPI amount/count-by-total-income-band targets and replay fences.", "status": "ported_national" }, { @@ -1066,7 +1066,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:302 (_parse_council_tax); Microcosm target ids obr.council_tax (OBR table 4.1 row 15, England, Scotland and Wales receipts over Great Britain's households, microcosm#1095) and country legs obr.council_tax_{england,scotland,wales}.", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:302 (_parse_council_tax); Microcosm target ids obr.council_tax (OBR table 4.1 row 15, England, Scotland and Wales receipts over Great Britain's households, microcosm#1095) and country legs obr.council_tax_{england,scotland,wales}.", "status": "ported_national" }, { @@ -1075,7 +1075,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:366 (_parse_nics); Microcosm target ids obr.ni_employee, obr.ni_employer, and obr.ni_self_employed; OBR's cash total obr.ni is a diagnostic on obr.ni_employee (microcosm#1095).", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:366 (_parse_nics); Microcosm target ids obr.ni_employee, obr.ni_employer, and obr.ni_self_employed; OBR's cash total obr.ni is a diagnostic on obr.ni_employee (microcosm#1095).", "status": "ported_national" }, { @@ -1084,7 +1084,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:420-427 (the Class 3 note in _parse_nics) and uk-data#88.", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:420-427 (the Class 3 note in _parse_nics) and uk-data#88.", "fence": { "origin": "uk-data skipped the Class 3 OBR row pending uk-data#88.", "purpose": "Avoid pretending voluntary Class 3 contributions are represented by another NIC variable.", @@ -1108,7 +1108,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:502 (_parse_welfare) welfare rows; Microcosm uk_population_targets.json OBR benefit-spending targets. Since uk-data a43e8c5 the Housing Benefit and Pension Credit lines come from DWP's benefit expenditure and caseload tables (targets/sources/dwp_housing_benefit.py, uk-data#490; dwp_pension_credit.py, uk-data#510) and Universal Credit is one Great Britain total (uk-data#530).", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:502 (_parse_welfare) welfare rows; Microcosm uk_population_targets.json OBR benefit-spending targets. Since uk-data a43e8c5 the Housing Benefit and Pension Credit lines come from DWP's benefit expenditure and caseload tables (targets/sources/dwp_housing_benefit.py, uk-data#490; dwp_pension_credit.py, uk-data#510) and Universal Credit is one Great Britain total (uk-data#530).", "status": "ported_national" }, { @@ -1126,7 +1126,7 @@ "covers": [ "policyengine_uk_data.targets.sources.ons_tenure" ], - "evidence": "uk-data targets/sources/ons_tenure.py:69; Microcosm local ons.tenure.* contract rows.", + "evidence": "uk-data at a43e8c5 targets/sources/ons_tenure.py:69 (get_targets); Microcosm local ons.tenure.* contract rows.", "fence": { "origin": "uk-data calibrated five England tenure controls.", "purpose": "Preserve an England-level representation check independently of the local Census tenure surface.", @@ -1211,7 +1211,7 @@ "covers": [ "policyengine_uk_data.targets.sources.obr" ], - "evidence": "uk-data targets/sources/obr.py:620 (_parse_tv_licence) and 668-684 (the private-school and salary-sacrifice static rows before get_targets); Microcosm uk_population_targets.json tv_licence, private-school, and salary-sacrifice target rows.", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:620 (_parse_tv_licence) and 668-684 (the private-school and salary-sacrifice static rows before get_targets); Microcosm uk_population_targets.json tv_licence, private-school, and salary-sacrifice target rows.", "status": "ported_national" }, { diff --git a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json index 23c632ae2d..f9f14db678 100644 --- a/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json +++ b/packages/microcosm-build/src/microcosm/build/uk/uk_population_targets.json @@ -685,9 +685,9 @@ "slc/student_loan_repayment/england/postgraduate": "Retired (microcosm#1095, the next policyengine-uk bump): the student-loans stage assigns no postgraduate plan (its domain is NONE, PLAN_1, PLAN_2 and PLAN_5) and policyengine-uk holds one plan a person, while a postgraduate loan is repaid beside the undergraduate plan most of its borrowers hold, so no record carries postgraduate repayments and no weights can reach SLC's figure (microcosm#757)." }, "unmapped_declarations": { - "hmrc.salary_sacrifice.it_relief_basic_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", - "hmrc.salary_sacrifice.it_relief_higher_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", - "hmrc.salary_sacrifice.it_relief_additional_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394.", + "hmrc.salary_sacrifice.it_relief_basic_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them.", + "hmrc.salary_sacrifice.it_relief_higher_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them.", + "hmrc.salary_sacrifice.it_relief_additional_rate": "New declaration backed by chronicle#139 salary-sacrifice facts; the incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them.", "ons.population.female_90_plus": "María ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference.", "ons.population.male_90_plus": "María ruling 2026-08-21: split the incumbent terminal sex bands into uniform five-year 85_89 bands plus 90_plus tails. The incumbent six-year 85_90 terminal rows leave ages 91+ unconstrained; Microcosm keeps the uniform five-year bands and constrains the tail. The 85_89-vs-85_90 value gap is the single-age-90 share and is a signed semantic difference.", "hmrc.tfc.government_top_up": "Post-freeze fact family, no uk-data metric.", @@ -892,23 +892,23 @@ "suppressed_ancestors": { "hmrc/salary_sacrifice_it_relief_basic_rate": { "contract_target_id": "hmrc.salary_sacrifice.it_relief_basic_rate", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." }, "hmrc/salary_sacrifice_it_relief_higher_rate": { "contract_target_id": "hmrc.salary_sacrifice.it_relief_higher_rate", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." }, "hmrc/salary_sacrifice_it_relief_additional_rate": { "contract_target_id": "hmrc.salary_sacrifice.it_relief_additional_rate", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." }, "hmrc/salary_sacrifice_employee_nics_relief": { "contract_target_id": "hmrc.salary_sacrifice.nics_relief_employee", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." }, "hmrc/salary_sacrifice_employer_nics_relief": { "contract_target_id": "hmrc.salary_sacrifice.nics_relief_employer", - "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are silently suppressed by the bare-except drop in build_loss_matrix.py:376-394." + "rationale": "The incumbent source CSV is still 410-Gone at 12a1e028 and its rows are dropped by the catch-all except Exception in uk-data targets/sources/hmrc_salary_sacrifice.py lines 34-109 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3, which only logs the download error and returns the salary_sacrifice_contributions target alone, so build_loss_matrix.py never receives them." } }, "accounting_notes": { @@ -2493,7 +2493,7 @@ "map_to": "household" } ], - "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (compute/council_tax.py:21-43). It binds row 15 ('Total council tax receipts'), the sum of the England, Scotland and Wales rows, over Great Britain's households (microcosm#1095). Row 19 ('Total net council tax receipts', obr.council_tax_total, bound before) adds Northern Ireland domestic rates, the council tax accruals adjustment and the high-value council tax surcharge: GBP 1.34bn, 2.7% of the calendar-2025 window, which no household's council tax carries (domestic rates are obr.domestic_rates, held). Chronicle stamps the row K02000001, as OBR's table is UK-wide, and the selector pins it there. Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." + "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (uk-data targets/compute/council_tax.py:21-43 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). It binds row 15 ('Total council tax receipts'), the sum of the England, Scotland and Wales rows, over Great Britain's households (microcosm#1095). Row 19 ('Total net council tax receipts', obr.council_tax_total, bound before) adds Northern Ireland domestic rates, the council tax accruals adjustment and the high-value council tax surcharge: GBP 1.34bn, 2.7% of the calendar-2025 window, which no household's council tax carries (domestic rates are obr.domestic_rates, held). Chronicle stamps the row K02000001, as OBR's table is UK-wide, and the selector pins it there. Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." }, "axiom": { "metric_name": "obr/council_tax", @@ -2536,7 +2536,7 @@ "equals": "ENGLAND" } ], - "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (compute/council_tax.py:21-43). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." + "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (uk-data targets/compute/council_tax.py:21-43 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." }, "axiom": { "metric_name": "obr/council_tax_england", @@ -2579,7 +2579,7 @@ "equals": "SCOTLAND" } ], - "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (compute/council_tax.py:21-43). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." + "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (uk-data targets/compute/council_tax.py:21-43 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." }, "axiom": { "metric_name": "obr/council_tax_scotland", @@ -2622,7 +2622,7 @@ "equals": "WALES" } ], - "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (compute/council_tax.py:21-43). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." + "notes": "OBR Table 4.1 reports net council tax receipts (net of council tax reduction), so the binding is council_tax_less_benefit, not gross council_tax (uk-data targets/compute/council_tax.py:21-43 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane)." }, "axiom": { "metric_name": "obr/council_tax_wales", @@ -2719,7 +2719,7 @@ "map_to": "household" } ], - "notes": "EFO incapacity-benefits line. OBR table 4.9 footnote 2: the line includes incapacity benefit, employment and support allowance, severe disablement allowance and the incapacity part of income support, and excludes the health element of universal credit. The model column sums benunit-level esa_income and person-level esa_contrib, incapacity_benefit and sda to household; uk-data sums the first two (compute/income.py:115-119), and adding the other two follows uk-data#582 (microcosm#1095). Incapacity Benefit has been closed to new claims since ESA began in October 2008, so its term carries only what the FRS still reports. Income Support has no incapacity part in the engine, which pays it as one benefit; that part stays a residual of the row, with no declared difference (María's ruling of 2026-10-16). Severe Disablement Allowance is a small part of the line: DWP reports GBP 53.8m for England and Wales in 2024-25 and GBP 46.8m in 2025-26, against OBR's GBP 12.41bn and GBP 7.21bn. The Scottish Government pays SDA in Scotland (GBP 5.0m in 2024-25, GBP 4.4m in 2025-26 in the Scottish Budget), outside DWP's line; at well under 0.1 percent of the row it stays inside the Great Britain scope below rather than a per-term mask. Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane). Scoped to Great Britain (microcosm#1095, uk-data#490): OBR table 4.9 reports this line inside 'DWP social security', whose figures cover Great Britain and residents overseas (DWP benefit expenditure and caseload tables, Spring 2026, Notes 3), and lists Northern Ireland social security on rows of its own, so the model column sums Great Britain households only. Since the abf0921 re-pin the Chronicle fact carries the Great Britain code K03000001 (microcosm#1300)." + "notes": "EFO incapacity-benefits line. OBR table 4.9 footnote 2: the line includes incapacity benefit, employment and support allowance, severe disablement allowance and the incapacity part of income support, and excludes the health element of universal credit. The model column sums benunit-level esa_income and person-level esa_contrib, incapacity_benefit and sda to household; uk-data sums the first two (uk-data targets/compute/income.py:98-102 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3), and adding the other two follows uk-data#582 (microcosm#1095). Incapacity Benefit has been closed to new claims since ESA began in October 2008, so its term carries only what the FRS still reports. Income Support has no incapacity part in the engine, which pays it as one benefit; that part stays a residual of the row, with no declared difference (María's ruling of 2026-10-16). Severe Disablement Allowance is a small part of the line: DWP reports GBP 53.8m for England and Wales in 2024-25 and GBP 46.8m in 2025-26, against OBR's GBP 12.41bn and GBP 7.21bn. The Scottish Government pays SDA in Scotland (GBP 5.0m in 2024-25, GBP 4.4m in 2025-26 in the Scottish Budget), outside DWP's line; at well under 0.1 percent of the row it stays inside the Great Britain scope below rather than a per-term mask. Bound at the calendar-2025 window of the OBR fiscal years (three twelfths of FY2024-25, nine twelfths of FY2025-26; calendar_year_window), María's rule of 2026-09-22 that fiscal-year facts take only the months to the end of the calibration calendar year (PolicyEngine/chronicle#280 lane). Scoped to Great Britain (microcosm#1095, uk-data#490): OBR table 4.9 reports this line inside 'DWP social security', whose figures cover Great Britain and residents overseas (DWP benefit expenditure and caseload tables, Spring 2026, Notes 3), and lists Northern Ireland social security on rows of its own, so the model column sums Great Britain households only. Since the abf0921 re-pin the Chronicle fact carries the Great Britain code K03000001 (microcosm#1300)." }, "axiom": { "metric_name": "obr/esa", @@ -3367,7 +3367,7 @@ "output_variable": "ni_employee", "from_entity": "person", "map_to": "household", - "notes": "NICs relief attributed to salary sacrifice (compute/income.py:59-68). uk-data's SPP-Review-based obr/salary_sacrifice_employee_ni_relief row (source lost) retires onto this hmrc-backed fact - vintage difference for the microcosm#622 parity gate." + "notes": "NICs relief attributed to salary sacrifice (uk-data targets/compute/income.py:59-68 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). uk-data's SPP-Review-based obr/salary_sacrifice_employee_ni_relief row (source lost) retires onto this hmrc-backed fact - vintage difference for the microcosm#622 parity gate." }, "axiom": { "metric_name": "hmrc/salary_sacrifice_employee_nics_relief", @@ -3403,7 +3403,7 @@ "output_variable": "ni_employer", "from_entity": "person", "map_to": "household", - "notes": "NICs relief attributed to salary sacrifice (compute/income.py:59-68). uk-data's SPP-Review-based obr/salary_sacrifice_employer_ni_relief row (source lost) retires onto this hmrc-backed fact - vintage difference for the microcosm#622 parity gate." + "notes": "NICs relief attributed to salary sacrifice (uk-data targets/compute/income.py:59-68 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). uk-data's SPP-Review-based obr/salary_sacrifice_employer_ni_relief row (source lost) retires onto this hmrc-backed fact - vintage difference for the microcosm#622 parity gate." }, "axiom": { "metric_name": "hmrc/salary_sacrifice_employer_nics_relief", @@ -3450,7 +3450,7 @@ "from_entity": "person", "map_to": "household", "threshold_price_base_year": 2023, - "notes": "The GBP 2,000 cap is defined at FRS base-year (2023) prices; deflate salary-sacrifice amounts before banding (compute/income.py:71-95)." + "notes": "The GBP 2,000 cap is defined at FRS base-year (2023) prices; deflate salary-sacrifice amounts before banding (uk-data targets/compute/income.py:71-95 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "obr/salary_sacrifice_users_below_cap", @@ -3493,7 +3493,7 @@ "from_entity": "person", "map_to": "household", "threshold_price_base_year": 2023, - "notes": "The GBP 2,000 cap is defined at FRS base-year (2023) prices; deflate salary-sacrifice amounts before banding (compute/income.py:71-95)." + "notes": "The GBP 2,000 cap is defined at FRS base-year (2023) prices; deflate salary-sacrifice amounts before banding (uk-data targets/compute/income.py:71-95 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "obr/salary_sacrifice_users_above_cap", @@ -3538,7 +3538,7 @@ "map_to": "household", "groupby_variable": "total_income_after_loss_relief_annual", "band_filter_dimension": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20).", + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3).", "filters": [ { "variable": "income_tax_before_winter_fuel_payment_charge_annual", @@ -3652,7 +3652,7 @@ "map_to": "household", "groupby_variable": "total_income_after_loss_relief_annual", "band_filter_dimension": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20). HMRC counts the profit after losses brought forward, which the spine keeps apart in loss_relief, so the value and the band read it after that relief (microcosm#1095).", + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). HMRC counts the profit after losses brought forward, which the spine keeps apart in loss_relief, so the value and the band read it after that relief (microcosm#1095).", "filters": [ { "variable": "income_tax_before_winter_fuel_payment_charge_annual", @@ -3766,7 +3766,7 @@ "map_to": "household", "groupby_variable": "total_income_after_loss_relief_annual", "band_filter_dimension": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20).", + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3).", "filters": [ { "variable": "income_tax_before_winter_fuel_payment_charge_annual", @@ -3880,7 +3880,7 @@ "map_to": "household", "groupby_variable": "total_income_after_loss_relief_annual", "band_filter_dimension": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20). SPI Table 3.7 net income from property is landlords' profit after allowable expenses and before residential finance costs, the concept property_income holds, so the facts bind unscaled. The incumbent's x1.9 adjustment set HMRC Property Rental Income Statistics receipts, which are before expenses, against this net amount and is retired (microcosm#1106, ruled 2026-10-05); the receipts are constructed on property_rental_income by the property_components stage, and PRIS stays a diagnostic (ruled 2026-10-08). Uprated to 2025 by total_income_growth_by_total_income_band, the index the paired count rows' taxpayer_count_growth_by_total_income_band belongs with (microcosm#1063): the flat obr.per_capita.gdp, which policyengine-uk gives property_income itself, would have implied per-band mean profit moves of about -13% (GBP 100,000 to 150,000) to +11% (GBP 15,000 to 20,000) between 2023 and 2025 against those counts, where the band index keeps the pairing within about 2.5%, the GBP 1,000,000 and over band at +2.51% (ruled 2026-10-11 on the microcosm#1145 review). The dividend amount rows keep the flat index.", + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). SPI Table 3.7 net income from property is landlords' profit after allowable expenses and before residential finance costs, the concept property_income holds, so the facts bind unscaled. The incumbent's x1.9 adjustment set HMRC Property Rental Income Statistics receipts, which are before expenses, against this net amount and is retired (microcosm#1106, ruled 2026-10-05); the receipts are constructed on property_rental_income by the property_components stage, and PRIS stays a diagnostic (ruled 2026-10-08). Uprated to 2025 by total_income_growth_by_total_income_band, the index the paired count rows' taxpayer_count_growth_by_total_income_band belongs with (microcosm#1063): the flat obr.per_capita.gdp, which policyengine-uk gives property_income itself, would have implied per-band mean profit moves of about -13% (GBP 100,000 to 150,000) to +11% (GBP 15,000 to 20,000) between 2023 and 2025 against those counts, where the band index keeps the pairing within about 2.5%, the GBP 1,000,000 and over band at +2.51% (ruled 2026-10-11 on the microcosm#1145 review). The dividend amount rows keep the flat index.", "filters": [ { "variable": "income_tax_before_winter_fuel_payment_charge_annual", @@ -3994,7 +3994,7 @@ "map_to": "household", "groupby_variable": "total_income_after_loss_relief_annual", "band_filter_dimension": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20).", + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3).", "filters": [ { "variable": "income_tax_before_winter_fuel_payment_charge_annual", @@ -4108,7 +4108,7 @@ "map_to": "household", "groupby_variable": "total_income_after_loss_relief_annual", "band_filter_dimension": "total_income", - "notes": "Banded by person-level total income; band edges come from the fact dimension (compute/income.py:8-20).", + "notes": "Banded by person-level total income; band edges come from the fact dimension (uk-data targets/compute/income.py:8-20 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3).", "filters": [ { "variable": "income_tax_before_winter_fuel_payment_charge_annual", @@ -4335,7 +4335,7 @@ "value": 0 } ], - "notes": "GB benefit units whose Universal Credit award loses an amount to the benefit cap (uc_benefit_cap_reduction_after_floor above zero), against Benefit cap tables to November 2025, Table 4, GB total row (point-in-time UC households capped at November 2025). DWP's 'UC household' is the unit of assessment, so the total binds at benefit-unit grain on the same reduction as the amount bands, which partition it (microcosm#1349); it was a 0/1 household indicator on benefit_cap_reduction, the incumbent's measure (compute/benefits.py:14-21), which also counts the Housing Benefit cap. The publisher's GBP 320.9m total-reduction companion is a computed annualization - excluded per the facts-only ADR. Scoped to Great Britain (microcosm#1095, uk-data#490): the DWP benefit cap statistics cover Great Britain (K03000001), so the model counts Great Britain benefit units only.", + "notes": "GB benefit units whose Universal Credit award loses an amount to the benefit cap (uc_benefit_cap_reduction_after_floor above zero), against Benefit cap tables to November 2025, Table 4, GB total row (point-in-time UC households capped at November 2025). DWP's 'UC household' is the unit of assessment, so the total binds at benefit-unit grain on the same reduction as the amount bands, which partition it (microcosm#1349); it was a 0/1 household indicator on benefit_cap_reduction, the incumbent's measure (uk-data targets/compute/benefits.py:14-21 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3), which also counts the Housing Benefit cap. The publisher's GBP 320.9m total-reduction companion is a computed annualization - excluded per the facts-only ADR. Scoped to Great Britain (microcosm#1095, uk-data#490): the DWP benefit cap statistics cover Great Britain (K03000001), so the model counts Great Britain benefit units only.", "household_conditions": [ { "variable": "region", @@ -5851,7 +5851,7 @@ "band_period_factor": 12, "band_semantics": "whole_pence_upper_closed", "band_upper_bound": 2500, - "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (compute/benefits.py:84-98). Each band starts a penny above its predecessor's top, so it reads as the upper-closed interval (label lower - GBP 0.01, next label lower - GBP 0.01] x 12, compared in whole pence (band_semantics whole_pence_upper_closed), and an award of a penny a month lands in the bottom band (microcosm#1095). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The separately published £2,500.01 or over band stays unbound, and band_upper_bound closes the last bound band at £2,500.00 a month, so it never absorbs that category. Bound to DWP's counts, the top band left OBR's Universal Credit total 5.6% short on the measured head (experiments/1095-uk-data-ports-3-receipts.md), so it waits until the awards below it are understood. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", + "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (uk-data targets/compute/benefits.py:84-98 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Each band starts a penny above its predecessor's top, so it reads as the upper-closed interval (label lower - GBP 0.01, next label lower - GBP 0.01] x 12, compared in whole pence (band_semantics whole_pence_upper_closed), and an award of a penny a month lands in the bottom band (microcosm#1095). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The separately published £2,500.01 or over band stays unbound, and band_upper_bound closes the last bound band at £2,500.00 a month, so it never absorbs that category. Bound to DWP's counts, the top band left OBR's Universal Credit total 5.6% short on the measured head (experiments/1095-uk-data-ports-3-receipts.md), so it waits until the awards below it are understood. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", "household_conditions": [ { "variable": "region", @@ -5943,7 +5943,7 @@ "band_period_factor": 12, "band_semantics": "whole_pence_upper_closed", "band_upper_bound": 2500, - "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (compute/benefits.py:84-98). Each band starts a penny above its predecessor's top, so it reads as the upper-closed interval (label lower - GBP 0.01, next label lower - GBP 0.01] x 12, compared in whole pence (band_semantics whole_pence_upper_closed), and an award of a penny a month lands in the bottom band (microcosm#1095). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The separately published £2,500.01 or over band stays unbound, and band_upper_bound closes the last bound band at £2,500.00 a month, so it never absorbs that category. Bound to DWP's counts, the top band left OBR's Universal Credit total 5.6% short on the measured head (experiments/1095-uk-data-ports-3-receipts.md), so it waits until the awards below it are understood. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", + "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (uk-data targets/compute/benefits.py:84-98 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Each band starts a penny above its predecessor's top, so it reads as the upper-closed interval (label lower - GBP 0.01, next label lower - GBP 0.01] x 12, compared in whole pence (band_semantics whole_pence_upper_closed), and an award of a penny a month lands in the bottom band (microcosm#1095). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The separately published £2,500.01 or over band stays unbound, and band_upper_bound closes the last bound band at £2,500.00 a month, so it never absorbs that category. Bound to DWP's counts, the top band left OBR's Universal Credit total 5.6% short on the measured head (experiments/1095-uk-data-ports-3-receipts.md), so it waits until the awards below it are understood. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", "household_conditions": [ { "variable": "region", @@ -6035,7 +6035,7 @@ "band_period_factor": 12, "band_semantics": "whole_pence_upper_closed", "band_upper_bound": 2500, - "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (compute/benefits.py:84-98). Each band starts a penny above its predecessor's top, so it reads as the upper-closed interval (label lower - GBP 0.01, next label lower - GBP 0.01] x 12, compared in whole pence (band_semantics whole_pence_upper_closed), and an award of a penny a month lands in the bottom band (microcosm#1095). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The separately published £2,500.01 or over band stays unbound, and band_upper_bound closes the last bound band at £2,500.00 a month, so it never absorbs that category. Bound to DWP's counts, the top band left OBR's Universal Credit total 5.6% short on the measured head (experiments/1095-uk-data-ports-3-receipts.md), so it waits until the awards below it are understood. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", + "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (uk-data targets/compute/benefits.py:84-98 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Each band starts a penny above its predecessor's top, so it reads as the upper-closed interval (label lower - GBP 0.01, next label lower - GBP 0.01] x 12, compared in whole pence (band_semantics whole_pence_upper_closed), and an award of a penny a month lands in the bottom band (microcosm#1095). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The separately published £2,500.01 or over band stays unbound, and band_upper_bound closes the last bound band at £2,500.00 a month, so it never absorbs that category. Bound to DWP's counts, the top band left OBR's Universal Credit total 5.6% short on the measured head (experiments/1095-uk-data-ports-3-receipts.md), so it waits until the awards below it are understood. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", "household_conditions": [ { "variable": "region", @@ -6127,7 +6127,7 @@ "band_period_factor": 12, "band_semantics": "whole_pence_upper_closed", "band_upper_bound": 2500, - "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (compute/benefits.py:84-98). Each band starts a penny above its predecessor's top, so it reads as the upper-closed interval (label lower - GBP 0.01, next label lower - GBP 0.01] x 12, compared in whole pence (band_semantics whole_pence_upper_closed), and an award of a penny a month lands in the bottom band (microcosm#1095). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The separately published £2,500.01 or over band stays unbound, and band_upper_bound closes the last bound band at £2,500.00 a month, so it never absorbs that category. Bound to DWP's counts, the top band left OBR's Universal Credit total 5.6% short on the measured head (experiments/1095-uk-data-ports-3-receipts.md), so it waits until the awards below it are understood. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", + "notes": "Benunit counts by annual UC award band: publisher bands are monthly award amounts, multiplied by 12 to band annual model UC (uk-data targets/compute/benefits.py:84-98 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Each band starts a penny above its predecessor's top, so it reads as the upper-closed interval (label lower - GBP 0.01, next label lower - GBP 0.01] x 12, compared in whole pence (band_semantics whole_pence_upper_closed), and an award of a penny a month lands in the bottom band (microcosm#1095). Administrative family classification uses model standard allowance above the model-year maximum single rate for couples, separately from retained claimant roles; zero allowance is UNKNOWN. Child presence retains own-benefit-unit reported-under-20/qualifying-child proxy and does not establish DWP declared-child attachment or ineligible-partner exceptions. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Family%20Type.html The separately published £2,500.01 or over band stays unbound, and band_upper_bound closes the last bound band at £2,500.00 a month, so it never absorbs that category. Bound to DWP's counts, the top band left OBR's Universal Credit total 5.6% short on the measured head (experiments/1095-uk-data-ports-3-receipts.md), so it waits until the awards below it are understood. Source: https://stat-xplore.dwp.gov.uk/webapi/metadata/UC_Households/Monthly%20Award%20Amount%20(bands).html Count only the Great Britain regions, matching K03000001; DWP removes Northern Ireland-office claims from these statistics. Source: https://www.gov.uk/government/publications/universal-credit-statistics-background-information-and-methodology/universal-credit-statistics-background-information-and-methodology#creating-a-great-britain-dataset", "household_conditions": [ { "variable": "region", @@ -12046,7 +12046,7 @@ } ], "value_variable": "household_count", - "notes": "Household indicator (compute/demographics.py:60-77)." + "notes": "Household indicator (uk-data targets/compute/demographics.py:60-77 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/scotland_households_3plus_children", @@ -12466,7 +12466,7 @@ } ], "value_variable": "household_count", - "notes": "Bound on the frame column household.ons_household_type == 'lone_parent_non_dependent_children_households', written by the frs_relationships stage (microcosm#791) from the FRS household grid (relhrp, R01-R14); the value is Chronicle's ons.household_type dimension value id, so the contract row, the frame and the published fact share one string and no engine variable is involved. ONS dependency is under 16, or 16-18 in full-time education (the engine's is_child, age under 18, is not used). Exactly one family, a lone parent whose children in the household are all non-dependent. The summed-age head-age proxy inherited from uk-data compute/households.py:45-52 is retired; the relationship grid makes the cell exact." + "notes": "Bound on the frame column household.ons_household_type == 'lone_parent_non_dependent_children_households', written by the frs_relationships stage (microcosm#791) from the FRS household grid (relhrp, R01-R14); the value is Chronicle's ons.household_type dimension value id, so the contract row, the frame and the published fact share one string and no engine variable is involved. ONS dependency is under 16, or 16-18 in full-time education (the engine's is_child, age under 18, is not used). Exactly one family, a lone parent whose children in the household are all non-dependent. The summed-age head-age proxy inherited from uk-data targets/compute/households.py:45-52 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3 is retired; the relationship grid makes the cell exact." }, "axiom": { "metric_name": "ons/lone_parent_non_dependent_children_households", @@ -14711,7 +14711,7 @@ "value_expression": "savings_interest_income + tax_free_savings_income", "from_entity": "person", "map_to": "household", - "notes": "uk-data holds the last actual flat for later years - a populace-side declaration (sources/ons_savings.py:39-44). The row measures cash interest in and outside tax-free accounts. From policyengine-uk 2.162.2 savings_interest_income is the interest outside them: the engine takes the frame's tax_free_savings_income out of the frame's gross interest when it loads the frame. The ISA part, tax_free_savings_income, is added back (microcosm#1095)." + "notes": "uk-data holds the last actual flat for later years - a populace-side declaration (uk-data targets/sources/ons_savings.py:39-44 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). The row measures cash interest in and outside tax-free accounts. From policyengine-uk 2.162.2 savings_interest_income is the interest outside them: the engine takes the frame's tax_free_savings_income out of the frame's gross interest when it loads the frame. The ISA part, tax_free_savings_income, is added back (microcosm#1095)." }, "axiom": { "metric_name": "ons/savings_interest_income", @@ -14794,7 +14794,7 @@ "policyengine": { "metric_name": "ons/land_value", "value_variable": "land_value", - "notes": "National balance sheet land series for the total_economy sector; uk-data carries the last estimate flat to later years (populace-side declaration, sources/_land.py:52-73)." + "notes": "National balance sheet land series for the total_economy sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:48-52 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/land_value", @@ -14828,7 +14828,7 @@ "policyengine": { "metric_name": "ons/household_land_value", "value_variable": "household_land_value", - "notes": "National balance sheet land series for the households_and_npish sector; uk-data carries the last estimate flat to later years (populace-side declaration, sources/_land.py:52-73)." + "notes": "National balance sheet land series for the households_and_npish sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:54-59 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/household_land_value", @@ -14862,7 +14862,7 @@ "policyengine": { "metric_name": "ons/corporate_land_value", "value_variable": "corporate_land_value", - "notes": "National balance sheet land series for the non_financial_corporations sector; uk-data carries the last estimate flat to later years (populace-side declaration, sources/_land.py:52-73)." + "notes": "National balance sheet land series for the non_financial_corporations sector; uk-data carries the last estimate flat to later years (populace-side declaration, uk-data targets/sources/_land.py:60-65 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "ons/corporate_land_value", @@ -14899,7 +14899,7 @@ "value_variable": "student_loan_repayment", "from_entity": "person", "map_to": "household", - "notes": "One fact per devolved country (Scotland, Wales, Northern Ireland); the resolved geography binds the model country filter (compute/other.py:116-137)." + "notes": "One fact per devolved country (Scotland, Wales, Northern Ireland); the resolved geography binds the model country filter (uk-data targets/compute/other.py:116-137 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/student_loan_repayment/devolved", @@ -15064,7 +15064,7 @@ "plan_2_full_time", "plan_2_part_time" ], - "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (compute/other.py:116-137)." + "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (uk-data targets/compute/other.py:116-137 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/student_loan_repayment/england/plan_2", @@ -15125,7 +15125,7 @@ "plan_5_full_time", "plan_5_part_time" ], - "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (compute/other.py:116-137)." + "notes": "Publisher splits full-time/part-time (and masters/doctoral) products the model does not distinguish; the listed source lines sum to this target (uk-data targets/compute/other.py:116-137 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3)." }, "axiom": { "metric_name": "slc/student_loan_repayment/england/plan_5", @@ -15177,7 +15177,7 @@ "value_variable": "person_count", "from_entity": "person", "map_to": "household", - "notes": "England borrowers on this plan earning above the repayment threshold (proxied by positive repayments, matching what the FRS captures via PAYE; compute/other.py:77-96). Bound at SLC academic year 2024/25, whose reference date falls inside calendar 2025, the stock the student_loans stage tops up to; Chronicle stores the academic year under its opening year, 2024 (María's ruling of 2026-10-16, microcosm#868)." + "notes": "England borrowers on this plan earning above the repayment threshold (proxied by positive repayments, matching what the FRS captures via PAYE; uk-data targets/compute/other.py:77-96 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at SLC academic year 2024/25, whose reference date falls inside calendar 2025, the stock the student_loans stage tops up to; Chronicle stores the academic year under its opening year, 2024 (María's ruling of 2026-10-16, microcosm#868)." }, "axiom": { "metric_name": "slc/plan_2_borrowers_above_threshold", @@ -15227,7 +15227,7 @@ "value_variable": "person_count", "from_entity": "person", "map_to": "household", - "notes": "England borrowers on this plan liable to repay, including below-threshold holders (compute/other.py:99-113). Bound at SLC academic year 2024/25, whose reference date falls inside calendar 2025, the stock the student_loans stage tops up to; Chronicle stores the academic year under its opening year, 2024 (María's ruling of 2026-10-16, microcosm#868)." + "notes": "England borrowers on this plan liable to repay, including below-threshold holders (uk-data targets/compute/other.py:99-113 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at SLC academic year 2024/25, whose reference date falls inside calendar 2025, the stock the student_loans stage tops up to; Chronicle stores the academic year under its opening year, 2024 (María's ruling of 2026-10-16, microcosm#868)." }, "axiom": { "metric_name": "slc/plan_2_borrowers_liable", @@ -15282,7 +15282,7 @@ "value_variable": "person_count", "from_entity": "person", "map_to": "household", - "notes": "England borrowers on this plan earning above the repayment threshold (proxied by positive repayments, matching what the FRS captures via PAYE; compute/other.py:77-96). Bound at SLC academic year 2024/25, whose reference date falls inside calendar 2025, the stock the student_loans stage tops up to; Chronicle stores the academic year under its opening year, 2024 (María's ruling of 2026-10-16, microcosm#868)." + "notes": "England borrowers on this plan earning above the repayment threshold (proxied by positive repayments, matching what the FRS captures via PAYE; uk-data targets/compute/other.py:77-96 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at SLC academic year 2024/25, whose reference date falls inside calendar 2025, the stock the student_loans stage tops up to; Chronicle stores the academic year under its opening year, 2024 (María's ruling of 2026-10-16, microcosm#868)." }, "axiom": { "metric_name": "slc/plan_5_borrowers_above_threshold", @@ -15332,7 +15332,7 @@ "value_variable": "person_count", "from_entity": "person", "map_to": "household", - "notes": "England borrowers on this plan liable to repay, including below-threshold holders (compute/other.py:99-113). Bound at SLC academic year 2024/25, whose reference date falls inside calendar 2025, the stock the student_loans stage tops up to; Chronicle stores the academic year under its opening year, 2024 (María's ruling of 2026-10-16, microcosm#868)." + "notes": "England borrowers on this plan liable to repay, including below-threshold holders (uk-data targets/compute/other.py:99-113 at 12a1e028afeef08d8b2d74ee03fd9de3a78b2dd3). Bound at SLC academic year 2024/25, whose reference date falls inside calendar 2025, the stock the student_loans stage tops up to; Chronicle stores the academic year under its opening year, 2024 (María's ruling of 2026-10-16, microcosm#868)." }, "axiom": { "metric_name": "slc/plan_5_borrowers_liable", diff --git a/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py b/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py index 57724d74e7..f834985a8e 100644 --- a/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py +++ b/packages/microcosm-build/src/microcosm/build/uk_runtime/data_target_parity.py @@ -188,14 +188,14 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "national_obr_national_insurance", "status": "ported_national", "classification": "red_line_national_family", - "evidence": "uk-data targets/sources/obr.py:366 (_parse_nics); Microcosm target ids obr.ni_employee, obr.ni_employer, and obr.ni_self_employed; OBR's cash total obr.ni is a diagnostic on obr.ni_employee (microcosm#1095).", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:366 (_parse_nics); Microcosm target ids obr.ni_employee, obr.ni_employer, and obr.ni_self_employed; OBR's cash total obr.ni is a diagnostic on obr.ni_employee (microcosm#1095).", }, { "concern_id": "national_obr_national_insurance_class_3", "status": "reviewed_exclusion", "classification": "not_modeled", "reason": "The incumbent deliberately skipped Class 3 and no Microcosm dataset path exists.", - "evidence": "uk-data targets/sources/obr.py:420-427 (the Class 3 note in _parse_nics) and uk-data#88.", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:420-427 (the Class 3 note in _parse_nics) and uk-data#88.", "fence": _fence( "uk-data skipped the Class 3 OBR row pending uk-data#88.", "Avoid pretending voluntary Class 3 contributions are represented by another NIC variable.", @@ -206,31 +206,31 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "national_obr_council_tax", "status": "ported_national", "classification": "red_line_national_family", - "evidence": "uk-data targets/sources/obr.py:302 (_parse_council_tax); Microcosm target ids obr.council_tax (OBR table 4.1 row 15, England, Scotland and Wales receipts over Great Britain's households, microcosm#1095) and country legs obr.council_tax_{england,scotland,wales}.", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:302 (_parse_council_tax); Microcosm target ids obr.council_tax (OBR table 4.1 row 15, England, Scotland and Wales receipts over Great Britain's households, microcosm#1095) and country legs obr.council_tax_{england,scotland,wales}.", }, { "concern_id": "national_obr_welfare", "status": "ported_national", "classification": "national_registry_family", - "evidence": "uk-data targets/sources/obr.py:502 (_parse_welfare) welfare rows; Microcosm uk_population_targets.json OBR benefit-spending targets. Since uk-data a43e8c5 the Housing Benefit and Pension Credit lines come from DWP's benefit expenditure and caseload tables (targets/sources/dwp_housing_benefit.py, uk-data#490; dwp_pension_credit.py, uk-data#510) and Universal Credit is one Great Britain total (uk-data#530).", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:502 (_parse_welfare) welfare rows; Microcosm uk_population_targets.json OBR benefit-spending targets. Since uk-data a43e8c5 the Housing Benefit and Pension Credit lines come from DWP's benefit expenditure and caseload tables (targets/sources/dwp_housing_benefit.py, uk-data#490; dwp_pension_credit.py, uk-data#510) and Universal Credit is one Great Britain total (uk-data#530).", }, { "concern_id": "national_tv_licence_and_policy_statics", "status": "ported_national", "classification": "national_registry_family", - "evidence": "uk-data targets/sources/obr.py:620 (_parse_tv_licence) and 668-684 (the private-school and salary-sacrifice static rows before get_targets); Microcosm uk_population_targets.json tv_licence, private-school, and salary-sacrifice target rows.", + "evidence": "uk-data at a43e8c5 targets/sources/obr.py:620 (_parse_tv_licence) and 668-684 (the private-school and salary-sacrifice static rows before get_targets); Microcosm uk_population_targets.json tv_licence, private-school, and salary-sacrifice target rows.", }, { "concern_id": "national_hmrc_spi_income_bands", "status": "ported_national", "classification": "national_registry_family", - "evidence": "uk-data targets/sources/hmrc_spi.py:230; Microcosm HMRC SPI amount/count-by-total-income-band targets and replay fences.", + "evidence": "uk-data at a43e8c5 targets/sources/hmrc_spi.py:229 (get_targets); Microcosm HMRC SPI amount/count-by-total-income-band targets and replay fences.", }, { "concern_id": "national_hmrc_cgt_size_bands", "status": "ported_national", "classification": "national_registry_family", - "evidence": "uk-data targets/sources/hmrc_cgt.py:195; Microcosm hmrc_cgt_conditioning_facts.json (vendored HMRC Table 2.1a 2024-25 rows read by the donor stage and the amounts redraw) and uk_population_targets.json CGT rows.", + "evidence": "uk-data at a43e8c5 targets/sources/hmrc_cgt.py:196 (_band_targets); Microcosm hmrc_cgt_conditioning_facts.json (vendored HMRC Table 2.1a 2024-25 rows read by the donor stage and the amounts redraw) and uk_population_targets.json CGT rows.", }, { "concern_id": "national_hmrc_salary_sacrifice", @@ -305,7 +305,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "status": "routed", "classification": "national_grain_follow_up", "reason": "The incumbent England-national tenure controls are not part of the current national contract; local tenure is declared separately.", - "evidence": "uk-data targets/sources/ons_tenure.py:69; Microcosm local ons.tenure.* contract rows.", + "evidence": "uk-data at a43e8c5 targets/sources/ons_tenure.py:69 (get_targets); Microcosm local ons.tenure.* contract rows.", "fence": _fence( "uk-data calibrated five England tenure controls.", "Preserve an England-level representation check independently of the local Census tenure surface.", @@ -358,7 +358,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "national_council_tax_stock", "status": "ported_national", "classification": "red_line_national_family", - "evidence": "uk-data targets/sources/voa_council_tax.py:235,275; Microcosm mhclg.council_tax_stock.* (England, composed from the MHCLG taxbase authority rows per region), welshgov.council_tax_stock.* and scotgov.council_tax_stock.* targets on the councils' taxbase basis (microcosm#929).", + "evidence": "uk-data at a43e8c5 targets/sources/voa_council_tax.py:235,275; Microcosm mhclg.council_tax_stock.* (England, composed from the MHCLG taxbase authority rows per region), welshgov.council_tax_stock.* and scotgov.council_tax_stock.* targets on the councils' taxbase basis (microcosm#929).", }, { "concern_id": "national_housing_rate_headcount_products", @@ -400,7 +400,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "concern_id": "local_council_tax_band_counts", "status": "ported_local_declared", "classification": "local_registry_family", - "evidence": "uk-data targets/sources/la_council_tax.py and datasets/local_areas/local_authorities/loss.py:251-300; Microcosm council_tax/band_a..i contract rows (mhclg, welshgov and scotgov by_area families) bind 2,511 authority cells on the taxbase basis with the 296 English band-H cells and Shetland's band-H cell signed deferred (microcosm#929).", + "evidence": "uk-data at a43e8c5 targets/sources/la_council_tax.py and datasets/local_areas/local_authorities/loss.py:251-300; Microcosm council_tax/band_a..i contract rows (mhclg, welshgov and scotgov by_area families) bind 2,511 authority cells on the taxbase basis with the 296 English band-H cells and Shetland's band-H cell signed deferred (microcosm#929).", }, { "concern_id": "local_private_rent_pipr", @@ -468,7 +468,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "status": "routed", "classification": "imputation_anchor", "reason": "The £21.6bn rail anchor belongs to WS-E/F (#148), not the local target solve.", - "evidence": "uk-data datasets/imputations/services/services.py:18; Chronicle transport facts landed in #200/#202.", + "evidence": "uk-data at a43e8c5 datasets/imputations/services/services.py:18; Chronicle transport facts landed in #200/#202.", "fence": _fence( "uk-data rescaled rail_usage to a national subsidy total.", "Anchor the imputed service-consumption mass.", @@ -480,7 +480,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "status": "routed", "classification": "imputation_anchor", "reason": "BUS05 subsidy/fare totals are consumed by the #789 rows; the NTS0705 quintile gradient remains routed to WS-E/F (#148).", - "evidence": "uk-data datasets/imputations/services/services.py:33 and datasets/imputations/consumption.py:790,830; Chronicle#200/#202.", + "evidence": "uk-data at a43e8c5 datasets/imputations/services/services.py:33 and datasets/imputations/consumption.py:790,830; Chronicle#200/#202.", "fence": _fence( "uk-data post-imputation-rescaled bus subsidy and fares, including London share and quintile rates.", "Preserve service-consumption totals and distribution.", @@ -492,7 +492,7 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "status": "reviewed_exclusion", "classification": "untargeted_both_sides", "reason": "Education spending was imputed but its aggregate test was commented out; corrected post-ebf733c DfE entitlement data from uk-data#474 remains input-side.", - "evidence": "uk-data datasets/imputations/frs_only.py and uk-data tests/test_aggregates.py:5; uk-data#199 and uk-data#474.", + "evidence": "uk-data at a43e8c5 datasets/imputations/services/services.py:137-148 (the dfe_education_spending imputation) and tests/test_aggregates.py:5 (its commented-out aggregate); uk-data#199 and uk-data#474.", "fence": _fence( "uk-data imputed DfE education spending without an active calibration target.", "Retain the imputed input while avoiding an unsupported aggregate claim.", diff --git a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json index cd769fa333..cc7ca73dd0 100644 --- a/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json +++ b/packages/microcosm-build/src/microcosm/build/us/ecps_parity_known_gaps.json @@ -77,7 +77,7 @@ } ], "aggregate_identity": "For every locked artifact, HFINVAL equals the household sum of FFINVAL; neither aggregate identifies a recipient person.", - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; tools/build_us_puf_support_base.py lines 85-88 and 659-699 consumes only those mappings; packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 61-75 reads the prebuilt person/household tables without refreshing omitted Census columns." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 records their SHA-256 digests; tools/build_us_puf_support_base.py _parse_args (--asec-h5) and _pooled_asec_sources_from_args consume those mappings, and _asec_sources_from_args binds each year to its pinned Census person file; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads the prebuilt person/household tables, and microcosm.build.us_runtime.asec_pool._prepare_year_input restores only microcosm.build.us_runtime.asec_census_person_columns.ASEC_CENSUS_PERSON_COLUMNS, which excludes FIN_VAL, FIN_YN, and I_FINVAL." } }, "is_unmarried_partner_of_household_head": { @@ -132,10 +132,10 @@ "semantic_non_substitutes": { "A_EXPRRP": "Of Build J's three HDF inputs only 2024 carries it (the pinned income-year 2025 input does too), but since microcosm #720 microcosm.build.us_runtime.asec_pool._prepare_year_input restores it for 2022 and 2023 from their pinned Census person files, so every vintage the two pooling tools pool carries the Census recode: code 13 on 4,312, 4,387, 4,257, and 3,816 unweighted person records in 2022, 2023, 2024, and 2025. Code 13 is the combined CPSRelationshipCode.PARTNER_OR_ROOMMATE category, not the PERRP unmarried-partner codes the retired derivation reads: in the pinned 2022-2024 Census person files it holds PERRP 44 and 47 (unmarried partners without relatives) and 55 (housemates or roommates without relatives: 1,500, 1,601, and 1,614 records in 2022, 2023, and 2024), while PERRP 43 and 46 (unmarried partners with relatives) fall in code 12 (nonrelatives with their own relatives in the household).", "PECOHAB": "Only the 2024 locked input carries the cohabiting-partner line pointer; it cannot recover partner status for the equally weighted 2022 and 2023 source years.", - "derived_A_EXPRRP": "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 397-422 derives older-year relationship codes from line, spouse, sex, and parent pointers and never emits code 13.", + "derived_A_EXPRRP": "microcosm.build.us_runtime.asec_pool._with_relationship_recode derives relationship codes from line, spouse, sex, and parent pointers only for a person table without A_EXPRRP, such as the raw 2022 and 2023 H5 inputs, and that fallback never emits code 13; since microcosm #720 the pooling tools bind each year's pinned Census person file and microcosm.build.us_runtime.asec_pool._prepare_year_input restores Census A_EXPRRP, whose reviewed domain includes code 13, before the recode, so pooled 2022 and 2023 persons no longer take the fallback.", "rejection": "Using only the 2024 alternatives and filling the equally weighted 2022-2023 source years as false, or splitting the combined partner/roommate category, would synthesize person statuses absent from the locked inputs." }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests and equal source-year shares; packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 61-75 reads those tables without refreshing omitted Census columns." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 and base_source.metadata.sources[].share record their SHA-256 digests and equal source-year shares; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads those tables, and since microcosm #720 microcosm.build.us_runtime.asec_pool._prepare_year_input restores the 2022 and 2023 inputs' missing A_EXPRRP from pinned Census person files but not PERRP, PECOHAB, or A_FAMREL. Build J predates #720: base_j.summary.json base_source.metadata.sources[].relationship_recode_source records derived:line_spouse_parent for 2022 and 2023 and source:A_EXPRRP for 2024." } }, "is_wic_at_nutritional_risk": { @@ -242,7 +242,7 @@ "retired_bernoulli_draw": "The early-1980s category priors are aggregate eligibility-estimation factors, not person-level assessment observations.", "rejection": "Assigning False to nonrecipients, allocating an SPM amount to a person, or replaying the retired Bernoulli draw would synthesize individual nutritional-risk assessments absent from the hermetic sources." }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 61-75 reads only their person and household tables without refreshing or linking separate WIC certification records." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 records their SHA-256 digests; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads only their person and household tables without refreshing or linking separate WIC certification records." } }, "survivor_benefits": { @@ -313,7 +313,7 @@ ], "aggregate_identity": "For every locked artifact, HSURVAL equals the household sum of FSURVAL; neither aggregate identifies the recipient person.", "semantic_rejection": "The retired derivation requires a reported person amount. Its CPS-only QRF cannot train without that leaf, and assigning FSURVAL or HSURVAL to a head, first member, or arbitrary split would synthesize a survivor-benefit observation absent from every locked person table.", - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-69 passes only the three prebuilt 2022-2024 artifacts via --asec-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 records their SHA-256 digests; packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py lines 61-75 reads the prebuilt person tables without refreshing omitted Census columns." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt 2022-2024 artifacts via its --asec-h5 2024=, 2023=, and 2022= arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 records their SHA-256 digests; microcosm.build.us_runtime.asec_pool.load_asec_h5_tables reads the prebuilt person tables, and microcosm.build.us_runtime.asec_pool._prepare_year_input restores only microcosm.build.us_runtime.asec_census_person_columns.ASEC_CENSUS_PERSON_COLUMNS, which excludes SRVS_VAL." } }, "takes_up_dc_ptc": { @@ -412,7 +412,7 @@ "PUF_other_credits": "Archived puf.py maps federal P08000 to other_credits; the locked PUF lacks state geography and cannot identify a DC credit claimant.", "rejection": "A published aggregate number of claims supplies no tax-unit claimant identity. Replaying a seeded draw, sorting eligible DC units, or otherwise allocating the count would synthesize the boolean values the locked microdata do not report." }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 70-81 and 103-108 passes only the three prebuilt ASEC artifacts and processed PUF via --asec-h5/--puf-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 and 317-318 records their SHA-256 digests." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt ASEC artifacts and processed PUF via its --asec-h5 2024=, 2023=, and 2022= and --puf-h5 arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 and puf_sha256 record their SHA-256 digests." } }, "takes_up_early_head_start_if_eligible": { @@ -485,7 +485,7 @@ "funded_enrollment": "Capacity or federally supported slots, not observed occupancy or person identity.", "source": "https://headstart.gov/sites/default/files/pdf/service-snapshot-ehs-2023-2024.pdf" }, - "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh lines 65-81 passes only the three prebuilt ASEC artifacts and processed PUF via --asec-h5/--puf-h5; experiments/build_j_recert/base_j.summary.json lines 55-75 and 317-318 records their SHA-256 digests. The separately pinned SIPP donor does not observe the required Early Head Start domain." + "hermetic_build_contract": "experiments/build_j_recert/buildj_base.sh passes only the three prebuilt ASEC artifacts and processed PUF via its --asec-h5 2024=, 2023=, and 2022= and --puf-h5 arguments; experiments/build_j_recert/base_j.summary.json base_source.sources[].sha256 and puf_sha256 record their SHA-256 digests. The separately pinned SIPP donor does not observe the required Early Head Start domain." } } } diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 15f72026d4..9acf2709ec 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -725,7 +725,7 @@ "nonnegative_outputs": [ "workers_compensation" ], - "notes": "Exact annual person-level WC_VAL carry from archived cps.py lines 1559-1571; code-1 DIS_VAL slots remain excluded from the separate disability_benefits leaf rather than added here. The CPS-only workers_compensation output is replaced on the PUF support half with the archived eight predictors and at most 5,000 ASEC training people (extended_cps.py lines 135-194, 234-248, 639-745, and 1014-1073). Microcosm pins 100 trees and typed person weights as reproducibility hardening." + "notes": "Exact annual person-level WC_VAL carry from archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1559-1571; code-1 DIS_VAL slots remain excluded from the separate disability_benefits leaf rather than added here. The CPS-only workers_compensation output is replaced on the PUF support half with the archived eight predictors and at most 5,000 ASEC training people (datasets/cps/extended_cps.py lines 135-194, 234-248, 639-745, and 1014-1073). Microcosm pins 100 trees and typed person weights as reproducibility hardening." }, { "stage": "weeks_unemployed_input", @@ -807,7 +807,7 @@ "nonnegative_outputs": [ "weeks_unemployed" ], - "notes": "Restores the final retired eCPS person-year weeks_unemployed input. Archived cps.py lines 1438-1442 carries Census ASEC LKWEEKS directly, mapping only the historical -1 not-in-universe sentinel to zero; the SHA-pinned 2023 ASEC public-use archive supplies the omitted 2022-income-year source rows exactly. Its 146,133 person records match the locked 2022 H5 one-for-one on PH_SEQ, P_SEQ, A_LINENO, A_AGE, A_FNLWGT, UC_VAL, and WKSWORK. The source has 4,200 positive LKWEEKS rows, range 0--51, 3.1456094276% A_FNLWGT-weighted positive support, and 172,563,622.04 weighted person-weeks; Census allocation flags are retained because the retired direct carry did not filter them. Archived calibration/puf_impute.py lines 703-785 replaces only the PUF support half with a QRF using age, sex, three tax-unit roles, joint filing, and unemployment compensation, caps training at 5,000 ASEC people, clips predictions to 0--52, and forces predictions to zero without positive unemployment compensation. Microcosm pins 100 trees and typed person weights as reproducibility hardening, and rounds its interpolating QRF draws back to the source's integer-week domain; no 2022 source value is filled statistically." + "notes": "Restores the final retired eCPS person-year weeks_unemployed input. Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1438-1442 carries Census ASEC LKWEEKS directly, mapping only the historical -1 not-in-universe sentinel to zero; the SHA-pinned 2023 ASEC public-use archive supplies the omitted 2022-income-year source rows exactly. Its 146,133 person records match the locked 2022 H5 one-for-one on PH_SEQ, P_SEQ, A_LINENO, A_AGE, A_FNLWGT, UC_VAL, and WKSWORK. The source has 4,200 positive LKWEEKS rows, range 0--51, 3.1456094276% A_FNLWGT-weighted positive support, and 172,563,622.04 weighted person-weeks; Census allocation flags are retained because the retired direct carry did not filter them. Archived calibration/puf_impute.py lines 703-785 replaces only the PUF support half with a QRF using age, sex, three tax-unit roles, joint filing, and unemployment compensation, caps training at 5,000 ASEC people, clips predictions to 0--52, and forces predictions to zero without positive unemployment compensation. Microcosm pins 100 trees and typed person weights as reproducibility hardening, and rounds its interpolating QRF draws back to the source's integer-week domain; no 2022 source value is filled statistically." }, { "stage": "capital_gain_distributions", @@ -1190,7 +1190,7 @@ "bond_assets", "stock_assets" ], - "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped." + "notes": "Microcosm #374 restores the retired 50/50 SIPP/SCF financial-asset blend: a seeded recipient-household draw selects one complete three-leaf vector, and the reference person carries it while all other people receive zero. The SIPP mapping matches archived commit 42ed5d45's final 13-predictor asset QRF, while the explicitly requested immutable pu2023.csv artifact replaces that commit's later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth QRF anchor; calibration/source_impute.py lines 1146-1164 add that anchor to the QRF outputs, lines 1227-1240 fit it with these eight predictors (SCF_PREDICTORS, lines 168-177), and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor directly, including indebted households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance preserves the retired pipeline's name for original financed amounts, not remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS's approximately six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped." }, { "stage": "ssi_disability_criteria", @@ -1568,7 +1568,7 @@ "outputs": [ "takes_up_ssi_if_eligible" ], - "notes": "Restores the final retired eCPS SSI take-up export from source-backed evidence at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe. datasets/cps/cps.py line 584 maps the direct ASEC SSI_VAL reporter signal, lines 650-657 derive the take-up leaf, and lines 1497-1499 export it; datasets/cps/takeup.py lines 10-35 define the retired stable random assignment. The retired all-age 0.50 rate has no full-population administrative source and is not reused. utils/ssi_targets.py lines 41-74 preserve the archived age-target method and cite SSA SSI Monthly Statistics, December 2024, Table 1, row Total with—Federal payment, by the three published age bands; those recipient counts enter the build only as ledger-fed calibration registry targets (role ssa_ssi_age_band_recipients, microcosm#469 and microcosm#470), never as count-matching goals in this stage. Direct SSI_VAL reporters remain true. Every other source-person identity draws once against the documented band prior — the anchored-mass-corrected count-truthful threshold (band target minus basis reporter floor) over (basis candidate capacity minus basis reporter floor) on weighted uncapped_ssi > 0 candidates, zero once the reporter floor meets the target, falling back to the basis reporter share of capacity when capacity cannot subsample the target — seeded and stable per person_source_id, with the decision fanned identically to both support clones. The capacity basis is this frame's weights by default, or a prior attempt's delivered-weight us_ssi_take_up.json diagnostics (microcosm#507/#508); either way the draw happens exactly once, with no reconcile loop. Delivered recipients on release weights are gated per enforced band (18_64, 65_plus) within a documented relative tolerance; the under-18 band is fenced pending microcosm#453/#509 and its gap ships in the release scorecard." + "notes": "Restores the final retired eCPS SSI take-up export from source-backed evidence at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe. datasets/cps/cps.py lines 1497-1499 map the direct ASEC SSI_VAL reporter signal to the temporary ssi_reported anchor, line 584 loads the retired SSI rate, lines 650-657 derive the take-up leaf from both, and lines 759-766 drop the anchor and save the leaf; datasets/cps/takeup.py lines 10-35 define the retired stable random assignment. The retired all-age 0.50 rate has no full-population administrative source and is not reused. utils/ssi_targets.py lines 41-74 preserve the archived age-target method and cite SSA SSI Monthly Statistics, December 2024, Table 1, row Total with—Federal payment, by the three published age bands; those recipient counts enter the build only as ledger-fed calibration registry targets (role ssa_ssi_age_band_recipients, microcosm#469 and microcosm#470), never as count-matching goals in this stage. Direct SSI_VAL reporters remain true. Every other source-person identity draws once against the documented band prior — the anchored-mass-corrected count-truthful threshold (band target minus basis reporter floor) over (basis candidate capacity minus basis reporter floor) on weighted uncapped_ssi > 0 candidates, zero once the reporter floor meets the target, falling back to the basis reporter share of capacity when capacity cannot subsample the target — seeded and stable per person_source_id, with the decision fanned identically to both support clones. The capacity basis is this frame's weights by default, or a prior attempt's delivered-weight us_ssi_take_up.json diagnostics (microcosm#507/#508); either way the draw happens exactly once, with no reconcile loop. Delivered recipients on release weights are gated per enforced band (18_64, 65_plus) within a documented relative tolerance; the under-18 band is fenced pending microcosm#453/#509 and its gap ships in the release scorecard." }, { "stage": "sipp_tips", @@ -1863,7 +1863,7 @@ "nonnegative_outputs": [ "employer_sponsored_insurance_premiums" ], - "notes": "Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for active and retired employees. Column universe: employed (PEMLR 1-2) policyholders whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed and workers without pay carry 0, and their scaled share stays out of the column rather than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium. Some (2): premium minus the contribution of an employee who pays one, which is the published average contribution (taken over every enrollee, including those who pay nothing) divided by one minus the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder's share so the weighted anchor-universe total equals BEA NIPA Table 7.8 line 17 (B4923C) for the build year ($977.0B CY2024 in the 2026-09-30 vintage, revised from $1,002.9B; banked in the pinned feed as bea_nipa.cy2024.private_group_health_insurance), the anchor decision d1317 chose; the column carries the employed policyholders' part. CMS NHE Table 24 ($1,047.0B CY2024) is a second estimate of the same active-plus-retired concept and is recorded beside the verdict, as is MEPS-IC's own private-sector enrollment total for active employees, which the private part of the column matches within half a percent. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent of BEA NIPA 7.8 line 17; employed share within its band)." + "notes": "Restores the retired eCPS employer-premium input (archived commit 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for active and retired employees. Column universe: employed (PEMLR 1-2) policyholders whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed and workers without pay carry 0, and their scaled share stays out of the column rather than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): premium. Some (2): premium minus the contribution of an employee who pays one, which is the published average contribution (taken over every enrollee, including those who pay nothing) divided by one minus the published share of enrollees whose coverage required no contribution (MEPS-IC 2025 Tables II.C/D/E.4.a, United States row by firm size; State cells of those tables are mostly flagged unreliable; Series III publishes none, so government cells take the private 50-or-more share). None (3): 0; MEPS-IC publishes no share of enrollees who pay the whole premium. One factor scales every policyholder's share so the weighted anchor-universe total equals BEA NIPA Table 7.8 line 17 (B4923C) for the build year ($977.0B CY2024 in the 2026-09-30 vintage, revised from $1,002.9B; banked in the pinned feed as bea_nipa.cy2024.private_group_health_insurance), the anchor decision d1317 chose; the column carries the employed policyholders' part. CMS NHE Table 24 ($1,047.0B CY2024) is a second estimate of the same active-plus-retired concept and is recorded beside the verdict, as is MEPS-IC's own private-sector enrollment total for active employees, which the private part of the column matches within half a percent. No MEPS-IC table prices retiree coverage, so non-employed policyholders are priced as active employees. Measured totals, counts and sensitivities are in experiments/us-esi-454. The stage does not produce pre_tax_health_insurance_premiums: PolicyEngine-US treats that input as disjoint from the other premium inputs (PolicyEngine/policyengine-us#10046), so it needs its own change. Release gates: us_esi_premiums_signal_gate (structure, clone agreement, cell multiples; fails without the raw ASEC columns) and us_esi_premiums_anchor_gate (calibrated anchor-universe total within 5 percent of BEA NIPA 7.8 line 17; employed share within its band)." }, { "stage": "prior_year_income", @@ -1930,7 +1930,7 @@ "nonnegative_outputs": [ "employment_income_last_year" ], - "notes": "Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1680-1783 returns without creating values when no prior-year artifact exists; otherwise it joins adjacent ASEC files on PERIDNUM, requires I_ERNVAL == I_SEVAL == 0 on the prior record, treats only -1/-9999 as unavailable, records availability before falling back to current WSAL_VAL/SEMP_VAL, and preserves signed self-employment losses. After support cloning, datasets/cps/extended_cps.py lines 140-194, 639-745, and 1014-1073 jointly QRF-imputes both earnings leaves onto the PUF half from the exact eight predictors with a 5,000-person cap. Microcosm adds typed design weights. The archived finalizer drops formula-owned employment_income_last_year; the signed self-employment leaf and availability flag persist." + "notes": "Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 1680-1783 returns without creating values when no prior-year artifact exists; otherwise it joins adjacent ASEC files on PERIDNUM, requires I_ERNVAL == I_SEVAL == 0 on the prior record, treats only -1/-9999 as unavailable, records availability before falling back to current WSAL_VAL/SEMP_VAL, and preserves signed self-employment losses. After support cloning, datasets/cps/extended_cps.py lines 140-194, 234-248, 639-745, and 1014-1073 jointly QRF-imputes both earnings leaves onto the PUF half from the exact eight predictors with a 5,000-person cap. Microcosm adds typed design weights. The archived finalizer drops formula-owned employment_income_last_year; the signed self-employment leaf and availability flag persist." }, { "stage": "immigration_status", @@ -2398,7 +2398,7 @@ "outputs": [ "takes_up_wic_if_eligible" ], - "notes": "The retired eCPS derives PolicyEngine-US wic_category_str and assigns a stable seeded Bernoulli at category-specific rates (archived cps.py lines 684-691, wic_takeup.yaml lines 1-33, and randomness.py lines 5-28). The archived rates exactly match USDA FNS CY2022 average-month coverage: pregnant 45.6%, all postpartum 68.9%, breastfeeding 66.3%, infants 78.4%, and children 46.0%. Microcosm deliberately runs this stage after pregnancy; the retired file computed WIC categories before its later pregnancy seed, which accidentally made the pregnant rate unreachable. The hermetic sources carry no breastfeeding assessment, so breastfeeding remains unavailable and mothers of infants use FNS's all-postpartum rate rather than an invented breastfeeding status. Stable source-identity hashes preserve paired support-clone outcomes. The separate is_wic_at_nutritional_risk input remains a reviewed source-unavailability exclusion." + "notes": "The retired eCPS derives PolicyEngine-US wic_category_str and assigns a stable seeded Bernoulli at category-specific rates (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/cps/cps.py lines 684-691, parameters/take_up/wic_takeup.yaml lines 1-33, and utils/randomness.py lines 5-28). The archived rates exactly match USDA FNS CY2022 average-month coverage: pregnant 45.6%, all postpartum 68.9%, breastfeeding 66.3%, infants 78.4%, and children 46.0%. Microcosm deliberately runs this stage after pregnancy; the retired file computed WIC categories before its later pregnancy seed, which accidentally made the pregnant rate unreachable. The hermetic sources carry no breastfeeding assessment, so breastfeeding remains unavailable and mothers of infants use FNS's all-postpartum rate rather than an invented breastfeeding status. Stable source-identity hashes preserve paired support-clone outcomes. The separate is_wic_at_nutritional_risk input remains a reviewed source-unavailability exclusion." }, { "stage": "snap_abawd_discretionary_exemption", @@ -2627,7 +2627,7 @@ "nonnegative_outputs": [ "pre_subsidy_rent" ], - "notes": "Exact CPS H_TENURE and SPM carries plus the archived 10-predictor ACS rent QRF. receives_housing_assistance receives the retired second-stage CPS-to-PUF QRF, and takes_up_housing_assistance_if_eligible is kept exactly equal to that measured/imputed receipt anchor. Archived cps.py lines 664-682 preserved reporters before adding eligible nonreporters from a 0.50 draw; parameters/take_up/housing_assistance.yaml lines 12-15 say 0.50 was an ad hoc eCPS benchmark adjustment pending fuller calibration. Archived db/etl_housing_assistance.py lines 35-169 use a live HUD workbook to produce household-grain state counts, not SPM-unit recipient identities; that workbook is not pinned in the hermetic build and allocating its counts to SPM units would synthesize recipients. Microcosm never turns a reporter off and does not add unobserved take-up. Rent and both tenure enums are cloned unchanged." + "notes": "Exact CPS H_TENURE and SPM carries plus the archived 10-predictor ACS rent QRF. receives_housing_assistance receives the retired second-stage CPS-to-PUF QRF, and takes_up_housing_assistance_if_eligible is kept exactly equal to that measured/imputed receipt anchor. Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe datasets/cps/cps.py lines 664-682 preserved reporters before adding eligible nonreporters from a 0.50 draw; parameters/take_up/housing_assistance.yaml lines 12-15 say 0.50 was an ad hoc eCPS benchmark adjustment pending fuller calibration. Archived db/etl_housing_assistance.py lines 35-169 use a live HUD workbook to produce household-grain state counts, not SPM-unit recipient identities; that workbook is not pinned in the hermetic build and allocating its counts to SPM units would synthesize recipients. Microcosm never turns a reporter off and does not add unobserved take-up. Rent and both tenure enums are cloned unchanged." }, { "stage": "vehicle_assets", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index 6198319651..16d52d4c50 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093 + sha256: e2de2b41a66a0d2712191803dbf405c5280f911eedcaa83f55710f8ca77d0ba5 stage_manifest: version: 1 country: us @@ -825,11 +825,12 @@ stages: - workers_compensation nonnegative_outputs: - workers_compensation - notes: Exact annual person-level WC_VAL carry from archived cps.py lines 1559-1571; code-1 DIS_VAL slots - remain excluded from the separate disability_benefits leaf rather than added here. The CPS-only workers_compensation - output is replaced on the PUF support half with the archived eight predictors and at most 5,000 ASEC - training people (extended_cps.py lines 135-194, 234-248, 639-745, and 1014-1073). Microcosm pins 100 - trees and typed person weights as reproducibility hardening. + notes: Exact annual person-level WC_VAL carry from archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe + datasets/cps/cps.py lines 1559-1571; code-1 DIS_VAL slots remain excluded from the separate disability_benefits + leaf rather than added here. The CPS-only workers_compensation output is replaced on the PUF support + half with the archived eight predictors and at most 5,000 ASEC training people (datasets/cps/extended_cps.py + lines 135-194, 234-248, 639-745, and 1014-1073). Microcosm pins 100 trees and typed person weights + as reproducibility hardening. - stage: weeks_unemployed_input survey: Census CPS ASEC source: https://www.census.gov/programs-surveys/cps.html @@ -907,18 +908,18 @@ stages: - weeks_unemployed nonnegative_outputs: - weeks_unemployed - notes: Restores the final retired eCPS person-year weeks_unemployed input. Archived cps.py lines 1438-1442 - carries Census ASEC LKWEEKS directly, mapping only the historical -1 not-in-universe sentinel to zero; - the SHA-pinned 2023 ASEC public-use archive supplies the omitted 2022-income-year source rows exactly. - Its 146,133 person records match the locked 2022 H5 one-for-one on PH_SEQ, P_SEQ, A_LINENO, A_AGE, - A_FNLWGT, UC_VAL, and WKSWORK. The source has 4,200 positive LKWEEKS rows, range 0--51, 3.1456094276% - A_FNLWGT-weighted positive support, and 172,563,622.04 weighted person-weeks; Census allocation flags - are retained because the retired direct carry did not filter them. Archived calibration/puf_impute.py - lines 703-785 replaces only the PUF support half with a QRF using age, sex, three tax-unit roles, - joint filing, and unemployment compensation, caps training at 5,000 ASEC people, clips predictions - to 0--52, and forces predictions to zero without positive unemployment compensation. Microcosm pins - 100 trees and typed person weights as reproducibility hardening, and rounds its interpolating QRF - draws back to the source's integer-week domain; no 2022 source value is filled statistically. + notes: Restores the final retired eCPS person-year weeks_unemployed input. Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe + datasets/cps/cps.py lines 1438-1442 carries Census ASEC LKWEEKS directly, mapping only the historical + -1 not-in-universe sentinel to zero; the SHA-pinned 2023 ASEC public-use archive supplies the omitted + 2022-income-year source rows exactly. Its 146,133 person records match the locked 2022 H5 one-for-one + on PH_SEQ, P_SEQ, A_LINENO, A_AGE, A_FNLWGT, UC_VAL, and WKSWORK. The source has 4,200 positive LKWEEKS + rows, range 0--51, 3.1456094276% A_FNLWGT-weighted positive support, and 172,563,622.04 weighted person-weeks; + Census allocation flags are retained because the retired direct carry did not filter them. Archived + calibration/puf_impute.py lines 703-785 replaces only the PUF support half with a QRF using age, sex, + three tax-unit roles, joint filing, and unemployment compensation, caps training at 5,000 ASEC people, + clips predictions to 0--52, and forces predictions to zero without positive unemployment compensation. + Microcosm pins 100 trees and typed person weights as reproducibility hardening, and rounds its interpolating + QRF draws back to the source's integer-week domain; no 2022 source value is filled statistically. - stage: capital_gain_distributions survey: IRS SOI Sales of Capital Assets (TY2015) + Pub 1304 Table 1.4 source: https://www.irs.gov/statistics/soi-tax-stats-sales-of-capital-assets-reported-on-individual-tax-returns @@ -1242,16 +1243,16 @@ stages: the explicitly requested immutable pu2023.csv artifact replaces that commit''s later pu2024 default. The full SCF joins all five implicates to the summary extract on (y1, yy1). Archived commit 42ed5d45 utils/asset_imputation.py lines 15-19 maps summary-extract networth directly to the scf_net_worth - QRF anchor; calibration/source_impute.py lines 1146-1164 fits that anchor with these eight predictors - and calibration/source_impute.py lines 1324-1338 forces its construction-only balance-sheet components - to sum back to it before exporting net_worth. Microcosm therefore persists that signed household anchor - directly, including indebted households, without manufacturing or exporting the internal scf_* components. - Legacy auto_loan_balance preserves the retired pipeline''s name for original financed amounts, not - remaining principal. Auto outputs and net_worth stay Household-grain; only the three SSI liquid-asset - leaves are head-carried to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS''s approximately - six million qualifying loans issued annually and is bounded by total auto-loan interest. Negative - amount/rate codes are floored at the source boundary for the nonnegative auto outputs; signed net_worth - is never clipped.' + QRF anchor; calibration/source_impute.py lines 1146-1164 add that anchor to the QRF outputs, lines + 1227-1240 fit it with these eight predictors (SCF_PREDICTORS, lines 168-177), and calibration/source_impute.py + lines 1324-1338 forces its construction-only balance-sheet components to sum back to it before exporting + net_worth. Microcosm therefore persists that signed household anchor directly, including indebted + households, without manufacturing or exporting the internal scf_* components. Legacy auto_loan_balance + preserves the retired pipeline''s name for original financed amounts, not remaining principal. Auto + outputs and net_worth stay Household-grain; only the three SSI liquid-asset leaves are head-carried + to Person. The OBBBA qualifying-interest proxy uses Treasury/IRS''s approximately six million qualifying + loans issued annually and is bounded by total auto-loan interest. Negative amount/rate codes are floored + at the source boundary for the nonnegative auto outputs; signed net_worth is never clipped.' - stage: ssi_disability_criteria survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html @@ -1597,23 +1598,25 @@ stages: outputs: - takes_up_ssi_if_eligible notes: Restores the final retired eCPS SSI take-up export from source-backed evidence at archived commit - 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe. datasets/cps/cps.py line 584 maps the direct ASEC SSI_VAL - reporter signal, lines 650-657 derive the take-up leaf, and lines 1497-1499 export it; datasets/cps/takeup.py - lines 10-35 define the retired stable random assignment. The retired all-age 0.50 rate has no full-population - administrative source and is not reused. utils/ssi_targets.py lines 41-74 preserve the archived age-target - method and cite SSA SSI Monthly Statistics, December 2024, Table 1, row Total with—Federal payment, - by the three published age bands; those recipient counts enter the build only as ledger-fed calibration - registry targets (role ssa_ssi_age_band_recipients, microcosm#469 and microcosm#470), never as count-matching - goals in this stage. Direct SSI_VAL reporters remain true. Every other source-person identity draws - once against the documented band prior — the anchored-mass-corrected count-truthful threshold (band - target minus basis reporter floor) over (basis candidate capacity minus basis reporter floor) on weighted - uncapped_ssi > 0 candidates, zero once the reporter floor meets the target, falling back to the basis - reporter share of capacity when capacity cannot subsample the target — seeded and stable per person_source_id, - with the decision fanned identically to both support clones. The capacity basis is this frame's weights - by default, or a prior attempt's delivered-weight us_ssi_take_up.json diagnostics (microcosm#507/#508); - either way the draw happens exactly once, with no reconcile loop. Delivered recipients on release - weights are gated per enforced band (18_64, 65_plus) within a documented relative tolerance; the under-18 - band is fenced pending microcosm#453/#509 and its gap ships in the release scorecard. + 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe. datasets/cps/cps.py lines 1497-1499 map the direct ASEC + SSI_VAL reporter signal to the temporary ssi_reported anchor, line 584 loads the retired SSI rate, + lines 650-657 derive the take-up leaf from both, and lines 759-766 drop the anchor and save the leaf; + datasets/cps/takeup.py lines 10-35 define the retired stable random assignment. The retired all-age + 0.50 rate has no full-population administrative source and is not reused. utils/ssi_targets.py lines + 41-74 preserve the archived age-target method and cite SSA SSI Monthly Statistics, December 2024, + Table 1, row Total with—Federal payment, by the three published age bands; those recipient counts + enter the build only as ledger-fed calibration registry targets (role ssa_ssi_age_band_recipients, + microcosm#469 and microcosm#470), never as count-matching goals in this stage. Direct SSI_VAL reporters + remain true. Every other source-person identity draws once against the documented band prior — the + anchored-mass-corrected count-truthful threshold (band target minus basis reporter floor) over (basis + candidate capacity minus basis reporter floor) on weighted uncapped_ssi > 0 candidates, zero once + the reporter floor meets the target, falling back to the basis reporter share of capacity when capacity + cannot subsample the target — seeded and stable per person_source_id, with the decision fanned identically + to both support clones. The capacity basis is this frame's weights by default, or a prior attempt's + delivered-weight us_ssi_take_up.json diagnostics (microcosm#507/#508); either way the draw happens + exactly once, with no reconcile loop. Delivered recipients on release weights are gated per enforced + band (18_64, 65_plus) within a documented relative tolerance; the under-18 band is fenced pending + microcosm#453/#509 and its gap ships in the release scorecard. - stage: sipp_tips survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html @@ -1897,18 +1900,18 @@ stages: - employer_sponsored_insurance_premiums nonnegative_outputs: - employer_sponsored_insurance_premiums - notes: 'Restores the retired eCPS employer-premium input (archived 42ed5d45 datasets/cps/cps.py L197-271/L1575-1581) - from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: every current ESI - policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer contributions for - active and retired employees. Column universe: employed (PEMLR 1-2) policyholders whose current job - has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated self-employed - and workers without pay carry 0, and their scaled share stays out of the column rather than being - loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium and employee - contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, 0 the State - all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier (NOW_GRPFTYP2 - family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government and local/federal - employers take the MEPS-IC 2024 Series III State-government and all-government census-division cells - (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 + notes: 'Restores the retired eCPS employer-premium input (archived commit 42ed5d45 datasets/cps/cps.py + L197-271/L1575-1581) from source fields the hermetic H5 inputs lacked (microcosm#454). Anchor universe: + every current ESI policyholder (NOW_OWNGRP 1), employed or not, because the anchor counts employer + contributions for active and retired employees. Column universe: employed (PEMLR 1-2) policyholders + whose current job has an employer (PEIO1COW 1-6); dependents, non-employed policyholders, the unincorporated + self-employed and workers without pay carry 0, and their scaled share stays out of the column rather + than being loaded onto workers. Cells: MEPS-IC 2025 Series II private-sector average total premium + and employee contribution per enrolled employee by State and firm size (NOEMP 1-2 under 50, 3-6 50+, + 0 the State all-sizes cell; the finer MEPS-IC bands do not nest the NOEMP bands) and coverage tier + (NOW_GRPFTYP2 family / self plus one / self-only, cross-checked against NOW_GRPFTYP); State government + and local/federal employers take the MEPS-IC 2024 Series III State-government and all-government census-division + cells (latest published; the federal cell is a stand-in), aged to 2025 by the private national 2025/2024 ratio of the same tier and measure; a policyholder with no employer class (PEIO1COW 0, 7, 8) takes the State all-sizes private cell. AHRQ suppressed nine small-firm contribution cells; each takes the national small-firm cell times the State total-to-national ratio. Employer paid all (NOW_HIPAID 1): @@ -1984,9 +1987,10 @@ stages: files on PERIDNUM, requires I_ERNVAL == I_SEVAL == 0 on the prior record, treats only -1/-9999 as unavailable, records availability before falling back to current WSAL_VAL/SEMP_VAL, and preserves signed self-employment losses. After support cloning, datasets/cps/extended_cps.py lines 140-194, - 639-745, and 1014-1073 jointly QRF-imputes both earnings leaves onto the PUF half from the exact eight - predictors with a 5,000-person cap. Microcosm adds typed design weights. The archived finalizer drops - formula-owned employment_income_last_year; the signed self-employment leaf and availability flag persist. + 234-248, 639-745, and 1014-1073 jointly QRF-imputes both earnings leaves onto the PUF half from the + exact eight predictors with a 5,000-person cap. Microcosm adds typed design weights. The archived + finalizer drops formula-owned employment_income_last_year; the signed self-employment leaf and availability + flag persist. - stage: immigration_status survey: CPS ASEC + published unauthorized-population estimates source: https://www.pewresearch.org/short-reads/2024/07/22/what-we-know-about-unauthorized-immigrants-living-in-the-us/ @@ -2435,14 +2439,15 @@ stages: outputs: - takes_up_wic_if_eligible notes: 'The retired eCPS derives PolicyEngine-US wic_category_str and assigns a stable seeded Bernoulli - at category-specific rates (archived cps.py lines 684-691, wic_takeup.yaml lines 1-33, and randomness.py - lines 5-28). The archived rates exactly match USDA FNS CY2022 average-month coverage: pregnant 45.6%, - all postpartum 68.9%, breastfeeding 66.3%, infants 78.4%, and children 46.0%. Microcosm deliberately - runs this stage after pregnancy; the retired file computed WIC categories before its later pregnancy - seed, which accidentally made the pregnant rate unreachable. The hermetic sources carry no breastfeeding - assessment, so breastfeeding remains unavailable and mothers of infants use FNS''s all-postpartum - rate rather than an invented breastfeeding status. Stable source-identity hashes preserve paired support-clone - outcomes. The separate is_wic_at_nutritional_risk input remains a reviewed source-unavailability exclusion.' + at category-specific rates (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/cps/cps.py + lines 684-691, parameters/take_up/wic_takeup.yaml lines 1-33, and utils/randomness.py lines 5-28). + The archived rates exactly match USDA FNS CY2022 average-month coverage: pregnant 45.6%, all postpartum + 68.9%, breastfeeding 66.3%, infants 78.4%, and children 46.0%. Microcosm deliberately runs this stage + after pregnancy; the retired file computed WIC categories before its later pregnancy seed, which accidentally + made the pregnant rate unreachable. The hermetic sources carry no breastfeeding assessment, so breastfeeding + remains unavailable and mothers of infants use FNS''s all-postpartum rate rather than an invented + breastfeeding status. Stable source-identity hashes preserve paired support-clone outcomes. The separate + is_wic_at_nutritional_risk input remains a reviewed source-unavailability exclusion.' - stage: snap_abawd_discretionary_exemption survey: Census CPS ASEC + statutory exemption cap (7 U.S.C. 2015(o)(6)) source: https://www.law.cornell.edu/uscode/text/7/2015#o_6 @@ -2665,13 +2670,14 @@ stages: - pre_subsidy_rent notes: Exact CPS H_TENURE and SPM carries plus the archived 10-predictor ACS rent QRF. receives_housing_assistance receives the retired second-stage CPS-to-PUF QRF, and takes_up_housing_assistance_if_eligible is kept - exactly equal to that measured/imputed receipt anchor. Archived cps.py lines 664-682 preserved reporters - before adding eligible nonreporters from a 0.50 draw; parameters/take_up/housing_assistance.yaml lines - 12-15 say 0.50 was an ad hoc eCPS benchmark adjustment pending fuller calibration. Archived db/etl_housing_assistance.py - lines 35-169 use a live HUD workbook to produce household-grain state counts, not SPM-unit recipient - identities; that workbook is not pinned in the hermetic build and allocating its counts to SPM units - would synthesize recipients. Microcosm never turns a reporter off and does not add unobserved take-up. - Rent and both tenure enums are cloned unchanged. + exactly equal to that measured/imputed receipt anchor. Archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe + datasets/cps/cps.py lines 664-682 preserved reporters before adding eligible nonreporters from a 0.50 + draw; parameters/take_up/housing_assistance.yaml lines 12-15 say 0.50 was an ad hoc eCPS benchmark + adjustment pending fuller calibration. Archived db/etl_housing_assistance.py lines 35-169 use a live + HUD workbook to produce household-grain state counts, not SPM-unit recipient identities; that workbook + is not pinned in the hermetic build and allocating its counts to SPM units would synthesize recipients. + Microcosm never turns a reporter off and does not add unobserved take-up. Rent and both tenure enums + are cloned unchanged. - stage: vehicle_assets survey: Census SIPP source: https://www.census.gov/programs-surveys/sipp.html diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index 6ff7939935..749c1d9fda 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -195,7 +195,7 @@ "evidence": "irs_soi.historic_table_2 (compiled) carries the tax-return interest; sample fact bea_nipa.cy2023.personal_interest_income.a064rc.amount", "fence": { "origin": "us-data PR #994 (added) -> PR #1059 (removed)", - "purpose": "\"BEA personal interest/dividends include imputed interest, pension-plan dividends, and trust flows, so those macro totals should not directly calibrate tax/CPS interest and dividend variables\" (loss.py:67-70, added by PR #1059)", + "purpose": "\"BEA personal interest/dividends include imputed interest, pension-plan dividends, and trust flows, so those macro totals should not directly calibrate tax/CPS interest and dividend variables\" (utils/loss.py:67-70 at 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, added by PR #1059)", "verdict_basis": "us-data itself declined it as a direct target (PR #1059): a macro benchmark, not a close microdata concept. Tax-return interest is calibrated from IRS SOI + CBO (compiled)." } }, diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py new file mode 100644 index 0000000000..7d114b32e6 --- /dev/null +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -0,0 +1,721 @@ +"""Country spec packages cite sources by symbol or at a commit, never drifting. + +A line number names text only at a fixed commit, so a citation such as +``asec_pool.py lines 61-75`` silently points at other code after an unrelated +edit. The validators live in ``test_support.microcosm_build. +spec_source_citations``; these tests state the invariants over every country +spec package (every build directory with a ``country_package.json``) and check +each refusal branch with Hypothesis, so a vacuous pass on today's data cannot +hide a broken check: + +- no resource cites a line of a file in this repository, pinned or not + (``in_repository_line_citations``), where a cited path is this + repository's when it resolves here or is the tail of two or more + components of a file here: this repository's sources are cited by symbol + (a bare file name cannot be resolved and falls to the next rule); +- no resource cites a line of any other source unless the citation's own + string names a commit, or the resource pins that file in a + ``path``/``commit`` mapping (``unpinned_line_citations``); +- every dotted ``microcosm.*`` name in a resource resolves through import + plus getattr (``resolve_dotted_symbol``). + +Differential checks compare two records of one fact: the UK citations' commits +with the incumbent commit each resource declares, and the US known gaps' +symbol citations with the code they name (the Census person-column restore +list, the relationship fallback, and the PUF support tool's functions). + +New Zealand's binding guard is the stricter grammar in +``test_nz_spec_package.py`` (#1282); this guard scans the NZ package too. +If an NZ resource starts failing here, ask the NZ hub before exempting it. +""" + +from __future__ import annotations + +import ast +import inspect +import json + +import pandas as pd +import pytest +from hypothesis import assume, given, settings +from hypothesis import strategies as st + +from microcosm.build.us_runtime import asec_pool +from microcosm.build.us_runtime.asec_census_person_columns import ( + ASEC_CENSUS_PERSON_COLUMN_NAMES, + ASEC_CENSUS_PERSON_COLUMNS, +) +from test_support.microcosm_build.spec_source_citations import ( + COUNTRY_PACKAGE_ROOT, + MICROCOSM_SHARDS, + REPOSITORY_ROOT, + LineCitation, + commit_pinned_sources, + country_packages, + dotted_microcosm_symbols, + in_repository_line_citations, + line_citations, + named_commits, + names_a_commit, + package_payloads, + resolve_dotted_symbol, + resolves_in_repository, + unpinned_line_citations, +) + +PACKAGES = ("am", "be", "nz", "uk", "us") + + +# --------------------------------------------------------------------------- +# Committed resources +# --------------------------------------------------------------------------- + + +def test_every_country_package_is_scanned() -> None: + assert tuple(country_packages()) == PACKAGES + for country in PACKAGES: + assert package_payloads(country) + + +@pytest.mark.parametrize("country", PACKAGES) +def test_no_resource_cites_a_line_it_does_not_pin(country: str) -> None: + offenders = { + name: hits + for name, payload in package_payloads(country).items() + if (hits := unpinned_line_citations(payload)) + } + assert offenders == {} + + +@pytest.mark.parametrize("country", PACKAGES) +def test_no_resource_cites_a_line_of_this_repository(country: str) -> None: + offenders = { + name: hits + for name, payload in package_payloads(country).items() + if (hits := in_repository_line_citations(payload)) + } + assert offenders == {} + + +@pytest.mark.parametrize( + ("country", "resource", "cited"), + [ + ("nz", "as_rate_bridge.json", {"run.py"}), + ("uk", "uk_data_target_parity.json", {"obr.py", "voa_council_tax.py"}), + ("uk", "uk_population_targets.json", {"income.py", "hmrc_salary_sacrifice.py"}), + ("us", "source_stages.json", {"cps.py", "puf.py", "sipp.py"}), + ("us", "spec/sources.yaml", {"cps.py", "puf.py", "sipp.py"}), + ("us", "ecps_parity_known_gaps.json", {"cps.py", "census_cps.py"}), + ("us", "target_parity_manifest.json", {"loss.py"}), + ], +) +def test_the_pinned_line_citations_are_scanned( + country: str, resource: str, cited: set[str] +) -> None: + payload = package_payloads(country)[resource] + assert cited <= {citation.name for citation in line_citations(payload)} + + +@pytest.mark.parametrize("country", PACKAGES) +def test_every_dotted_microcosm_symbol_resolves(country: str) -> None: + named = set().union( + *map(dotted_microcosm_symbols, package_payloads(country).values()) + ) + unresolved = [] + for dotted in sorted(named): + try: + resolve_dotted_symbol(dotted) + except (ImportError, AttributeError) as error: + unresolved.append(f"{dotted}: {error}") + assert unresolved == [] + + +@pytest.mark.parametrize( + ("country", "cited"), + [ + ( + "nz", + { + "microcosm.graph.executor._structural_columns", + "microcosm.graph.executor._validate_population_declaration", + }, + ), + ( + "us", + { + "microcosm.build.us_runtime.asec_pool.load_asec_h5_tables", + "microcosm.build.us_runtime.asec_pool._prepare_year_input", + "microcosm.build.us_runtime.asec_pool._with_relationship_recode", + }, + ), + ], +) +def test_the_dotted_symbol_citations_are_scanned(country: str, cited: set[str]) -> None: + named = set().union( + *map(dotted_microcosm_symbols, package_payloads(country).values()) + ) + assert cited <= named + + +def test_only_names_under_a_shard_are_symbol_citations() -> None: + assert {"build", "frame", "graph"} <= MICROCOSM_SHARDS + text = ( + "format microcosm.axiom_input_closure.v1, read by " + "microcosm.frame.concepts.concept and microcosm.build.us/source_stages.json" + ) + assert dotted_microcosm_symbols({"note": text}) == { + "microcosm.frame.concepts.concept", + "microcosm.build.us", + } + + +@pytest.mark.parametrize( + ("dotted", "error"), + [ + ("microcosm.graph.executor._no_such_symbol", AttributeError), + ("microcosm.no_such_shard.module.symbol", ModuleNotFoundError), + ], +) +def test_an_unresolvable_dotted_symbol_raises( + dotted: str, error: type[Exception] +) -> None: + with pytest.raises(error): + resolve_dotted_symbol(dotted) + + +# --------------------------------------------------------------------------- +# Validator properties +# --------------------------------------------------------------------------- + + +_STEM = st.from_regex(r"[a-z_][a-z0-9_]{0,15}", fullmatch=True) +_SUFFIX = st.sampled_from(("py", "yaml", "json", "sh", "csv", "toml")) +#: Directories that do not exist in this checkout, so a path under one is +#: another repository's (an empty directory leaves a bare file name). +_FOREIGN_DIRECTORY = st.sampled_from( + ( + "", + "archived_data/datasets/cps/", + "targets/sources/", + "nz-lane/emtr_reproduction/", + ) +) +#: Line-locator spellings: {file} is the cited source, {a} and {b} lines. +_LINE_LOCATORS = ( + "{file}:{a}", + "{file}:{a}-{b}", + "{file}:{a},{b}", + "{file}#L{a}", + "{file}#L{a}-L{b}", + "{file} L{a}", + "{file} line {a}", + "{file} lines {a}-{b}", + "{file}, lines {a}-{b}", + "{file} (lines {a}-{b})", + "Lines {a}-{b} of {file}", + "line {a} in {file}", + "`{file}` lines {a}-{b}", + "{file}: lines {a}-{b}", + "{file}, L{a}", + "{file} (L{a})", + "line {a} of `{file}`", +) +#: Symbol-citation spellings, which carry no line number. +_SYMBOL_LOCATORS = ( + "{file} {symbol}", + '{file} _pointer("{symbol}")', + '{file}, the "# --- Relationships" block', + "{file}: {symbol}", + "{symbol} in {file} at main", +) +#: Inline-commit spellings. +_INLINE_PINS = ( + "commit {sha}", + "Commit {sha}", + "at {sha}", + "At {sha}", + "uk-data@{sha}", + "https://github.com/PolicyEngine/archived-data/blob/{sha}/x", +) +#: Prose around a citation. It has no digit, so it can neither extend a +#: planted line number nor name a commit of its own. +_PROSE = st.text( + alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ,;()'\"-_", + max_size=40, +) +_HEX = st.text(alphabet="0123456789abcdef", min_size=7, max_size=40) +_BRANCH_NAMES = ("main", "HEAD", "launch/ga-path-execution-gate", "ba6e7d", "") +#: Files in this checkout, in each path form a citation can use. +_IN_REPOSITORY_FILES = ( + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py", + "us_runtime/asec_pool.py", + "graph/executor.py", + "microcosm-frame/src/microcosm/frame/concepts.py", + "microcosm/graph/executor.py", + "tools/build_us_puf_support_base.py", + "experiments/build_j_recert/buildj_base.sh", + "experiments/build_j_recert/base_j.summary.json", +) + + +def _plant(text: str, depth: int, as_key: bool) -> object: + planted: object = {text: "cited"} if as_key else text + for level in range(depth): + planted = {"level": level, "note": [planted]} + return planted + + +def _is_commit_like(sha: str) -> bool: + return any(c.isdigit() for c in sha) and any(c.isalpha() for c in sha) + + +@settings(max_examples=300, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + prefix=_PROSE, + tail=_PROSE, + depth=st.integers(0, 3), + as_key=st.booleans(), +) +def test_an_unpinned_line_citation_is_refused_at_any_depth( + stem: str, + suffix: str, + directory: str, + locator: str, + a: int, + b: int, + prefix: str, + tail: str, + depth: int, + as_key: bool, +) -> None: + file = f"{directory}{stem}.{suffix}" + text = f"{prefix} {locator.format(file=file, a=a, b=b)} {tail}" + hits = unpinned_line_citations(_plant(text, depth, as_key)) + assert LineCitation(file, text) in hits + assert not resolves_in_repository(file) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + pin_directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + depth=st.integers(0, 3), + as_key=st.booleans(), + commit=_HEX, +) +def test_a_line_citation_of_a_file_pinned_at_a_commit_passes( + stem: str, + suffix: str, + directory: str, + pin_directory: str, + locator: str, + a: int, + b: int, + depth: int, + as_key: bool, + commit: str, +) -> None: + assume(_is_commit_like(commit)) + citation = locator.format(file=f"{directory}{stem}.{suffix}", a=a, b=b) + document = { + "source": { + "repository": "TheAxiomFoundation/ops", + "path": f"{pin_directory}{stem}.{suffix}", + "commit": commit, + }, + "notes": [_plant(citation, depth, as_key)], + } + assert line_citations(document) + assert unpinned_line_citations(document) == [] + + +@settings(max_examples=300, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_HEX, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + prefix=_PROSE, + middle=_PROSE, + tail=_PROSE, + pin_first=st.booleans(), + depth=st.integers(0, 3), + as_key=st.booleans(), +) +def test_a_commit_named_in_the_citing_string_pins_it( + stem: str, + suffix: str, + directory: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, + prefix: str, + middle: str, + tail: str, + pin_first: bool, + depth: int, + as_key: bool, +) -> None: + assume(_is_commit_like(sha)) + citation = locator.format(file=f"{directory}{stem}.{suffix}", a=a, b=b) + named = pin.format(sha=sha) + first, second = (named, citation) if pin_first else (citation, named) + text = f"{prefix} {first} {middle} {second} {tail}" + document = _plant(text, depth, as_key) + assert names_a_commit(text) + assert line_citations(document) + assert unpinned_line_citations(document) == [] + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_HEX, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), +) +def test_a_commit_named_in_another_string_does_not_pin_a_citation( + stem: str, + suffix: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, +) -> None: + assume(_is_commit_like(sha)) + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=b) + document = {"provenance": pin.format(sha=sha), "note": citation} + assert LineCitation(f"{stem}.{suffix}", citation) in unpinned_line_citations( + document + ) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + keyword=st.sampled_from(("commit ", "at ", "At ", "@")), + not_a_commit=st.one_of( + st.text(alphabet="0123456789", min_size=7, max_size=40), + st.text(alphabet="abcdef", min_size=7, max_size=40), + st.text(alphabet="0123456789abcdef", min_size=1, max_size=6), + st.text(alphabet="0123456789abcdef", min_size=41, max_size=64), + st.sampled_from(_BRANCH_NAMES), + ), + a=st.integers(1, 99_999), +) +def test_a_decimal_a_word_a_digest_or_a_branch_names_no_commit( + stem: str, + suffix: str, + locator: str, + keyword: str, + not_a_commit: str, + a: int, +) -> None: + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=a) + text = f"{citation} {keyword}{not_a_commit}" + assert not names_a_commit(f"{keyword}{not_a_commit}") + assert LineCitation(f"{stem}.{suffix}", text) in unpinned_line_citations( + {"note": text} + ) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + other=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + commit=_HEX, + branch=st.sampled_from(_BRANCH_NAMES), + pin_other_file=st.booleans(), +) +def test_a_pin_of_another_file_or_at_a_branch_does_not_cover_a_citation( + stem: str, + other: str, + suffix: str, + locator: str, + a: int, + b: int, + commit: str, + branch: str, + pin_other_file: bool, +) -> None: + if pin_other_file: + assume(other != stem) + pin = {"path": f"{other}.{suffix}", "commit": commit} + else: + pin = {"path": f"{stem}.{suffix}", "commit": branch} + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=b) + hits = unpinned_line_citations({"source": pin, "note": citation}) + assert LineCitation(f"{stem}.{suffix}", citation) in hits + assert commit_pinned_sources({"source": pin}).isdisjoint({f"{stem}.{suffix}"}) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + not_a_commit=st.one_of( + st.text(alphabet="0123456789", min_size=7, max_size=40), + st.text(alphabet="abcdef", min_size=7, max_size=40), + st.text(alphabet="0123456789ABCDEF", min_size=7, max_size=40).filter( + lambda value: any(c.isalpha() for c in value) + ), + ), + a=st.integers(1, 99_999), +) +def test_a_path_pin_whose_commit_is_no_sha_pins_nothing( + stem: str, suffix: str, locator: str, not_a_commit: str, a: int +) -> None: + pin = {"path": f"{stem}.{suffix}", "commit": not_a_commit} + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=a) + assert commit_pinned_sources({"source": pin}) == set() + assert LineCitation(f"{stem}.{suffix}", citation) in unpinned_line_citations( + {"source": pin, "note": citation} + ) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_SYMBOL_LOCATORS), + symbol=_STEM, + prefix=_PROSE, + tail=_PROSE, +) +def test_a_symbol_citation_is_not_a_line_citation( + stem: str, + suffix: str, + directory: str, + locator: str, + symbol: str, + prefix: str, + tail: str, +) -> None: + citation = locator.format(file=f"{directory}{stem}.{suffix}", symbol=symbol) + assert line_citations({"rule": f"{prefix} {citation} {tail}"}) == [] + + +@settings(max_examples=200, deadline=None) +@given( + file=st.sampled_from(_IN_REPOSITORY_FILES), + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_HEX, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + pinned_inline=st.booleans(), + pinned_by_path=st.booleans(), +) +def test_a_line_citation_of_this_repository_is_refused_even_when_pinned( + file: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, + pinned_inline: bool, + pinned_by_path: bool, +) -> None: + assume(_is_commit_like(sha)) + text = locator.format(file=file, a=a, b=b) + if pinned_inline: + text = f"{text} {pin.format(sha=sha)}" + document: dict[str, object] = {"note": text} + if pinned_by_path: + document["source"] = {"path": file, "commit": sha} + assert resolves_in_repository(file) + assert LineCitation(file, text) in in_repository_line_citations(document) + + +def test_in_repository_paths_resolve_from_the_root_packages_or_a_shard() -> None: + for file in _IN_REPOSITORY_FILES: + assert resolves_in_repository(file), file + for file in ( + "concepts.py", + "datasets/cps/cps.py", + "src/lib.rs", + "archived_data/datasets/cps/cps.py", + "targets/sources/obr.py", + "../microcosm/tools/build_us_puf_support_base.py", + "/tools/build_us_puf_support_base.py", + "microcosm/graph", + ): + assert not resolves_in_repository(file), file + assert (REPOSITORY_ROOT / _IN_REPOSITORY_FILES[0]).is_file() + assert COUNTRY_PACKAGE_ROOT.is_dir() + + +@settings(max_examples=200, deadline=None) +@given( + pin=st.sampled_from(_INLINE_PINS), + sha=_HEX, + prefix=_PROSE, + tail=_PROSE, +) +def test_named_commits_returns_the_bare_sha( + pin: str, sha: str, prefix: str, tail: str +) -> None: + assume(_is_commit_like(sha)) + assert named_commits(f"{prefix} {pin.format(sha=sha)} {tail}") == [sha] + + +# --------------------------------------------------------------------------- +# Differential: citations against the records and code they name +# --------------------------------------------------------------------------- + + +def _line_citing_commits(payload: object) -> set[str]: + texts = {citation.text for citation in line_citations(payload)} + assert texts + return {commit for text in texts for commit in named_commits(text)} + + +def test_uk_data_parity_citations_name_the_incumbent_commit() -> None: + parity = package_payloads("uk")["uk_data_target_parity.json"] + incumbent = parity["reference"]["incumbent_commit"] + assert _line_citing_commits(parity) == {incumbent} + + +def test_uk_population_citations_name_the_registry_pinned_ref() -> None: + contract = package_payloads("uk")["uk_population_targets.json"] + pinned_ref = contract["registry_parity"]["pinned_ref"] + commits = _line_citing_commits(contract) + assert pinned_ref in commits + assert all(pinned_ref.startswith(commit) for commit in commits) + + +def _known_gaps() -> dict[str, dict[str, object]]: + return package_payloads("us")["ecps_parity_known_gaps.json"]["known_gaps"] + + +#: Each US known gap whose build contract names Census person columns that +#: the reviewed restore leaves out. +_UNRESTORED_GAP_COLUMNS = { + "financial_assistance": ("FIN_VAL", "FIN_YN", "I_FINVAL"), + "is_unmarried_partner_of_household_head": ("PERRP", "PECOHAB", "A_FAMREL"), + "survivor_benefits": ("SRVS_VAL",), +} + + +@pytest.mark.parametrize(("gap", "columns"), sorted(_UNRESTORED_GAP_COLUMNS.items())) +def test_a_known_gap_names_columns_the_census_restore_leaves_out( + gap: str, columns: tuple[str, ...] +) -> None: + contract = _known_gaps()[gap]["evidence"]["hermetic_build_contract"] + assert "microcosm.build.us_runtime.asec_pool._prepare_year_input" in contract + for column in columns: + assert column in contract + assert column not in ASEC_CENSUS_PERSON_COLUMN_NAMES + + +def test_the_restore_runs_before_the_relationship_fallback() -> None: + contract = _known_gaps()["is_unmarried_partner_of_household_head"]["evidence"][ + "hermetic_build_contract" + ] + assert "restores the 2022 and 2023 inputs' missing A_EXPRRP" in contract + (exprrp,) = [ + column for column in ASEC_CENSUS_PERSON_COLUMNS if column.name == "A_EXPRRP" + ] + assert exprrp.domain is not None and 13 in exprrp.domain + source = inspect.getsource(asec_pool._prepare_year_input) + restore = source.index("restore_asec_census_person_columns(") + assert restore < source.index("_with_relationship_recode(person)") + assert "Build J predates #720" in contract + summary = json.loads( + (REPOSITORY_ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert { + source["year"]: source["relationship_recode_source"] + for source in summary["base_source"]["metadata"]["sources"] + } == { + 2022: "derived:line_spouse_parent", + 2023: "derived:line_spouse_parent", + 2024: "source:A_EXPRRP", + } + + +@settings(max_examples=100, deadline=None) +@given( + households=st.lists( + st.lists( + st.tuples( + st.integers(0, 3), + st.integers(1, 2), + st.integers(0, 3), + st.integers(0, 3), + ), + min_size=1, + max_size=4, + ), + min_size=1, + max_size=4, + ) +) +def test_the_relationship_fallback_never_emits_code_13( + households: list[list[tuple[int, int, int, int]]], +) -> None: + rows = [ + { + "PH_SEQ": household, + "A_LINENO": line, + "A_SPOUSE": spouse, + "A_SEX": sex, + "PEPAR1": parent1, + "PEPAR2": parent2, + } + for household, members in enumerate(households, start=1) + for line, (spouse, sex, parent1, parent2) in enumerate(members, start=1) + ] + person, source = asec_pool._with_relationship_recode(pd.DataFrame(rows)) + assert source == "derived:line_spouse_parent" + assert 13 not in set(person["A_EXPRRP"]) + with_source = person.assign(A_EXPRRP=13) + kept, kept_source = asec_pool._with_relationship_recode(with_source) + assert kept is with_source + assert kept_source == "source:A_EXPRRP" + + +def test_the_known_gaps_name_functions_the_puf_support_tool_defines() -> None: + tool = REPOSITORY_ROOT / "tools/build_us_puf_support_base.py" + module = ast.parse(tool.read_text(encoding="utf-8")) + defined = { + node.name: node for node in module.body if isinstance(node, ast.FunctionDef) + } + named = ("_parse_args", "_pooled_asec_sources_from_args", "_asec_sources_from_args") + contract = _known_gaps()["financial_assistance"]["evidence"][ + "hermetic_build_contract" + ] + assert all(name in contract for name in named) + assert set(named) <= set(defined) + assert '"--asec-h5"' in ast.get_source_segment( + tool.read_text(encoding="utf-8"), defined["_parse_args"] + ) diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index ee8e0a62d4..84a64a5705 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093", + "sha256": "e2de2b41a66a0d2712191803dbf405c5280f911eedcaa83f55710f8ca77d0ba5", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_financial_assistance_exclusion.py b/packages/microcosm-build/tests/engine_free/us/test_us_financial_assistance_exclusion.py index f37819fd1b..73e8b33520 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_financial_assistance_exclusion.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_financial_assistance_exclusion.py @@ -81,8 +81,17 @@ def test_exclusion_pins_all_sha_locked_hermetic_inputs_and_grain() -> None: build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() for year in (2022, 2023, 2024): assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] + contract = evidence["hermetic_build_contract"] + assert "--asec-h5 2024=, 2023=, and 2022= arguments" in contract + assert "base_source.sources[].sha256" in contract + summary = json.loads( + (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert [len(source["sha256"]) for source in summary["base_source"]["sources"]] == [ + 64, + 64, + 64, + ] asec_pool = ( ROOT / "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py" ).read_text() diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_survivor_benefits_exclusion.py b/packages/microcosm-build/tests/engine_free/us/test_us_survivor_benefits_exclusion.py index 03a12b4a24..725fdd28f2 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_survivor_benefits_exclusion.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_survivor_benefits_exclusion.py @@ -77,8 +77,17 @@ def test_exclusion_pins_all_sha_locked_hermetic_inputs() -> None: build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() for year in (2022, 2023, 2024): assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] + contract = evidence["hermetic_build_contract"] + assert "--asec-h5 2024=, 2023=, and 2022= arguments" in contract + assert "base_source.sources[].sha256" in contract + summary = json.loads( + (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert [len(source["sha256"]) for source in summary["base_source"]["sources"]] == [ + 64, + 64, + 64, + ] def test_generated_release_manifest_preserves_the_evidenced_exclusion() -> None: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_wic_nutritional_risk_exclusion.py b/packages/microcosm-build/tests/engine_free/us/test_us_wic_nutritional_risk_exclusion.py index dfc90e49e5..832d82b0c2 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_wic_nutritional_risk_exclusion.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_wic_nutritional_risk_exclusion.py @@ -123,8 +123,17 @@ def test_exclusion_pins_all_sha_locked_hermetic_inputs() -> None: build_script = (ROOT / "experiments/build_j_recert/buildj_base.sh").read_text() for year in (2022, 2023, 2024): assert f'--asec-h5 {year}="$USD/census_cps_{year}.h5"' in build_script - assert "buildj_base.sh lines 65-69" in evidence["hermetic_build_contract"] - assert "base_j.summary.json lines 55-75" in evidence["hermetic_build_contract"] + contract = evidence["hermetic_build_contract"] + assert "--asec-h5 2024=, 2023=, and 2022= arguments" in contract + assert "base_source.sources[].sha256" in contract + summary = json.loads( + (ROOT / "experiments/build_j_recert/base_j.summary.json").read_text() + ) + assert [len(source["sha256"]) for source in summary["base_source"]["sources"]] == [ + 64, + 64, + 64, + ] asec_pool = ( ROOT / "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py" ).read_text() diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py new file mode 100644 index 0000000000..51a632ac15 --- /dev/null +++ b/test_support/microcosm_build/spec_source_citations.py @@ -0,0 +1,285 @@ +"""Source-citation validators shared by the country spec package tests. + +A line number names text only at a fixed commit. A country spec resource +therefore cites this repository's own sources by symbol (a dotted +``microcosm.*`` name, a function or class name, a ``# ---`` section header, +or a key or flag name), because their line numbers move under unrelated +edits. It cites another repository's source by line only when the citation +is pinned at a commit, in one of two ways: + +- the same string names the commit inline: ``commit ``, ``at ``, + ``@`` or a GitHub ``/blob//`` permalink, where ```` is 7-40 + lowercase hex digits with at least one digit and one letter, so a decimal + amount ("at 2500000") or a word ("at defaced") pins nothing; +- the resource holds a mapping, at any depth, whose ``path`` names the file + and whose ``commit`` is a sha of the same form (``source``-style pins, + matched by file name across the whole resource). + +An inline commit pins every citation in its string. A cited path is +in-repository when it resolves in this checkout, or when it has at least two +components and is the tail of a file here (``us_runtime/asec_pool.py``); a +line citation of it is refused even when pinned. A bare file name cannot be +told apart from another repository's, so it must be pinned. + +The recognised spellings are the ones ``_LINE_CITATIONS`` documents. A symbol +between the file and its lines (``x.py some_function, lines 12-19``) is not +recognised, and neither is a dotted name with lines +(``build_loss_matrix.create_target_matrix:93-103``): that spelling is also how +ledger fact keys such as ``ledger.aggregate_fact.v2:5973fe86…`` read. +""" + +from __future__ import annotations + +import importlib +import json +import os +import re +from collections.abc import Iterator, Mapping +from functools import cache +from pathlib import Path, PurePosixPath +from typing import NamedTuple + +from microcosm.build.spec_engine import load_yaml12 +from test_support.paths import REPOSITORY_ROOT + +#: Every country spec package: a build directory with a country_package.json. +COUNTRY_PACKAGE_ROOT = REPOSITORY_ROOT / "packages/microcosm-build/src/microcosm/build" + +#: Text source kinds a line citation can name. +_SOURCE_SUFFIXES = ( + "py|pyi|ipynb|yaml|yml|jsonl|json|toml|cfg|ini|sh|md|rst|txt|csv|sql" + "|rs|tsx|ts|mjs|js" +) +#: A source file named in prose: ``concepts.py``, or a path ending in one. +_SOURCE = ( + r"(?P[A-Za-z0-9_./-]*[A-Za-z0-9_]\.(?:" + _SOURCE_SUFFIXES + r"))" + r"(?![A-Za-z0-9_])" +) +#: A line locator into a source, in each spelling the packages have used or +#: GitHub produces: ``x.py:12``, ``x.py:12-19``, ``x.py:12,19``, ``x.py#L12``, +#: ``x.py L12``, ``x.py, L12``, ``x.py (L12)``, ``x.py line 12``, +#: ``x.py, lines 12-19``, ``x.py: lines 12-19``, ``x.py (lines 12-19)``, and +#: the reverse ``lines 12-19 of x.py`` and ``line 12 in x.py``. The file name +#: may be quoted or backticked (```x.py` lines 12-19``). +_LINE_CITATIONS = ( + re.compile( + _SOURCE + r"[`'\"]?(?::|#L|,?\s*\(?\s*L|,?\s*:?\s*\(?\s*[Ll]ines?\s+)(?=\d)" + ), + re.compile( + r"\b[Ll]ines?\s+\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+)*\s+(?:of|in)\s+[`'\"]?" + + _SOURCE + ), +) +#: A commit sha: 7-40 lowercase hex with at least one digit and one letter, +#: so a decimal amount ("2500000"), a word ("defaced") or a branch name such +#: as ``main`` is not one. +_SHA = r"(?=[0-9a-f]*[0-9])(?=[0-9a-f]*[a-f])[0-9a-f]{7,40}" +#: A commit pin's value in a ``path``/``commit`` mapping. +_COMMIT_PIN = re.compile(_SHA) +#: A commit named inline. The sha must not run on into a file name +#: (``at a1b2c3d.py:12`` names a file, not a commit). +INLINE_COMMIT = re.compile( + r"(?:(?i:\bcommit\s+|\bat\s+)|@|/(?:blob|tree)/)" + + _SHA + + r"(?![0-9A-Za-z_]|\.[0-9A-Za-z_])" +) +#: Top-level directories whose files are this repository's sources. +_SOURCE_ROOTS = ("packages", "tools", "experiments", "test_support", "docs") +_SKIPPED_DIRECTORIES = {"__pycache__", ".venv", "node_modules", ".git"} +#: A dotted Microcosm symbol named in prose, such as +#: ``microcosm.graph.executor._structural_columns``. +DOTTED_MICROCOSM_SYMBOL = re.compile( + r"(?`` namespaces in this checkout. A dotted name under +#: any other second component, such as the ``microcosm.axiom_input_closure.v1`` +#: format id, names no Python symbol. +MICROCOSM_SHARDS = frozenset( + path.name + for path in REPOSITORY_ROOT.glob("packages/microcosm-*/src/microcosm/*") + if path.is_dir() and path.name != "__pycache__" +) + + +class LineCitation(NamedTuple): + """A line citation: the source as cited, and the string carrying it.""" + + file: str + text: str + + @property + def name(self) -> str: + return PurePosixPath(self.file).name + + +def nested_strings(value: object) -> Iterator[str]: + """Every string in a payload, mapping keys included, at any depth.""" + + if isinstance(value, Mapping): + for key, child in value.items(): + yield str(key) + yield from nested_strings(child) + elif isinstance(value, list): + for child in value: + yield from nested_strings(child) + elif isinstance(value, str): + yield value + + +def line_citations(value: object) -> list[LineCitation]: + """Each line citation of a source in a payload, in string order.""" + + return [ + LineCitation(match["file"], text) + for text in nested_strings(value) + for pattern in _LINE_CITATIONS + for match in pattern.finditer(text) + ] + + +def names_a_commit(text: str) -> bool: + """Whether a string names a commit inline.""" + + return INLINE_COMMIT.search(text) is not None + + +def named_commits(text: str) -> list[str]: + """The commits a string names inline, in order.""" + + return [ + re.sub(r"^(?:(?i:commit|at)\s+|@|/(?:blob|tree)/)", "", match.group(0)) + for match in INLINE_COMMIT.finditer(text) + ] + + +def commit_pinned_sources(value: object) -> set[str]: + """File names of the sources a payload pins in ``path``/``commit`` maps.""" + + pinned: set[str] = set() + if isinstance(value, Mapping): + path, commit = value.get("path"), value.get("commit") + if ( + isinstance(path, str) + and isinstance(commit, str) + and _COMMIT_PIN.fullmatch(commit) + ): + pinned.add(PurePosixPath(path).name) + for child in value.values(): + pinned |= commit_pinned_sources(child) + elif isinstance(value, list): + for child in value: + pinned |= commit_pinned_sources(child) + return pinned + + +def unpinned_line_citations(document: object) -> list[LineCitation]: + """Line citations that neither their string nor the document pins.""" + + pinned = commit_pinned_sources(document) + return [ + citation + for citation in line_citations(document) + if citation.name not in pinned and not names_a_commit(citation.text) + ] + + +@cache +def _repository_path_tails(root: Path) -> frozenset[str]: + """Every tail of two or more components of a source file in ``root``.""" + + tails: set[str] = set() + for top in _SOURCE_ROOTS: + for directory, subdirectories, files in os.walk(root / top): + subdirectories[:] = [ + name for name in subdirectories if name not in _SKIPPED_DIRECTORIES + ] + parts = PurePosixPath(Path(directory).relative_to(root).as_posix()).parts + for name in files: + path = (*parts, name) + tails.update("/".join(path[cut:]) for cut in range(len(path) - 1)) + return frozenset(tails) + + +def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: + """Whether a cited path names a file in this checkout. + + A path of two or more components resolves when it is the tail of a source + file here: ``tools/x.py``, ``packages/microcosm-build/src/...``, + ``microcosm/graph/executor.py`` and ``us_runtime/asec_pool.py`` all do. + """ + + relative = PurePosixPath(file) + if relative.is_absolute() or ".." in relative.parts or len(relative.parts) < 2: + return False + return relative.as_posix() in _repository_path_tails(root) + + +def in_repository_line_citations( + document: object, root: Path = REPOSITORY_ROOT +) -> list[LineCitation]: + """Line citations of this repository's files, pinned or not.""" + + return [ + citation + for citation in line_citations(document) + if resolves_in_repository(citation.file, root) + ] + + +def dotted_microcosm_symbols(value: object) -> set[str]: + """Every dotted ``microcosm..*`` name in a payload's strings.""" + + return { + match + for text in nested_strings(value) + for match in DOTTED_MICROCOSM_SYMBOL.findall(text) + if match.split(".")[1] in MICROCOSM_SHARDS + } + + +def resolve_dotted_symbol(dotted: str) -> object: + """Import the longest module prefix of a dotted name, then get the rest.""" + + parts = dotted.split(".") + for cut in range(len(parts), 0, -1): + name = ".".join(parts[:cut]) + try: + target: object = importlib.import_module(name) + except ModuleNotFoundError as error: + if error.name != name: + raise + continue + for attribute in parts[cut:]: + target = getattr(target, attribute) + return target + raise ModuleNotFoundError(dotted, name=dotted) + + +def country_packages() -> list[str]: + """Each country spec package directory name, sorted.""" + + return sorted( + path.parent.name for path in COUNTRY_PACKAGE_ROOT.glob("*/country_package.json") + ) + + +def package_payloads(country: str) -> dict[str, object]: + """Every resource in a country package, parsed, by relative path. + + JSON and YAML resources are parsed; any other file is one string. + """ + + root = COUNTRY_PACKAGE_ROOT / country + payloads: dict[str, object] = {} + for path in sorted(root.rglob("*")): + if not path.is_file() or "__pycache__" in path.parts: + continue + name = path.relative_to(root).as_posix() + text = path.read_text(encoding="utf-8") + if path.suffix == ".json": + payloads[name] = json.loads(text) + elif path.suffix in {".yaml", ".yml"}: + payloads[name] = load_yaml12(text, source=name) + else: + payloads[name] = text + return payloads diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index 906c3a7440..527207b31e 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093" + "e2de2b41a66a0d2712191803dbf405c5280f911eedcaa83f55710f8ca77d0ba5" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index 2ed0075d17..ae15c161cc 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -377,7 +377,8 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: '"BEA personal interest/dividends include imputed interest, ' "pension-plan dividends, and trust flows, so those macro totals " "should not directly calibrate tax/CPS interest and dividend " - 'variables" (loss.py:67-70, added by PR #1059)' + 'variables" (utils/loss.py:67-70 at ' + "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, added by PR #1059)" ), verdict_basis=( "us-data itself declined it as a direct target (PR #1059): a " diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 2454909041..9d922e82df 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "dc3b72982675b95d30cfabdeb5f7f4c492e6cd3788009ef3094ab4d48bfd9093" + "e2de2b41a66a0d2712191803dbf405c5280f911eedcaa83f55710f8ca77d0ba5" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" From a7fdc7cc799443daac2a65738865c85ab8cff16a Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 14:44:12 -0700 Subject: [PATCH 02/21] Re-pin the US digests built on source_stages.json The source-stage citation edits move the US spec_sha256 (c677d50b -> ca232744) and documentation_sha256 (801a99da -> 3cb60930), and three inventory digests: source_manifest (87b1e124 -> ecf9ca2b), late_resource_semantics (531b48cd -> e8797144) and full_checkpoint (030c8422 -> 189d4e28). Regenerated us-f0-coverage.json with tools/spec_engine_coverage.py: 41/41 inventory checks, and exactly ten leaves move (the five above, their observed copies and the pool-code observed digest). Co-Authored-By: Claude Opus 5.5 --- docs/evidence/spec-engine/us-f0-coverage.json | 20 +++++++++---------- .../build/spec_engine/inventory_coverage.py | 6 +++--- .../us/test_us_multispine_pool_tool.py | 7 ++++--- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index ba31d65a39..d2689a29e6 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,7 +4,7 @@ "version": 1 }, "country": "us", - "documentation_sha256": "801a99da54b8a4834a8c5e6c4b013fdae1e7576e0132de7e87cb1ef098f1ad8c", + "documentation_sha256": "3cb60930222f5308b503fa27fe8774f91e6c1a71cd399af59e94b79275249159", "field_usage": { "authored_normative_field_count": 32454, "claim_count": 49, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "531b48cd1f9737c3ba04aaa430a1aaa25d035217b59be9da24844948687e78f1" + "sha256": "e8797144459c2150441ce5843047486e3b96ecd62dff3f6f0b6c7d8a01ff9830" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "531b48cd1f9737c3ba04aaa430a1aaa25d035217b59be9da24844948687e78f1" + "sha256": "e8797144459c2150441ce5843047486e3b96ecd62dff3f6f0b6c7d8a01ff9830" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", + "sha256": "ecf9ca2bd6ed8c25f749eb5d64d6ce0642e9220e0a57494a5f16db1f8e513beb", "stages": 38 }, "failures": [], "observed": { - "sha256": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", + "sha256": "ecf9ca2bd6ed8c25f749eb5d64d6ce0642e9220e0a57494a5f16db1f8e513beb", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "030c842261a9f0b2378443633b92f96a876cdbc9cb9e75433967fbf97fdeec1c" + "sha256": "189d4e28f26f188a4e0ae4f9fd2a3fe6269fc39be51966cde1c9eee28beaa1ce" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "030c842261a9f0b2378443633b92f96a876cdbc9cb9e75433967fbf97fdeec1c" + "sha256": "189d4e28f26f188a4e0ae4f9fd2a3fe6269fc39be51966cde1c9eee28beaa1ce" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "af44d50258bd8dc0fdc1c06e798b0a9f6f48d48d403c217072a9808d96b152f4" + "sha256": "20e3cd3be59b8ac9a2860875837dd7bfc487c2e9c08f4d4b57056bcc3db0b055" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c677d50bd0bbcd76c8c141c2c281a7bd49145369f08da8d5db324f2febbada5f" + "spec_sha256": "ca232744664b778d2da7f23802b77db05fae294f363c92527d041b3049548863" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "c677d50bd0bbcd76c8c141c2c281a7bd49145369f08da8d5db324f2febbada5f" + "spec_sha256": "ca232744664b778d2da7f23802b77db05fae294f363c92527d041b3049548863" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 6eaf600159..35e142c892 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,12 +350,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "030c842261a9f0b2378443633b92f96a876cdbc9cb9e75433967fbf97fdeec1c", + "full_checkpoint": "189d4e28f26f188a4e0ae4f9fd2a3fe6269fc39be51966cde1c9eee28beaa1ce", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "531b48cd1f9737c3ba04aaa430a1aaa25d035217b59be9da24844948687e78f1", + "late_resource_semantics": "e8797144459c2150441ce5843047486e3b96ecd62dff3f6f0b6c7d8a01ff9830", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", @@ -373,7 +373,7 @@ # self-employment component step changes acs_transfer's attested bytes. "seed_map": "eb6b685478a35cbbf3295773c8396527d70167a4675432afa7530b0b64b611b2", "seed_protocol": "961eb7d68291fb5ed45fa96f6949af91a315a0c1db6ced1f514898c65fab935f", - "source_manifest": "87b1e124969ec11b836740c027d28dc2a9d319d279ed55be1d9fc1523390f634", + "source_manifest": "ecf9ca2bd6ed8c25f749eb5d64d6ce0642e9220e0a57494a5f16db1f8e513beb", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", } diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index 0295ba2a38..ebf59f847b 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -344,9 +344,10 @@ def capture_equality(expected: object, actual: object) -> None: # move it. It covers microcosm.fit.qrf's attested bytes too, so # fit_draw_next's caller-owned uniforms (microcosm#1332) move it, # and so do microcosm#454's meps_esi_premiums stage and - # microcosm#1376's puma_county_vintage draw module, and - # microcosm#1380's self-employment post-transfer contract. - "spec_sha256": "c677d50bd0bbcd76c8c141c2c281a7bd49145369f08da8d5db324f2febbada5f", + # microcosm#1376's puma_county_vintage draw module, + # microcosm#1380's self-employment post-transfer contract, and + # the source-stage citation rewrites that pin us-data at a commit. + "spec_sha256": "ca232744664b778d2da7f23802b77db05fae294f363c92527d041b3049548863", }, } From 38ba22c45ded3bbffadba951eac2c7ccd4ae0fdf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 18:30:19 -0700 Subject: [PATCH 03/21] Answer the round-1 review of the citation guard Bind each line citation to the nearest inline commit in its own clause, else the nearest one before it, and drop that fallback when the clause names a moving ref. Refuse commit-like tokens in no pin spelling, widen the locator grammar with a backstop for unrecognised spellings, strip this-repository prefixes and count only anchored path tails, make a path pin cover only citations whose components end its path, and stop reading truncated hashes as pins. Name the archived commit before the US PUF note's first citations and give the ESI origin its own commit, then re-pin every digest built on us/source_stages.json. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- docs/evidence/spec-engine/us-f0-coverage.json | 20 +- .../build/spec_engine/inventory_coverage.py | 6 +- .../src/microcosm/build/us/source_stages.json | 2 +- .../src/microcosm/build/us/spec/sources.yaml | 97 +-- .../build/us/target_parity_manifest.json | 2 +- .../shared/test_spec_source_citations.py | 596 ++++++++++++++++-- .../us/test_us_bundle_core_contracts.py | 2 +- .../us/test_us_multispine_pool_tool.py | 2 +- .../microcosm_build/spec_source_citations.py | 327 ++++++++-- .../microcosm_build/us_spec_bundle.py | 2 +- tools/build_us_target_parity_manifest.py | 3 +- tools/generate_us_bundle_from_constants.py | 2 +- 13 files changed, 887 insertions(+), 176 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index 6467087a62..c297c4adc0 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, and a line citation of any other source unless its string names the commit (`commit `, `at `, `@` or a `/blob//` link) or the resource pins that file in a `path`/`commit` mapping. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` name archived commit `42ed5d45`. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string. A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) pins nothing. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/docs/evidence/spec-engine/us-f0-coverage.json b/docs/evidence/spec-engine/us-f0-coverage.json index d2689a29e6..dc9322c326 100644 --- a/docs/evidence/spec-engine/us-f0-coverage.json +++ b/docs/evidence/spec-engine/us-f0-coverage.json @@ -4,7 +4,7 @@ "version": 1 }, "country": "us", - "documentation_sha256": "3cb60930222f5308b503fa27fe8774f91e6c1a71cd399af59e94b79275249159", + "documentation_sha256": "811b4091f9419c8792e5cc7fcda30d54c5a7c6d35dd944c26c005e32cae06647", "field_usage": { "authored_normative_field_count": 32454, "claim_count": 49, @@ -1361,12 +1361,12 @@ ], "expected": { "producer_count": 38, - "sha256": "e8797144459c2150441ce5843047486e3b96ecd62dff3f6f0b6c7d8a01ff9830" + "sha256": "6634f7512bee554e291b635860e0efdb90b38647099be43e8e85ccfea3f9221d" }, "failures": [], "observed": { "producer_count": 38, - "sha256": "e8797144459c2150441ce5843047486e3b96ecd62dff3f6f0b6c7d8a01ff9830" + "sha256": "6634f7512bee554e291b635860e0efdb90b38647099be43e8e85ccfea3f9221d" }, "status": "covered" }, @@ -1775,12 +1775,12 @@ "legacy_adapter.imputation.late_producer_resource_semantics" ], "expected": { - "sha256": "ecf9ca2bd6ed8c25f749eb5d64d6ce0642e9220e0a57494a5f16db1f8e513beb", + "sha256": "10c05f0fd057f75536b47e3a0160ffb2b47cdec094b6694ebd62cc6bef50d9e8", "stages": 38 }, "failures": [], "observed": { - "sha256": "ecf9ca2bd6ed8c25f749eb5d64d6ce0642e9220e0a57494a5f16db1f8e513beb", + "sha256": "10c05f0fd057f75536b47e3a0160ffb2b47cdec094b6694ebd62cc6bef50d9e8", "stages": 38 }, "status": "covered" @@ -1887,7 +1887,7 @@ "alpha", "zeta" ], - "sha256": "189d4e28f26f188a4e0ae4f9fd2a3fe6269fc39be51966cde1c9eee28beaa1ce" + "sha256": "c88b0d7fce767e7415c10bfd003b6eb0508244775605a5adf27a430299ffbb4e" }, "failures": [], "observed": { @@ -1911,7 +1911,7 @@ "alpha", "zeta" ], - "sha256": "189d4e28f26f188a4e0ae4f9fd2a3fe6269fc39be51966cde1c9eee28beaa1ce" + "sha256": "c88b0d7fce767e7415c10bfd003b6eb0508244775605a5adf27a430299ffbb4e" }, "status": "covered" }, @@ -1975,7 +1975,7 @@ "take_up_contract", "us_qbi_reconciliation_contract" ], - "sha256": "20e3cd3be59b8ac9a2860875837dd7bfc487c2e9c08f4d4b57056bcc3db0b055" + "sha256": "bbd77902cc3254268df0cf138a0cfef7b2b0effb931747fd0ce92b3cbf403b0c" }, "status": "covered" }, @@ -2599,7 +2599,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "ca232744664b778d2da7f23802b77db05fae294f363c92527d041b3049548863" + "spec_sha256": "b5c87a41b665901bdbd08a9cefd816b9c93b26c1928e841f99435ee895f24278" } }, "report_schema_version": 3, @@ -2609,7 +2609,7 @@ "country": "us", "schema_id": "country_spec", "schema_version": 1, - "spec_sha256": "ca232744664b778d2da7f23802b77db05fae294f363c92527d041b3049548863" + "spec_sha256": "b5c87a41b665901bdbd08a9cefd816b9c93b26c1928e841f99435ee895f24278" }, "status": "pass" } diff --git a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py index 35e142c892..55d2d3bf50 100644 --- a/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py +++ b/packages/microcosm-build/src/microcosm/build/spec_engine/inventory_coverage.py @@ -350,12 +350,12 @@ "acs_person_predictors": "878c788a6f037d7aca12b3586ea034eff04f3034ffa11935a736493042551f25", "authority": "9d4a9672a0f03039b1fe874b9fe21ed575be0d29f14afc396d03cdf5c809bdd2", "early_families": "4aa9f736fd76e83955477ad1667e58f48f264783f05bdc7f0102cd32d61323bd", - "full_checkpoint": "189d4e28f26f188a4e0ae4f9fd2a3fe6269fc39be51966cde1c9eee28beaa1ce", + "full_checkpoint": "c88b0d7fce767e7415c10bfd003b6eb0508244775605a5adf27a430299ffbb4e", "gap_fill_schedule": "1c31f9868f7884347cc19cf1ff65da43f950b9114941a715bab168246db414a7", "graph_nodes": "40cd51ffdfe2e9d9d08d48c08e8ded9de1e4b134783bab05c4abc6ad5c72ca1e", "geography_assignment": "44e7f5f6cd7ceeef85d532c3d46306d03de4f4ee811376e6d29920742746b65c", "late_families": "d91f9ff0eb52f43e7b6eed3d5c58c37abe1620c3a11021da15dae9c10e16d382", - "late_resource_semantics": "e8797144459c2150441ce5843047486e3b96ecd62dff3f6f0b6c7d8a01ff9830", + "late_resource_semantics": "6634f7512bee554e291b635860e0efdb90b38647099be43e8e85ccfea3f9221d", "late_schedule": "e59c019d3d454eac99ac0ac209b6c5b6faaf9bdfcaeee18c36a25be19bf7da2f", "ownership": "5f64f0aac49e2313177564f71876bffc8c81b3ded4df701e70930e60e9c98356", "primary_tuples": "fdf23da429f8c501e198a2f5b719763d523a41565c2153fdf17b0344f21bcd7c", @@ -373,7 +373,7 @@ # self-employment component step changes acs_transfer's attested bytes. "seed_map": "eb6b685478a35cbbf3295773c8396527d70167a4675432afa7530b0b64b611b2", "seed_protocol": "961eb7d68291fb5ed45fa96f6949af91a315a0c1db6ced1f514898c65fab935f", - "source_manifest": "ecf9ca2bd6ed8c25f749eb5d64d6ce0642e9220e0a57494a5f16db1f8e513beb", + "source_manifest": "10c05f0fd057f75536b47e3a0160ffb2b47cdec094b6694ebd62cc6bef50d9e8", "take_up": "9522ce40f7dea569312dd7a7beb474e5a5afadd2f1e10cb5876534c3ef623d35", "tail": "256947cd7c1bbd85f5d1fa3911a07a170ae87830f098110a40ed25d93f78347c", } diff --git a/packages/microcosm-build/src/microcosm/build/us/source_stages.json b/packages/microcosm-build/src/microcosm/build/us/source_stages.json index 9acf2709ec..b44de5c8da 100644 --- a/packages/microcosm-build/src/microcosm/build/us/source_stages.json +++ b/packages/microcosm-build/src/microcosm/build/us/source_stages.json @@ -269,7 +269,7 @@ "unadjusted_basis_qualified_property", "w2_wages_from_qualified_business" ], - "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. The pinned PUF lineage uprates the raw TY2015 rows before replacing the four IRS disclosure aggregate records (archived puf.py lines 1378-1398). Its seed-42 disaggregator enables the Forbes backbone for aggregate RECID 999999 (archived disaggregate_puf.py lines 55-60 and 85-123), producing the 3,900-record open tail retained by puf_2024.h5. This build consumes that versioned H5 and does not request unsupported Forbes synthesis from the source-stage runtime. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Microcosm places this PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target has no EARNSPLIT. Microcosm aggregates each processed donor person leaf to a tax-unit QRF target and sparsifies only the PUF support channel to the donor positive rate. Predicted person-output totals are redistributed using copied ASEC within-unit shares or leaf-specific allocation bases, with a stable first-person fallback only where the target support has no allocation mass; this preserves person-sensitive caps and eligibility where a defensible allocation basis exists. Alimony reproduces the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); Microcosm deliberately preserves measured ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets and fit. Microcosm source-aligns boolean person counts and restores the archived split/exposure identities after its weighted PUF QRF." + "notes": "PUF detail is a primary-source donor and must stay separate from benchmark comparisons. The pinned PUF lineage (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe) uprates the raw TY2015 rows before replacing the four IRS disclosure aggregate records (datasets/puf/puf.py lines 1378-1398). Its seed-42 disaggregator enables the Forbes backbone for aggregate RECID 999999 (datasets/puf/disaggregate_puf.py lines 55-60 and 85-123), producing the 3,900-record open tail retained by puf_2024.h5. This build consumes that versioned H5 and does not request unsupported Forbes synthesis from the source-stage runtime. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Microcosm places this PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target has no EARNSPLIT. Microcosm aggregates each processed donor person leaf to a tax-unit QRF target and sparsifies only the PUF support channel to the donor positive rate. Predicted person-output totals are redistributed using copied ASEC within-unit shares or leaf-specific allocation bases, with a stable first-person fallback only where the target support has no allocation mass; this preserves person-sensitive caps and eligibility where a defensible allocation basis exists. Alimony reproduces the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); Microcosm deliberately preserves measured ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets and fit. Microcosm source-aligns boolean person counts and restores the archived split/exposure identities after its weighted PUF QRF." }, { "stage": "education_inputs", diff --git a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml index 16d52d4c50..7281df5a12 100644 --- a/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml +++ b/packages/microcosm-build/src/microcosm/build/us/spec/sources.yaml @@ -99,7 +99,7 @@ sources: stage_asset: id: source_stages path: microcosm.build.us/source_stages.json - sha256: e2de2b41a66a0d2712191803dbf405c5280f911eedcaa83f55710f8ca77d0ba5 + sha256: 5fc76f9bb64f5e05aa13acdcc5dd05e7a1a0143125e32eb87c8101219bc77899 stage_manifest: version: 1 country: us @@ -372,53 +372,54 @@ stages: - unadjusted_basis_qualified_property - w2_wages_from_qualified_business notes: 'PUF detail is a primary-source donor and must stay separate from benchmark comparisons. The - pinned PUF lineage uprates the raw TY2015 rows before replacing the four IRS disclosure aggregate - records (archived puf.py lines 1378-1398). Its seed-42 disaggregator enables the Forbes backbone for - aggregate RECID 999999 (archived disaggregate_puf.py lines 55-60 and 85-123), producing the 3,900-record - open tail retained by puf_2024.h5. This build consumes that versioned H5 and does not request unsupported - Forbes synthesis from the source-stage runtime. Qualified tuition reproduces the retired source transform - as max(E03230, E87530), falling back to E03230 when the optional field is absent. Educator expense - reproduces the retired direct mapping educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, - datasets/puf/puf.py lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 - and export at lines 804-815; the processed 1.8.0 artifact carries the leaf. Microcosm places this - PUF-only detail on its dedicated PUF support channel and sparsifies to the donor positive rate, rather - than reproducing the retired both-half override. Form 4952 elected investment income reproduces the - archived direct mapping investment_income_elected_form_4952 = E58990 at puf.py line 708, its export - at lines 804-850, and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; - the pinned processed 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces - the archived direct mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, - and its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned - processed 1.8.0 artifact carries nondefault source signal. The retired processed person array embodies - a randomized EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic - ASEC target has no EARNSPLIT. Microcosm aggregates each processed donor person leaf to a tax-unit - QRF target and sparsifies only the PUF support channel to the donor positive rate. Predicted person-output - totals are redistributed using copied ASEC within-unit shares or leaf-specific allocation bases, with - a stable first-person fallback only where the target support has no allocation mass; this preserves - person-sensitive caps and eligibility where a defensible allocation basis exists. Alimony reproduces - the retired direct PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit - 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py lines 640-641, and is re-derived after - aggregate-row disaggregation; reported ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while - codes 20 (alimony) and 12 (strike benefits) are excluded from miscellaneous_income per datasets/cps/cps.py - lines 1481-1492. Domestic production reproduces the retired direct mapping domestic_production_ald - = E03240 at archived puf.py line 646, appears in the exported field list at lines 808-815, and is - re-derived after aggregate-row disaggregation; it is the former Section 199 deduction, not Section - 199A QBI. The retired weighted-QRF target and override lists include it at calibration/puf_impute.py - lines 90-198, with the fit at lines 940-1075. Casualty loss reproduces the retired direct mapping - casualty_loss = E20500 at the same commit, datasets/puf/puf.py lines 636-643, and is re-derived after - aggregate-row disaggregation. Unreimbursed employee business expenses reproduce the archived E20400 - proxy for all deductible miscellaneous expenses at the same commit, datasets/puf/puf.py lines 663-665, - and are likewise re-derived after disaggregation. Signed farm business inputs reproduce farm_operations_income - = E02100 and farm_rent_income = E27200 at archived puf.py lines 636-704 and are re-derived after disaggregation - without clipping; ASEC farm_operations_income = FRSE_VAL at archived cps.py lines 1363-1382 remains - measured on that channel, while PUF farm_income = T27800 is a distinct Schedule J leaf and a forbidden - substitute. The retired farm QRF overwrote both clone halves (calibration/puf_impute.py lines 513-672); - Microcosm deliberately preserves measured ASEC operations income and replaces only the PUF support - channel. Section 199A inputs are carried from the versioned processed PUF artifact rather than redrawn: - archived puf.py lines 105-405 define the seeded qualification, investment-income, SSTB, W-2, and UBIA - simulations; lines 748-787 enforce the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; - qbi_assumptions.yaml lines 1-118 pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 - pin the retired QRF targets and fit. Microcosm source-aligns boolean person counts and restores the - archived split/exposure identities after its weighted PUF QRF.' + pinned PUF lineage (archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe) uprates the raw TY2015 + rows before replacing the four IRS disclosure aggregate records (datasets/puf/puf.py lines 1378-1398). + Its seed-42 disaggregator enables the Forbes backbone for aggregate RECID 999999 (datasets/puf/disaggregate_puf.py + lines 55-60 and 85-123), producing the 3,900-record open tail retained by puf_2024.h5. This build + consumes that versioned H5 and does not request unsupported Forbes synthesis from the source-stage + runtime. Qualified tuition reproduces the retired source transform as max(E03230, E87530), falling + back to E03230 when the optional field is absent. Educator expense reproduces the retired direct mapping + educator_expense = E03220 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, datasets/puf/puf.py + lines 636-649, with filer/spouse allocation by earnings share at lines 617-620 and export at lines + 804-815; the processed 1.8.0 artifact carries the leaf. Microcosm places this PUF-only detail on its + dedicated PUF support channel and sparsifies to the donor positive rate, rather than reproducing the + retired both-half override. Form 4952 elected investment income reproduces the archived direct mapping + investment_income_elected_form_4952 = E58990 at puf.py line 708, its export at lines 804-850, and + its retired QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed + 1.8.0 artifact carries nondefault source signal. SALT refund income reproduces the archived direct + mapping salt_refund_income = E00700 at puf.py line 707, its export at lines 804-850, and its retired + QRF target/override entries at calibration/puf_impute.py lines 90-198; the pinned processed 1.8.0 + artifact carries nondefault source signal. The retired processed person array embodies a randomized + EARNSPLIT filer/spouse allocation (puf.py lines 477-546 and 1513-1601), but the hermetic ASEC target + has no EARNSPLIT. Microcosm aggregates each processed donor person leaf to a tax-unit QRF target and + sparsifies only the PUF support channel to the donor positive rate. Predicted person-output totals + are redistributed using copied ASEC within-unit shares or leaf-specific allocation bases, with a stable + first-person fallback only where the target support has no allocation mass; this preserves person-sensitive + caps and eligibility where a defensible allocation basis exists. Alimony reproduces the retired direct + PUF mappings alimony_income = E00800 and alimony_expense = E03500 at archived commit 42ed5d45c56df80d754fbe24cce21cfeb8d05cbe, + datasets/puf/puf.py lines 640-641, and is re-derived after aggregate-row disaggregation; reported + ASEC OI_VAL with OI_OFF = 20 remains on the ASEC half while codes 20 (alimony) and 12 (strike benefits) + are excluded from miscellaneous_income per datasets/cps/cps.py lines 1481-1492. Domestic production + reproduces the retired direct mapping domestic_production_ald = E03240 at archived puf.py line 646, + appears in the exported field list at lines 808-815, and is re-derived after aggregate-row disaggregation; + it is the former Section 199 deduction, not Section 199A QBI. The retired weighted-QRF target and + override lists include it at calibration/puf_impute.py lines 90-198, with the fit at lines 940-1075. + Casualty loss reproduces the retired direct mapping casualty_loss = E20500 at the same commit, datasets/puf/puf.py + lines 636-643, and is re-derived after aggregate-row disaggregation. Unreimbursed employee business + expenses reproduce the archived E20400 proxy for all deductible miscellaneous expenses at the same + commit, datasets/puf/puf.py lines 663-665, and are likewise re-derived after disaggregation. Signed + farm business inputs reproduce farm_operations_income = E02100 and farm_rent_income = E27200 at archived + puf.py lines 636-704 and are re-derived after disaggregation without clipping; ASEC farm_operations_income + = FRSE_VAL at archived cps.py lines 1363-1382 remains measured on that channel, while PUF farm_income + = T27800 is a distinct Schedule J leaf and a forbidden substitute. The retired farm QRF overwrote + both clone halves (calibration/puf_impute.py lines 513-672); Microcosm deliberately preserves measured + ASEC operations income and replaces only the PUF support channel. Section 199A inputs are carried + from the versioned processed PUF artifact rather than redrawn: archived puf.py lines 105-405 define + the seeded qualification, investment-income, SSTB, W-2, and UBIA simulations; lines 748-787 enforce + the all-or-nothing SSTB split; lines 860-879 export the 15 leaves; qbi_assumptions.yaml lines 1-118 + pin the assumptions; calibration/puf_impute.py lines 90-198 and 940-1075 pin the retired QRF targets + and fit. Microcosm source-aligns boolean person counts and restores the archived split/exposure identities + after its weighted PUF QRF.' - stage: education_inputs survey: IRS PUF 2015 (uprated) + Census CPS ASEC source: https://www.irs.gov/statistics/soi-tax-stats-individual-public-use-microdata-files diff --git a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json index 749c1d9fda..449ba98bea 100644 --- a/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json +++ b/packages/microcosm-build/src/microcosm/build/us/target_parity_manifest.json @@ -238,7 +238,7 @@ "reason": "BEA NIPA Table 7.8 line 17 (B4923C), employer contributions for private group health insurance: the scale anchor of the meps_esi_premiums base stage (microcosm#454, decision d1317). The stage prices every current ESI policyholder from MEPS-IC, scales that universe to this fact (BEA State Personal Income methodology, December 2025, paragraphs 3.18-3.19: contributions for active and retired employees) and writes the pe-us input (employer_sponsored_insurance_premiums) for employed policyholders only. So the fact is not a sum target on the column: the column carries about 88% of it and the rest is priced, not exported. Validation-gated instead: us_esi_premiums_anchor_gate recomputes the anchor-universe total at release weights and holds it within 5% of this fact, and the release tool refuses a feed whose values differ from the stage's pinned anchor (us_esi_premiums_anchor_feed_mismatches).", "evidence": "experiments/us-esi-454/receipts/stage_on_pool_2023_2025.json: 92.4M weighted current policyholders; anchor-universe total $977.0B at scale factor 0.971; employed column $863.9B (88.4%); other policyholders $113.2B; private-sector part of the column $665.6B against MEPS-IC's own private active-employee total of $668.6B (2024)", "fence": { - "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (this series, the NIPA 7.8 employer-contribution aggregate, before BEA's 2026-09-30 revision; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581).", + "origin": "a REAL us-data calibration target: utils/loss.py pinned employer_sponsored_insurance_premiums at 1_002.9e9 (this series, the NIPA 7.8 employer-contribution aggregate, before BEA's 2026-09-30 revision; verified at archived commit 2e044571, loss.py line 33, with etl_national_targets carrying the row), removed at 9f320d2c ('Remove survey-based national calibration targets', PR #831). The input itself was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* policyholder fields, archived commit 42ed5d45 datasets/cps/cps.py lines 197-271 and 1575-1581).", "purpose": "anchor the CBO market-income ESI component once the column has a live producer. The BEA annual update of 2026-09-30 revised CY2024 from the $1,002.9B the retired target used to $977.034B.", "verdict_basis": "deferred, not dropped: the producer is live and scaled to this fact, but the column is the employed policyholders' part of it, so the fact cannot be compiled as a sum target on the column. Release validation (us_esi_premiums_anchor_gate, both dense and sparse exports) holds the anchor-universe total to it. Whether and how to calibrate the column is decision d1264." } diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 7d114b32e6..13f854f29f 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -9,20 +9,31 @@ hide a broken check: - no resource cites a line of a file in this repository, pinned or not - (``in_repository_line_citations``), where a cited path is this - repository's when it resolves here or is the tail of two or more - components of a file here: this repository's sources are cited by symbol - (a bare file name cannot be resolved and falls to the next rule); -- no resource cites a line of any other source unless the citation's own - string names a commit, or the resource pins that file in a - ``path``/``commit`` mapping (``unpinned_line_citations``); + (``in_repository_line_citations``). A cited path is this repository's when, + after any prefix naming this repository (a ``PolicyEngine/microcosm`` + GitHub URL, or ``microcosm/`` before a top-level source directory), it is a + file here or an anchored path form of one: its path from the root, from its + distribution directory, from ``src/``, from the ``microcosm`` namespace, + from its shard, or from a first-level subpackage of its shard. A bare file + name cannot be resolved and falls to the next rule; +- no resource cites a line of any other source unless the citation binds to + an inline commit (the nearest one in its own clause, else the nearest one + before it in its string; ``citing_commit``), or the resource pins a path + that the cited path ends in, in a ``path``/``commit`` mapping + (``unpinned_line_citations``); +- no commit-like token in an unrecognised spelling sits in a citation's + clause or between the citation and the commit it binds to + (``misspelled_commit_pins``); +- no source name is followed by a line locator in an unrecognised spelling + (``unrecognised_line_locators``); - every dotted ``microcosm.*`` name in a resource resolves through import plus getattr (``resolve_dotted_symbol``). -Differential checks compare two records of one fact: the UK citations' commits -with the incumbent commit each resource declares, and the US known gaps' -symbol citations with the code they name (the Census person-column restore -list, the relationship fallback, and the PUF support tool's functions). +Differential checks compare two records of one fact: the commit each UK +citation binds to with the incumbent commit its resource declares, and the US +known gaps' symbol citations with the code they name (the Census person-column +restore list, the relationship fallback, and the PUF support tool's +functions). New Zealand's binding guard is the stricter grammar in ``test_nz_spec_package.py`` (#1282); this guard scans the NZ package too. @@ -34,6 +45,7 @@ import ast import inspect import json +from pathlib import Path, PurePosixPath import pandas as pd import pytest @@ -50,17 +62,21 @@ MICROCOSM_SHARDS, REPOSITORY_ROOT, LineCitation, + citing_commit, commit_pinned_sources, country_packages, dotted_microcosm_symbols, in_repository_line_citations, line_citations, + misspelled_commit_pins, named_commits, names_a_commit, package_payloads, + pin_covers, resolve_dotted_symbol, resolves_in_repository, unpinned_line_citations, + unrecognised_line_locators, ) PACKAGES = ("am", "be", "nz", "uk", "us") @@ -97,16 +113,44 @@ def test_no_resource_cites_a_line_of_this_repository(country: str) -> None: assert offenders == {} +@pytest.mark.parametrize("country", PACKAGES) +def test_no_resource_misspells_a_commit_pin(country: str) -> None: + offenders = { + name: hits + for name, payload in package_payloads(country).items() + if (hits := misspelled_commit_pins(payload)) + } + assert offenders == {} + + +@pytest.mark.parametrize("country", PACKAGES) +def test_no_resource_cites_a_line_in_an_unrecognised_spelling(country: str) -> None: + offenders = { + name: hits + for name, payload in package_payloads(country).items() + if (hits := unrecognised_line_locators(payload)) + } + assert offenders == {} + + @pytest.mark.parametrize( ("country", "resource", "cited"), [ ("nz", "as_rate_bridge.json", {"run.py"}), ("uk", "uk_data_target_parity.json", {"obr.py", "voa_council_tax.py"}), ("uk", "uk_population_targets.json", {"income.py", "hmrc_salary_sacrifice.py"}), - ("us", "source_stages.json", {"cps.py", "puf.py", "sipp.py"}), - ("us", "spec/sources.yaml", {"cps.py", "puf.py", "sipp.py"}), + ( + "us", + "source_stages.json", + {"cps.py", "puf.py", "disaggregate_puf.py", "sipp.py"}, + ), + ( + "us", + "spec/sources.yaml", + {"cps.py", "puf.py", "disaggregate_puf.py", "sipp.py"}, + ), ("us", "ecps_parity_known_gaps.json", {"cps.py", "census_cps.py"}), - ("us", "target_parity_manifest.json", {"loss.py"}), + ("us", "target_parity_manifest.json", {"loss.py", "cps.py"}), ], ) def test_the_pinned_line_citations_are_scanned( @@ -196,28 +240,52 @@ def test_an_unresolvable_dotted_symbol_raises( ( "", "archived_data/datasets/cps/", + "datasets/cps/", "targets/sources/", "nz-lane/emtr_reproduction/", ) ) +#: Leading directories a pin's path may carry beyond the cited path. +_PIN_ROOT = st.sampled_from(("", "upstream_data/", "src/lane/", "a/b/c/")) #: Line-locator spellings: {file} is the cited source, {a} and {b} lines. _LINE_LOCATORS = ( "{file}:{a}", "{file}:{a}-{b}", "{file}:{a},{b}", + "{file}:L{a}", "{file}#L{a}", "{file}#L{a}-L{b}", "{file} L{a}", + "{file} L{a}-L{b}", + "{file} [L{a}]", + "{file} @ L{a}", "{file} line {a}", "{file} lines {a}-{b}", + "{file} lines {a}–{b}", + "{file} lines {a} to {b}", + "{file} lines {a} through {b}", + "{file} lines {a}-{b} and {a}", "{file}, lines {a}-{b}", + "{file}; lines {a}-{b}", + "{file} — lines {a}-{b}", + "{file} - lines {a}-{b}", "{file} (lines {a}-{b})", - "Lines {a}-{b} of {file}", - "line {a} in {file}", - "`{file}` lines {a}-{b}", + "{file} at line {a}", + "{file} on lines {a}-{b}", + "{file} ll. {a}-{b}", + "{file} (l. {a})", "{file}: lines {a}-{b}", + "{file}: line {a}", "{file}, L{a}", "{file} (L{a})", + "`{file}` lines {a}-{b}", + "“{file}” lines {a}-{b}", + "Lines {a}-{b} of {file}", + "lines {a} to {b} of {file}", + "lines {a}-{b} and {a} of {file}", + "ll. {a}-{b} of {file}", + "L{a} of {file}", + "line {a} in {file}", "line {a} of `{file}`", ) #: Symbol-citation spellings, which carry no line number. @@ -228,6 +296,16 @@ def test_an_unresolvable_dotted_symbol_raises( "{file}: {symbol}", "{symbol} in {file} at main", ) +#: Line locators in no recognised spelling: a symbol sits between the file +#: and its lines. +_UNRECOGNISED_LOCATORS = ( + "{file} {symbol}, lines {a}-{b}", + "{file} {symbol} L{a}", + "{file} {symbol} (line {a})", + "{file}, {symbol} ll. {a}-{b}", +) +#: Symbols that would complete a recognised spelling in those locators. +_LOCATOR_WORDS = {"at", "on", "l", "ll", "line", "lines"} #: Inline-commit spellings. _INLINE_PINS = ( "commit {sha}", @@ -236,6 +314,12 @@ def test_an_unresolvable_dotted_symbol_raises( "At {sha}", "uk-data@{sha}", "https://github.com/PolicyEngine/archived-data/blob/{sha}/x", + "https://github.com/PolicyEngine/archived-data/tree/{sha}/x", + "https://github.com/PolicyEngine/archived-data/raw/{sha}/x", + "https://github.com/PolicyEngine/archived-data/blame/{sha}/x", + "https://github.com/PolicyEngine/archived-data/commit/{sha}", + "https://github.com/PolicyEngine/archived-data/commits/{sha}", + "https://raw.githubusercontent.com/PolicyEngine/archived-data/{sha}/x", ) #: Prose around a citation. It has no digit, so it can neither extend a #: planted line number nor name a commit of its own. @@ -243,18 +327,43 @@ def test_an_unresolvable_dotted_symbol_raises( alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ,;()'\"-_", max_size=40, ) +#: Prose within one clause: no ``;`` and no sentence end. +_CLAUSE_PROSE = st.text( + alphabet="abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ ,()'\"-_", + max_size=40, +) +#: Moving refs a clause can name. +_MOVING_REFS = ( + "on main", + "On main", + "at HEAD", + "@master", + "https://github.com/PolicyEngine/upstream-data/blob/main/x", +) +#: Clause breaks, each followed by text that opens the next clause. +_CLAUSE_BREAKS = ("; ", ";", ". See ", "! See ", "? (", ". `") _HEX = st.text(alphabet="0123456789abcdef", min_size=7, max_size=40) _BRANCH_NAMES = ("main", "HEAD", "launch/ga-path-execution-gate", "ba6e7d", "") #: Files in this checkout, in each path form a citation can use. _IN_REPOSITORY_FILES = ( "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py", + "microcosm-build/src/microcosm/build/us_runtime/asec_pool.py", + "src/microcosm/build/us_runtime/asec_pool.py", + "build/us_runtime/asec_pool.py", "us_runtime/asec_pool.py", "graph/executor.py", "microcosm-frame/src/microcosm/frame/concepts.py", "microcosm/graph/executor.py", "tools/build_us_puf_support_base.py", - "experiments/build_j_recert/buildj_base.sh", + "microcosm/tools/build_us_puf_support_base.py", + "../microcosm/tools/build_us_puf_support_base.py", + "PolicyEngine/microcosm/experiments/build_j_recert/buildj_base.sh", "experiments/build_j_recert/base_j.summary.json", + "https://github.com/PolicyEngine/microcosm/blob/main/tools/build_us_puf_support_base.py", + "https://github.com/PolicyEngine/microcosm/blob/0a1b2c3d4e/packages/microcosm-build/" + "src/microcosm/build/us_runtime/asec_pool.py", + "https://raw.githubusercontent.com/PolicyEngine/microcosm/main/tools/" + "build_us_puf_support_base.py", ) @@ -269,6 +378,9 @@ def _is_commit_like(sha: str) -> bool: return any(c.isdigit() for c in sha) and any(c.isalpha() for c in sha) +_SHA = _HEX.filter(_is_commit_like) + + @settings(max_examples=300, deadline=None) @given( stem=_STEM, @@ -296,8 +408,9 @@ def test_an_unpinned_line_citation_is_refused_at_any_depth( ) -> None: file = f"{directory}{stem}.{suffix}" text = f"{prefix} {locator.format(file=file, a=a, b=b)} {tail}" - hits = unpinned_line_citations(_plant(text, depth, as_key)) - assert LineCitation(file, text) in hits + document = _plant(text, depth, as_key) + assert LineCitation(file, text) in unpinned_line_citations(document) + assert unrecognised_line_locators(document) == [] assert not resolves_in_repository(file) @@ -306,19 +419,19 @@ def test_an_unpinned_line_citation_is_refused_at_any_depth( stem=_STEM, suffix=_SUFFIX, directory=_FOREIGN_DIRECTORY, - pin_directory=_FOREIGN_DIRECTORY, + root=_PIN_ROOT, locator=st.sampled_from(_LINE_LOCATORS), a=st.integers(1, 99_999), b=st.integers(1, 99_999), depth=st.integers(0, 3), as_key=st.booleans(), - commit=_HEX, + commit=_SHA, ) def test_a_line_citation_of_a_file_pinned_at_a_commit_passes( stem: str, suffix: str, directory: str, - pin_directory: str, + root: str, locator: str, a: int, b: int, @@ -326,12 +439,11 @@ def test_a_line_citation_of_a_file_pinned_at_a_commit_passes( as_key: bool, commit: str, ) -> None: - assume(_is_commit_like(commit)) citation = locator.format(file=f"{directory}{stem}.{suffix}", a=a, b=b) document = { "source": { "repository": "TheAxiomFoundation/ops", - "path": f"{pin_directory}{stem}.{suffix}", + "path": f"{root}{directory}{stem}.{suffix}", "commit": commit, }, "notes": [_plant(citation, depth, as_key)], @@ -340,6 +452,38 @@ def test_a_line_citation_of_a_file_pinned_at_a_commit_passes( assert unpinned_line_citations(document) == [] +@settings(max_examples=300, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + cited_directory=_FOREIGN_DIRECTORY, + pin_directory=_FOREIGN_DIRECTORY, + root=_PIN_ROOT, + locator=st.sampled_from(_LINE_LOCATORS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + commit=_SHA, +) +def test_a_path_pin_covers_exactly_the_citations_its_path_ends_in( + stem: str, + suffix: str, + cited_directory: str, + pin_directory: str, + root: str, + locator: str, + a: int, + b: int, + commit: str, +) -> None: + cited = f"{cited_directory}{stem}.{suffix}" + pinned = f"{root}{pin_directory}{stem}.{suffix}" + citation = locator.format(file=cited, a=a, b=b) + document = {"source": {"path": pinned, "commit": commit}, "note": citation} + covered = pinned == cited or pinned.endswith("/" + cited) + assert pin_covers(cited, pinned) is covered + assert (unpinned_line_citations(document) == []) is covered + + @settings(max_examples=300, deadline=None) @given( stem=_STEM, @@ -347,17 +491,18 @@ def test_a_line_citation_of_a_file_pinned_at_a_commit_passes( directory=_FOREIGN_DIRECTORY, locator=st.sampled_from(_LINE_LOCATORS), pin=st.sampled_from(_INLINE_PINS), - sha=_HEX, + sha=_SHA, a=st.integers(1, 99_999), b=st.integers(1, 99_999), prefix=_PROSE, middle=_PROSE, + clause_middle=_CLAUSE_PROSE, tail=_PROSE, pin_first=st.booleans(), depth=st.integers(0, 3), as_key=st.booleans(), ) -def test_a_commit_named_in_the_citing_string_pins_it( +def test_a_commit_before_a_citation_or_later_in_its_clause_pins_it( stem: str, suffix: str, directory: str, @@ -368,29 +513,139 @@ def test_a_commit_named_in_the_citing_string_pins_it( b: int, prefix: str, middle: str, + clause_middle: str, tail: str, pin_first: bool, depth: int, as_key: bool, ) -> None: - assume(_is_commit_like(sha)) citation = locator.format(file=f"{directory}{stem}.{suffix}", a=a, b=b) named = pin.format(sha=sha) - first, second = (named, citation) if pin_first else (citation, named) - text = f"{prefix} {first} {middle} {second} {tail}" + if pin_first: + text = f"{prefix} {named} {middle} {citation} {tail}" + else: + text = f"{prefix} {citation} {clause_middle} {named} {tail}" document = _plant(text, depth, as_key) assert names_a_commit(text) - assert line_citations(document) + assert [citing_commit(found) for found in line_citations(document)] == [sha] assert unpinned_line_citations(document) == [] +@settings(max_examples=300, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_SHA, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + prefix=_PROSE, + middle=_CLAUSE_PROSE, + clause_break=st.sampled_from(_CLAUSE_BREAKS), + tail=_PROSE, +) +def test_a_commit_only_in_a_later_clause_does_not_pin_a_citation( + stem: str, + suffix: str, + directory: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, + prefix: str, + middle: str, + clause_break: str, + tail: str, +) -> None: + file = f"{directory}{stem}.{suffix}" + citation = locator.format(file=file, a=a, b=b) + text = f"{prefix} {citation} {middle}{clause_break}{pin.format(sha=sha)} {tail}" + assert names_a_commit(text) + assert [citing_commit(found) for found in line_citations(text)] == [None] + assert LineCitation(file, text) in unpinned_line_citations({"note": text}) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_SHA, + moving=st.sampled_from(_MOVING_REFS), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + middle=_CLAUSE_PROSE, + ref_first=st.booleans(), +) +def test_an_earlier_commit_does_not_pin_a_citation_whose_clause_names_a_moving_ref( + stem: str, + suffix: str, + directory: str, + locator: str, + pin: str, + sha: str, + moving: str, + a: int, + b: int, + middle: str, + ref_first: bool, +) -> None: + file = f"{directory}{stem}.{suffix}" + citation = locator.format(file=file, a=a, b=b) + clause = ( + f"{moving} {middle} {citation}" + if ref_first + else f"{citation} {middle} {moving}" + ) + text = f"{pin.format(sha=sha)}; {clause}" + assert [citing_commit(found) for found in line_citations(text)] == [None] + assert LineCitation(file, text) in unpinned_line_citations({"note": text}) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + other=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + first_sha=_SHA, + second_sha=_SHA, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + middle=_CLAUSE_PROSE, +) +def test_each_citation_binds_the_commit_in_its_own_clause( + stem: str, + other: str, + suffix: str, + locator: str, + first_sha: str, + second_sha: str, + a: int, + b: int, + middle: str, +) -> None: + assume(stem != other) + first = locator.format(file=f"{stem}.{suffix}", a=a, b=b) + second = locator.format(file=f"{other}.{suffix}", a=b, b=a) + text = f"{first} at {first_sha} {middle}; {second} at {second_sha}" + assert { + citation.name: citing_commit(citation) for citation in line_citations(text) + } == {f"{stem}.{suffix}": first_sha, f"{other}.{suffix}": second_sha} + + @settings(max_examples=200, deadline=None) @given( stem=_STEM, suffix=_SUFFIX, locator=st.sampled_from(_LINE_LOCATORS), pin=st.sampled_from(_INLINE_PINS), - sha=_HEX, + sha=_SHA, a=st.integers(1, 99_999), b=st.integers(1, 99_999), ) @@ -403,7 +658,6 @@ def test_a_commit_named_in_another_string_does_not_pin_a_citation( a: int, b: int, ) -> None: - assume(_is_commit_like(sha)) citation = locator.format(file=f"{stem}.{suffix}", a=a, b=b) document = {"provenance": pin.format(sha=sha), "note": citation} assert LineCitation(f"{stem}.{suffix}", citation) in unpinned_line_citations( @@ -416,17 +670,20 @@ def test_a_commit_named_in_another_string_does_not_pin_a_citation( stem=_STEM, suffix=_SUFFIX, locator=st.sampled_from(_LINE_LOCATORS), - keyword=st.sampled_from(("commit ", "at ", "At ", "@")), + keyword=st.sampled_from(("commit ", "at ", "At ", "@", "/blob/")), not_a_commit=st.one_of( st.text(alphabet="0123456789", min_size=7, max_size=40), st.text(alphabet="abcdef", min_size=7, max_size=40), st.text(alphabet="0123456789abcdef", min_size=1, max_size=6), st.text(alphabet="0123456789abcdef", min_size=41, max_size=64), st.sampled_from(_BRANCH_NAMES), + _SHA.map(lambda sha: f"{sha}…"), + _SHA.map(lambda sha: f"{sha}..."), + _SHA.map(lambda sha: f"{sha}-dirty"), ), a=st.integers(1, 99_999), ) -def test_a_decimal_a_word_a_digest_or_a_branch_names_no_commit( +def test_a_decimal_a_word_a_digest_a_truncation_or_a_branch_names_no_commit( stem: str, suffix: str, locator: str, @@ -450,7 +707,7 @@ def test_a_decimal_a_word_a_digest_or_a_branch_names_no_commit( locator=st.sampled_from(_LINE_LOCATORS), a=st.integers(1, 99_999), b=st.integers(1, 99_999), - commit=_HEX, + commit=_SHA, branch=st.sampled_from(_BRANCH_NAMES), pin_other_file=st.booleans(), ) @@ -473,7 +730,10 @@ def test_a_pin_of_another_file_or_at_a_branch_does_not_cover_a_citation( citation = locator.format(file=f"{stem}.{suffix}", a=a, b=b) hits = unpinned_line_citations({"source": pin, "note": citation}) assert LineCitation(f"{stem}.{suffix}", citation) in hits - assert commit_pinned_sources({"source": pin}).isdisjoint({f"{stem}.{suffix}"}) + assert not any( + pin_covers(f"{stem}.{suffix}", pinned) + for pinned in commit_pinned_sources({"source": pin}) + ) @settings(max_examples=200, deadline=None) @@ -501,6 +761,78 @@ def test_a_path_pin_whose_commit_is_no_sha_pins_nothing( ) +@settings(max_examples=300, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_SHA, + stray=_SHA, + word=st.sampled_from(("archived ", "rev ", "sha ", "(", "")), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + middle=_CLAUSE_PROSE, + stray_first=st.booleans(), +) +def test_a_commit_in_no_pin_spelling_beside_a_citation_is_refused( + stem: str, + suffix: str, + locator: str, + pin: str, + sha: str, + stray: str, + word: str, + a: int, + b: int, + middle: str, + stray_first: bool, +) -> None: + file = f"{stem}.{suffix}" + citation = locator.format(file=file, a=a, b=b) + named = pin.format(sha=sha) + if stray_first: + # Between the commit the citation binds to and the citation. + text = f"{named}; {word}{stray} {middle} {citation}" + else: + # In the citation's own clause, after it. + text = f"{named} {middle} {citation} {word}{stray}" + assert unpinned_line_citations({"note": text}) == [] + assert (LineCitation(file, text), stray) in misspelled_commit_pins({"note": text}) + + +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + locator=st.sampled_from(_LINE_LOCATORS), + sha=_SHA, + stray=_SHA, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + middle=_CLAUSE_PROSE, + stray_first=st.booleans(), +) +def test_a_commit_like_token_in_another_clause_is_no_misspelled_pin( + stem: str, + suffix: str, + locator: str, + sha: str, + stray: str, + a: int, + b: int, + middle: str, + stray_first: bool, +) -> None: + citation = locator.format(file=f"{stem}.{suffix}", a=a, b=b) + if stray_first: + text = f"archived {stray} {middle}; {citation} at {sha}" + else: + text = f"{citation} at {sha}; archived {stray} {middle}" + assert unpinned_line_citations({"note": text}) == [] + assert misspelled_commit_pins({"note": text}) == [] + + @settings(max_examples=200, deadline=None) @given( stem=_STEM, @@ -521,15 +853,52 @@ def test_a_symbol_citation_is_not_a_line_citation( tail: str, ) -> None: citation = locator.format(file=f"{directory}{stem}.{suffix}", symbol=symbol) - assert line_citations({"rule": f"{prefix} {citation} {tail}"}) == [] + document = {"rule": f"{prefix} {citation} {tail}"} + assert line_citations(document) == [] + assert unrecognised_line_locators(document) == [] @settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + locator=st.sampled_from(_UNRECOGNISED_LOCATORS), + symbol=_STEM, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + prefix=_PROSE, + tail=_PROSE, + depth=st.integers(0, 3), + as_key=st.booleans(), +) +def test_a_line_locator_in_an_unrecognised_spelling_is_refused( + stem: str, + suffix: str, + directory: str, + locator: str, + symbol: str, + a: int, + b: int, + prefix: str, + tail: str, + depth: int, + as_key: bool, +) -> None: + assume(symbol not in _LOCATOR_WORDS) + file = f"{directory}{stem}.{suffix}" + text = f"{prefix} {locator.format(file=file, symbol=symbol, a=a, b=b)} {tail}" + document = _plant(text, depth, as_key) + assert line_citations(document) == [] + assert LineCitation(file, text) in unrecognised_line_locators(document) + + +@settings(max_examples=300, deadline=None) @given( file=st.sampled_from(_IN_REPOSITORY_FILES), locator=st.sampled_from(_LINE_LOCATORS), pin=st.sampled_from(_INLINE_PINS), - sha=_HEX, + sha=_SHA, a=st.integers(1, 99_999), b=st.integers(1, 99_999), pinned_inline=st.booleans(), @@ -545,7 +914,6 @@ def test_a_line_citation_of_this_repository_is_refused_even_when_pinned( pinned_inline: bool, pinned_by_path: bool, ) -> None: - assume(_is_commit_like(sha)) text = locator.format(file=file, a=a, b=b) if pinned_inline: text = f"{text} {pin.format(sha=sha)}" @@ -556,17 +924,25 @@ def test_a_line_citation_of_this_repository_is_refused_even_when_pinned( assert LineCitation(file, text) in in_repository_line_citations(document) -def test_in_repository_paths_resolve_from_the_root_packages_or_a_shard() -> None: - for file in _IN_REPOSITORY_FILES: +def test_in_repository_paths_resolve_from_an_anchor_or_a_repository_prefix() -> None: + for file in ( + *_IN_REPOSITORY_FILES, + "github.com/PolicyEngine/microcosm/blob/x/pyproject.toml", + ): assert resolves_in_repository(file), file for file in ( "concepts.py", + "pyproject.toml", "datasets/cps/cps.py", "src/lib.rs", "archived_data/datasets/cps/cps.py", "targets/sources/obr.py", - "../microcosm/tools/build_us_puf_support_base.py", + "build_j_recert/buildj_base.sh", + "../tools/build_us_puf_support_base.py", "/tools/build_us_puf_support_base.py", + "https://github.com/PolicyEngine/upstream-data/blob/main/tools/" + "build_us_puf_support_base.py", + "https://github.com/PolicyEngine/microcosm/blob/main/no/such/file.py", "microcosm/graph", ): assert not resolves_in_repository(file), file @@ -574,17 +950,98 @@ def test_in_repository_paths_resolve_from_the_root_packages_or_a_shard() -> None assert COUNTRY_PACKAGE_ROOT.is_dir() +def test_only_anchored_tails_name_a_file_here(tmp_path: Path) -> None: + files = ( + "packages/microcosm-build/src/microcosm/build/us_runtime/utils/loss.py", + "packages/microcosm-build/tests/utils/loss.py", + "tools/utils/loss.py", + ) + for file in files: + (tmp_path / file).parent.mkdir(parents=True, exist_ok=True) + (tmp_path / file).write_text("", encoding="utf-8") + tails = { + "/".join(PurePosixPath(file).parts[cut:]) + for file in files + for cut in range(len(PurePosixPath(file).parts)) + } + assert {tail for tail in tails if resolves_in_repository(tail, tmp_path)} == { + "packages/microcosm-build/src/microcosm/build/us_runtime/utils/loss.py", + "microcosm-build/src/microcosm/build/us_runtime/utils/loss.py", + "src/microcosm/build/us_runtime/utils/loss.py", + "microcosm/build/us_runtime/utils/loss.py", + "build/us_runtime/utils/loss.py", + "us_runtime/utils/loss.py", + "packages/microcosm-build/tests/utils/loss.py", + "microcosm-build/tests/utils/loss.py", + "tools/utils/loss.py", + } + for file in ( + "microcosm/tools/utils/loss.py", + "../microcosm/tools/utils/loss.py", + "https://github.com/PolicyEngine/microcosm/blob/main/tools/utils/loss.py", + "https://github.com/policyengine/microcosm/tree/a1b2c3d/packages/" + "microcosm-build/tests/utils/loss.py", + ): + assert resolves_in_repository(file, tmp_path), file + + +def test_the_round_one_review_counterexamples_are_refused() -> None: + obr = "targets/sources/obr.py" + for text in ( + f"uk-data {obr}:302 on main; compare us-data at 42ed5d4", + f"us-data at 42ed5d4; uk-data {obr}:302 on main", + ): + assert unpinned_line_citations({"note": text}) == [LineCitation(obr, text)] + origin = ( + "verified at archived commit 2e044571, loss.py line 33. The input was " + "imputed (archived 42ed5d45 cps.py L197-271)." + ) + assert [ + (citation.file, token) for citation, token in misspelled_commit_pins(origin) + ] == [("cps.py", "42ed5d45")] + pinned_elsewhere = { + "source": { + "path": "upstream_data/datasets/cps/cps.py", + "commit": "a1b2c3d", + }, + "note": "archived_data/datasets/cps/cps.py:12", + } + assert [ + citation.file for citation in unpinned_line_citations(pinned_elsewhere) + ] == ["archived_data/datasets/cps/cps.py"] + tool = "tools/build_us_puf_support_base.py" + for text in ( + f"https://github.com/PolicyEngine/microcosm/blob/a1b2c3d/{tool}#L12", + f"microcosm/{tool}:12 at a1b2c3d", + ): + assert len(in_repository_line_citations({"note": text})) == 1, text + assert unpinned_line_citations({"note": "x.py:12 at e2de2b41…"}) + for text in ( + "x.py:L12", + "x.py at line 12", + "x.py at lines 12-19", + "x.py; lines 12-19", + "x.py — lines 12-19", + "x.py [L12]", + "lines 12 to 19 of x.py", + "lines 12 through 19 in x.py", + "lines 12-19 and 30 of x.py", + "lines 12—19 of x.py", + "x.py ll. 12-19", + ): + assert line_citations(text) == [LineCitation("x.py", text)], text + + @settings(max_examples=200, deadline=None) @given( pin=st.sampled_from(_INLINE_PINS), - sha=_HEX, + sha=_SHA, prefix=_PROSE, tail=_PROSE, ) def test_named_commits_returns_the_bare_sha( pin: str, sha: str, prefix: str, tail: str ) -> None: - assume(_is_commit_like(sha)) assert named_commits(f"{prefix} {pin.format(sha=sha)} {tail}") == [sha] @@ -593,24 +1050,55 @@ def test_named_commits_returns_the_bare_sha( # --------------------------------------------------------------------------- -def _line_citing_commits(payload: object) -> set[str]: - texts = {citation.text for citation in line_citations(payload)} - assert texts - return {commit for text in texts for commit in named_commits(text)} +def _citing_commits(payload: object) -> set[str | None]: + citations = line_citations(payload) + assert citations + return {citing_commit(citation) for citation in citations} -def test_uk_data_parity_citations_name_the_incumbent_commit() -> None: +def test_uk_data_parity_citations_bind_the_incumbent_commit() -> None: parity = package_payloads("uk")["uk_data_target_parity.json"] incumbent = parity["reference"]["incumbent_commit"] - assert _line_citing_commits(parity) == {incumbent} + assert _citing_commits(parity) == {incumbent} -def test_uk_population_citations_name_the_registry_pinned_ref() -> None: +def test_uk_population_citations_bind_the_registry_pinned_ref() -> None: contract = package_payloads("uk")["uk_population_targets.json"] pinned_ref = contract["registry_parity"]["pinned_ref"] - commits = _line_citing_commits(contract) + commits = _citing_commits(contract) assert pinned_ref in commits - assert all(pinned_ref.startswith(commit) for commit in commits) + assert all( + commit is not None and pinned_ref.startswith(commit) for commit in commits + ) + + +#: The us-data commit the archived PUF and CPS pipelines are cited at. +_ARCHIVED_US_DATA = "42ed5d45c56df80d754fbe24cce21cfeb8d05cbe" + + +@pytest.mark.parametrize("resource", ["source_stages.json", "spec/sources.yaml"]) +def test_the_puf_note_binds_its_disclosure_citations_to_the_archived_commit( + resource: str, +) -> None: + bound = { + citation.text[citation.start : citation.end]: citing_commit(citation) + for citation in line_citations(package_payloads("us")[resource]) + } + assert bound["datasets/puf/puf.py lines 1378-1398"] == _ARCHIVED_US_DATA + assert ( + bound["datasets/puf/disaggregate_puf.py lines 55-60 and 85-123"] + == _ARCHIVED_US_DATA + ) + + +def test_the_esi_origin_binds_each_citation_to_its_own_commit() -> None: + manifest = package_payloads("us")["target_parity_manifest.json"] + origin = manifest["families"]["bea_nipa.private_group_health_insurance"]["fence"][ + "origin" + ] + assert { + (citation.file, citing_commit(citation)) for citation in line_citations(origin) + } == {("loss.py", "2e044571"), ("datasets/cps/cps.py", _ARCHIVED_US_DATA[:8])} def _known_gaps() -> dict[str, dict[str, object]]: diff --git a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py index 84a64a5705..cb966b2b6d 100644 --- a/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py +++ b/packages/microcosm-build/tests/engine_free/us/test_us_bundle_core_contracts.py @@ -25,7 +25,7 @@ def test_source_surface_classification_is_complete() -> None: } assert normative["stage_asset"] == { "id": "source_stages", - "sha256": "e2de2b41a66a0d2712191803dbf405c5280f911eedcaa83f55710f8ca77d0ba5", + "sha256": "5fc76f9bb64f5e05aa13acdcc5dd05e7a1a0143125e32eb87c8101219bc77899", } assert operational["stage_asset"] == { "path": "microcosm.build.us/source_stages.json" diff --git a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py index ebf59f847b..777c68d1ac 100644 --- a/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py +++ b/packages/microcosm-build/tests/engine_workflow/us/test_us_multispine_pool_tool.py @@ -347,7 +347,7 @@ def capture_equality(expected: object, actual: object) -> None: # microcosm#1376's puma_county_vintage draw module, # microcosm#1380's self-employment post-transfer contract, and # the source-stage citation rewrites that pin us-data at a commit. - "spec_sha256": "ca232744664b778d2da7f23802b77db05fae294f363c92527d041b3049548863", + "spec_sha256": "b5c87a41b665901bdbd08a9cefd816b9c93b26c1928e841f99435ee895f24278", }, } diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 51a632ac15..72ce2fc914 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -7,25 +7,45 @@ edits. It cites another repository's source by line only when the citation is pinned at a commit, in one of two ways: -- the same string names the commit inline: ``commit ``, ``at ``, - ``@`` or a GitHub ``/blob//`` permalink, where ```` is 7-40 - lowercase hex digits with at least one digit and one letter, so a decimal - amount ("at 2500000") or a word ("at defaced") pins nothing; -- the resource holds a mapping, at any depth, whose ``path`` names the file - and whose ``commit`` is a sha of the same form (``source``-style pins, - matched by file name across the whole resource). - -An inline commit pins every citation in its string. A cited path is -in-repository when it resolves in this checkout, or when it has at least two -components and is the tail of a file here (``us_runtime/asec_pool.py``); a -line citation of it is refused even when pinned. A bare file name cannot be -told apart from another repository's, so it must be pinned. - -The recognised spellings are the ones ``_LINE_CITATIONS`` documents. A symbol -between the file and its lines (``x.py some_function, lines 12-19``) is not -recognised, and neither is a dotted name with lines -(``build_loss_matrix.create_target_matrix:93-103``): that spelling is also how -ledger fact keys such as ``ledger.aggregate_fact.v2:5973fe86…`` read. +- the string carrying the citation names the commit inline: ``commit ``, + ``at ``, ``@``, a GitHub ``/blob/``, ``/tree/``, ``/raw/``, + ``/blame/``, ``/commit/`` or ``/commits/`` URL segment, or a + ``raw.githubusercontent.com////`` URL, where ```` is + 7-40 lowercase hex digits with at least one digit and one letter, so a + decimal amount ("at 2500000"), a word ("at defaced") or a truncated hash + ("at e2de2b41…") pins nothing; +- the resource holds a mapping, at any depth, whose ``path`` ends with the + cited path and whose ``commit`` is a sha of the same form (``source``-style + pins): a pin of ``upstream_data/datasets/cps/cps.py`` covers + ``cps.py`` and ``datasets/cps/cps.py``, not ``other/cps.py``. + +Each citation binds to one inline commit (``citing_commit``): the nearest one +in the citation's own clause, else the nearest one before it in the string, +unless the citation's clause names a moving ref (``on main``, ``@HEAD``, +``/blob/master/``). A clause ends at a ``;`` or a sentence end. So +"obr.py:302 on main; compare us-data at 42ed5d4" and "us-data at 42ed5d4; +obr.py:302 on main" pin nothing, and in "x.py:12 at ; y.py:30 at " each +file binds its own commit. A commit-like token in no recognised +spelling, in a citation's clause or between the citation and the commit it +binds to ("archived 42ed5d45 cps.py L197"), is refused as a misspelled pin +(``misspelled_commit_pins``), so a stray sha cannot sit beside a citation +that some other commit pins. + +A cited path is in-repository when, after any prefix naming this repository +is removed (a ``github.com/PolicyEngine/microcosm/blob//`` URL, or +``microcosm/`` before a top-level source directory), it is a file here or one +of the path forms ``_repository_path_tails`` lists. A line citation of it is +refused even when pinned. A bare file name cannot be told apart from another +repository's, so it must be pinned. + +The recognised spellings are the ones ``_LINE_CITATIONS`` documents. A source +name followed by a line locator in no recognised spelling, with no ``.``, +``;`` or line break between (``x.py some_function, lines 12-19``), is refused +too (``unrecognised_line_locators``), so a new spelling fails loudly instead +of passing unscanned. A dotted name with lines +(``build_loss_matrix.create_target_matrix:93-103``) is out of scope: that +spelling is also how ledger fact keys such as +``ledger.aggregate_fact.v2:5973fe86…`` read. """ from __future__ import annotations @@ -35,9 +55,9 @@ import os import re from collections.abc import Iterator, Mapping +from dataclasses import dataclass, field from functools import cache from pathlib import Path, PurePosixPath -from typing import NamedTuple from microcosm.build.spec_engine import load_yaml12 from test_support.paths import REPOSITORY_ROOT @@ -50,26 +70,64 @@ "py|pyi|ipynb|yaml|yml|jsonl|json|toml|cfg|ini|sh|md|rst|txt|csv|sql" "|rs|tsx|ts|mjs|js" ) -#: A source file named in prose: ``concepts.py``, or a path ending in one. +#: A source file named in prose: ``concepts.py``, a path ending in one, or a +#: URL of one. _SOURCE = ( - r"(?P[A-Za-z0-9_./-]*[A-Za-z0-9_]\.(?:" + _SOURCE_SUFFIXES + r"))" - r"(?![A-Za-z0-9_])" + r"(?P(?:https?:)?[A-Za-z0-9_./-]*[A-Za-z0-9_]\.(?:" + + _SOURCE_SUFFIXES + + r"))(?![A-Za-z0-9_])" ) +#: Quotes or backticks around a file name or a locator. +_QUOTE = r"[`'\"‘’“”]*" +#: A line number, plain or GitHub-style (``L12``). +_NUMBER = r"L?\s*\d+(?!\w)" +#: One line or a range of them: ``12-19``, ``L12-L19``, ``12–19``, +#: ``12 to 19``, ``12 through 19``. +_SPAN = rf"{_NUMBER}(?:\s*(?:[-–—]|(?i:to|through))\s*{_NUMBER})?" +#: Several lines or ranges: ``12-19, 30`` or ``12-19 and 30``. +_NUMBERS = rf"{_SPAN}(?:\s*(?:,|,?\s*(?i:and))\s*{_SPAN})*" +#: A line label: ``line``, ``lines`` (any case), ``l.`` or ``ll.``. +_LABEL = r"(?i:lines?\b\s+|ll?\.\s*)" #: A line locator into a source, in each spelling the packages have used or -#: GitHub produces: ``x.py:12``, ``x.py:12-19``, ``x.py:12,19``, ``x.py#L12``, -#: ``x.py L12``, ``x.py, L12``, ``x.py (L12)``, ``x.py line 12``, -#: ``x.py, lines 12-19``, ``x.py: lines 12-19``, ``x.py (lines 12-19)``, and -#: the reverse ``lines 12-19 of x.py`` and ``line 12 in x.py``. The file name -#: may be quoted or backticked (```x.py` lines 12-19``). +#: GitHub produces, followed by its line numbers: +#: +#: - forward: ``x.py:12``, ``x.py:L12``, ``x.py#L12``, ``x.py L12``, +#: ``x.py line 12``, ``x.py: lines 12``, ``x.py @ L12``, and a separator +#: (``,``, ``;``, ``—``, ``–`` or a spaced ``-``), a bracket and ``at`` or +#: ``on`` before a label or an ``L`` number: ``x.py, lines 12-19``, +#: ``x.py (lines 12-19)``, ``x.py [L12]``, ``x.py at line 12``, +#: ``x.py — ll. 12-19``; +#: - reverse: ``lines 12-19 of x.py``, ``line 12 in x.py``, ``L12 of x.py``. +#: +#: The numbers are any ``_NUMBERS``; the file name and the locator may be +#: quoted or backticked (```x.py` lines 12-19``). _LINE_CITATIONS = ( re.compile( - _SOURCE + r"[`'\"]?(?::|#L|,?\s*\(?\s*L|,?\s*:?\s*\(?\s*[Ll]ines?\s+)(?=\d)" + _SOURCE + + _QUOTE + + r"(?::(?=L?\d)" + + rf"|\s*[:：]\s*{_QUOTE}{_LABEL}" + + rf"|\s*@\s*{_QUOTE}(?:{_LABEL}|(?=L\d))" + + r"|#(?=L\d)" + + r"|\s*(?:[,;—–]|\s-(?=\s))?\s*[\[(]?\s*(?:(?i:at|on)\s+)?" + + rf"{_QUOTE}(?:{_LABEL}|(?=L\d)))" + + _QUOTE + + _NUMBERS ), re.compile( - r"\b[Ll]ines?\s+\d+(?:\s*[-–]\s*\d+)?(?:\s*,\s*\d+)*\s+(?:of|in)\s+[`'\"]?" + rf"\b(?:{_LABEL}|(?=L\d)){_QUOTE}{_NUMBERS}{_QUOTE}\s+(?i:of|in)\s+{_QUOTE}" + _SOURCE ), ) +#: A source name followed by a labelled line locator (``L12``, ``line 12``, +#: ``ll. 12``) within 24 characters, with no ``.``, ``;`` or line break +#: between: the shape of every forward spelling, recognised or not. A bare +#: number after a colon is left out: ``x.json: 63 rows`` is a count. +_LOCATOR_NEAR_A_SOURCE = re.compile( + _SOURCE + + _QUOTE + + r"[^.;\n]{0,24}?(?" + _SHA - + r"(?![0-9A-Za-z_]|\.[0-9A-Za-z_])" + + r")(?![0-9A-Za-z_…-]|\.[0-9A-Za-z_.])" +) +#: A commit-like token in any spelling: not part of a path, a URL, an +#: identifier, a longer token or a truncated hash. +_COMMIT_LIKE = re.compile( + r"(? str: @@ -126,17 +219,40 @@ def nested_strings(value: object) -> Iterator[str]: yield value +def _citation_matches(text: str) -> list[re.Match[str]]: + return sorted( + (match for pattern in _LINE_CITATIONS for match in pattern.finditer(text)), + key=lambda match: match.start(), + ) + + def line_citations(value: object) -> list[LineCitation]: """Each line citation of a source in a payload, in string order.""" return [ - LineCitation(match["file"], text) + LineCitation(match["file"], text, match.start(), match.end()) for text in nested_strings(value) - for pattern in _LINE_CITATIONS - for match in pattern.finditer(text) + for match in _citation_matches(text) ] +def unrecognised_line_locators(value: object) -> list[LineCitation]: + """Source names followed by a line locator in no recognised spelling.""" + + found: list[LineCitation] = [] + for text in nested_strings(value): + matches = _citation_matches(text) + files = {match.start("file") for match in matches} + spans = [(match.start(), match.end()) for match in matches] + for near in _LOCATOR_NEAR_A_SOURCE.finditer(text): + if near.start("file") in files or any( + start <= near.end() < end for start, end in spans + ): + continue + found.append(LineCitation(near["file"], text, near.start(), near.end())) + return found + + def names_a_commit(text: str) -> bool: """Whether a string names a commit inline.""" @@ -146,14 +262,83 @@ def names_a_commit(text: str) -> bool: def named_commits(text: str) -> list[str]: """The commits a string names inline, in order.""" - return [ - re.sub(r"^(?:(?i:commit|at)\s+|@|/(?:blob|tree)/)", "", match.group(0)) - for match in INLINE_COMMIT.finditer(text) - ] + return [match["sha"] for match in INLINE_COMMIT.finditer(text)] + + +def _clause(citation: LineCitation) -> tuple[int, int]: + """The bounds of the clauses of a citation's string that it touches. + + A break inside the citation (``x.py; lines 12-19``) does not split it. + """ + + if citation.start < 0: + raise ValueError("only a citation from line_citations carries its position") + first, last = 0, len(citation.text) + for clause_break in _CLAUSE_BREAK.finditer(citation.text): + if clause_break.end() <= citation.start: + first = clause_break.end() + elif clause_break.start() >= citation.end: + last = clause_break.start() + break + return first, last + + +def _binding(citation: LineCitation) -> re.Match[str] | None: + """The inline commit match a line citation binds to, if any.""" + + commits = list(INLINE_COMMIT.finditer(citation.text)) + start, end = _clause(citation) + in_clause = [match for match in commits if start <= match.start() < end] + if in_clause: + return min( + in_clause, + key=lambda match: max( + match.start() - citation.end, citation.start - match.end() + ), + ) + if _MOVING_REF.search(citation.text, start, end): + return None + before = [match for match in commits if match.start() < start] + return before[-1] if before else None + + +def citing_commit(citation: LineCitation) -> str | None: + """The inline commit a line citation binds to, if any. + + That is the nearest commit in the citation's own clause, else the nearest + one before the citation in its string, unless the citation's clause names + a moving ref. + """ + + binding = _binding(citation) + return None if binding is None else binding["sha"] + + +def misspelled_commit_pins(value: object) -> list[tuple[LineCitation, str]]: + """Commit-like tokens in no pin spelling beside a line citation. + + A token counts when it sits in the citation's own clause, or between the + citation and an earlier commit the citation binds to. + """ + + found: list[tuple[LineCitation, str]] = [] + for citation in line_citations(value): + text = citation.text + pinned = {match.start("sha") for match in INLINE_COMMIT.finditer(text)} + start, end = _clause(citation) + binding = _binding(citation) + if binding is not None and binding.end() < start: + start = binding.end() + found.extend( + (citation, token.group(0)) + for token in _COMMIT_LIKE.finditer(text, start, end) + if token.start() not in pinned + ) + return found def commit_pinned_sources(value: object) -> set[str]: - """File names of the sources a payload pins in ``path``/``commit`` maps.""" + """Paths of the sources a payload pins in ``path``/``commit`` mappings.""" pinned: set[str] = set() if isinstance(value, Mapping): @@ -163,7 +348,7 @@ def commit_pinned_sources(value: object) -> set[str]: and isinstance(commit, str) and _COMMIT_PIN.fullmatch(commit) ): - pinned.add(PurePosixPath(path).name) + pinned.add(PurePosixPath(path).as_posix()) for child in value.values(): pinned |= commit_pinned_sources(child) elif isinstance(value, list): @@ -172,20 +357,42 @@ def commit_pinned_sources(value: object) -> set[str]: return pinned +def pin_covers(cited: str, pinned: str) -> bool: + """Whether a ``path`` pin covers a cited path: the cited components end it.""" + + cited_parts = PurePosixPath(cited).parts + pinned_parts = PurePosixPath(pinned).parts + return ( + 0 < len(cited_parts) <= len(pinned_parts) + and pinned_parts[-len(cited_parts) :] == cited_parts + ) + + def unpinned_line_citations(document: object) -> list[LineCitation]: - """Line citations that neither their string nor the document pins.""" + """Line citations that neither an inline commit nor a path pin covers.""" pinned = commit_pinned_sources(document) return [ citation for citation in line_citations(document) - if citation.name not in pinned and not names_a_commit(citation.text) + if citing_commit(citation) is None + and not any(pin_covers(citation.file, pin) for pin in pinned) ] @cache def _repository_path_tails(root: Path) -> frozenset[str]: - """Every tail of two or more components of a source file in ``root``.""" + """The path forms that name a source file in ``root``. + + A file is named by its path from the root (``tools/x.py``) and, under + ``packages``, by its path from its distribution directory + (``microcosm-build/src/...``). A file in a package's ``src/microcosm/`` + is also named by its path from ``src/``, from the ``microcosm`` namespace + (``microcosm/graph/executor.py``), from its shard (``graph/executor.py``) + and from a first-level subpackage of its shard + (``us_runtime/asec_pool.py``), never by a deeper tail, so an unrelated + ``utils/loss.py`` stays another repository's. + """ tails: set[str] = set() for top in _SOURCE_ROOTS: @@ -196,22 +403,36 @@ def _repository_path_tails(root: Path) -> frozenset[str]: parts = PurePosixPath(Path(directory).relative_to(root).as_posix()).parts for name in files: path = (*parts, name) - tails.update("/".join(path[cut:]) for cut in range(len(path) - 1)) + cuts = {0} + if path[0] == "packages": + cuts.add(1) + if path[2:4] == ("src", "microcosm"): + cuts.update(range(2, min(6, len(path) - 1))) + tails.update( + "/".join(path[cut:]) for cut in cuts if cut < len(path) - 1 + ) return frozenset(tails) def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: """Whether a cited path names a file in this checkout. - A path of two or more components resolves when it is the tail of a source - file here: ``tools/x.py``, ``packages/microcosm-build/src/...``, + A prefix naming this repository is removed first; what is left resolves + when it is a file here (after such a prefix) or one of the path forms + ``_repository_path_tails`` lists: ``tools/x.py``, + ``microcosm/tools/x.py``, ``packages/microcosm-build/src/...``, ``microcosm/graph/executor.py`` and ``us_runtime/asec_pool.py`` all do. """ - relative = PurePosixPath(file) - if relative.is_absolute() or ".." in relative.parts or len(relative.parts) < 2: + named = _THIS_REPOSITORY.match(file) + relative = PurePosixPath(file[named.end() :] if named else file) + if relative.is_absolute() or ".." in relative.parts: return False - return relative.as_posix() in _repository_path_tails(root) + if named is not None and (root / relative).is_file(): + return True + return len(relative.parts) >= 2 and relative.as_posix() in _repository_path_tails( + root + ) def in_repository_line_citations( diff --git a/test_support/microcosm_build/us_spec_bundle.py b/test_support/microcosm_build/us_spec_bundle.py index 527207b31e..f21b80ff79 100644 --- a/test_support/microcosm_build/us_spec_bundle.py +++ b/test_support/microcosm_build/us_spec_bundle.py @@ -195,7 +195,7 @@ def _load_generator_module(): LEGACY_COMPATIBILITY_SHA256 = { "source_stages.json": ( - "e2de2b41a66a0d2712191803dbf405c5280f911eedcaa83f55710f8ca77d0ba5" + "5fc76f9bb64f5e05aa13acdcc5dd05e7a1a0143125e32eb87c8101219bc77899" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" diff --git a/tools/build_us_target_parity_manifest.py b/tools/build_us_target_parity_manifest.py index ae15c161cc..f98c06cec8 100644 --- a/tools/build_us_target_parity_manifest.py +++ b/tools/build_us_target_parity_manifest.py @@ -248,7 +248,8 @@ def _fence(origin: str, purpose: str, verdict_basis: str) -> dict[str, str]: "etl_national_targets carrying the row), removed at 9f320d2c ('Remove " "survey-based national calibration targets', PR #831). The input itself " "was imputed by the retired eCPS (MEPS-IC priors over ASEC NOW_* " - "policyholder fields, archived 42ed5d45 cps.py L197-271/L1575-1581)." + "policyholder fields, archived commit 42ed5d45 datasets/cps/cps.py lines " + "197-271 and 1575-1581)." ) _ESI_NHE_ORIGIN = ( diff --git a/tools/generate_us_bundle_from_constants.py b/tools/generate_us_bundle_from_constants.py index 9d922e82df..41c42c9c28 100644 --- a/tools/generate_us_bundle_from_constants.py +++ b/tools/generate_us_bundle_from_constants.py @@ -122,7 +122,7 @@ def _domain_builders() -> dict[str, Callable[[], object]]: # frozen files, is the forward YAML -> legacy-payload path. FROZEN_LEGACY_RESOURCE_SHA256 = { "source_stages.json": ( - "e2de2b41a66a0d2712191803dbf405c5280f911eedcaa83f55710f8ca77d0ba5" + "5fc76f9bb64f5e05aa13acdcc5dd05e7a1a0143125e32eb87c8101219bc77899" ), "support_spine.json": ( "68f37dc6ae6e0cde7ebccb53f88dd4a800e63456f838fa214ff98d1db8d815be" From 01da9a37c3951b3889265238b753a25b4c25b8ca Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 19:37:41 -0700 Subject: [PATCH 04/21] Test a stray sha between a citation's clause and its binding commit The misspelled-pin property placed its "between" token after the clause break, inside the citation's own clause, so dropping the gap rule left it passing. Add a placement in the binding commit's clause. Co-Authored-By: Claude Opus 5.5 --- .../engine_free/shared/test_spec_source_citations.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 13f854f29f..5c53f5dbde 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -773,7 +773,7 @@ def test_a_path_pin_whose_commit_is_no_sha_pins_nothing( a=st.integers(1, 99_999), b=st.integers(1, 99_999), middle=_CLAUSE_PROSE, - stray_first=st.booleans(), + place=st.sampled_from(("gap", "clause_start", "clause_end")), ) def test_a_commit_in_no_pin_spelling_beside_a_citation_is_refused( stem: str, @@ -786,13 +786,16 @@ def test_a_commit_in_no_pin_spelling_beside_a_citation_is_refused( a: int, b: int, middle: str, - stray_first: bool, + place: str, ) -> None: file = f"{stem}.{suffix}" citation = locator.format(file=file, a=a, b=b) named = pin.format(sha=sha) - if stray_first: - # Between the commit the citation binds to and the citation. + if place == "gap": + # Between the commit the citation binds to and the citation's clause. + text = f"{named} {word}{stray}; {middle} {citation}" + elif place == "clause_start": + # In the citation's own clause, before it. text = f"{named}; {word}{stray} {middle} {citation}" else: # In the citation's own clause, after it. From f30da07cd5dbe7e93f502d05b8ff3b4a8494d936 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 19:57:29 -0700 Subject: [PATCH 05/21] Close the round-2 review's three gaps in the citation guard Accept quotes before a colon or # locator's number, so x.py:"12-19" is a citation. Count raw.githubusercontent.com URLs at a branch, refs/heads/ branches and quoted branch names as moving refs, so an earlier commit no longer pins them. Stop reading a prose pin (commit, at, @) whose sha runs on into a path: in "at a1b2c3d/foo.py:12" the hex is a directory. Add a property for each, plus the review's exact inputs. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 159 ++++++++++++++++++ .../microcosm_build/spec_source_citations.py | 35 ++-- 3 files changed, 182 insertions(+), 14 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index c297c4adc0..722fadc0da 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string. A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) pins nothing. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 5c53f5dbde..462f87bc2e 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -288,6 +288,12 @@ def test_an_unresolvable_dotted_symbol_raises( "line {a} in {file}", "line {a} of `{file}`", ) +#: The spellings that name the file before its lines. +_FORWARD_LOCATORS = tuple( + locator + for locator in _LINE_LOCATORS + if locator.index("{file}") < locator.index("{a}") +) #: Symbol-citation spellings, which carry no line number. _SYMBOL_LOCATORS = ( "{file} {symbol}", @@ -338,7 +344,36 @@ def test_an_unresolvable_dotted_symbol_raises( "On main", "at HEAD", "@master", + "on `main`", + "at 'HEAD'", + "@“master”", + "on refs/heads/main", "https://github.com/PolicyEngine/upstream-data/blob/main/x", + "https://github.com/PolicyEngine/upstream-data/blob/refs/heads/master/x", + "https://raw.githubusercontent.com/PolicyEngine/upstream-data/main/x", + "https://raw.githubusercontent.com/PolicyEngine/upstream-data/refs/heads/main/x", +) +#: URL prefixes of another repository's files at a moving ref. +_MOVING_URLS = ( + "https://github.com/PolicyEngine/upstream-data/blob/main/", + "https://github.com/PolicyEngine/upstream-data/blob/refs/heads/master/", + "https://raw.githubusercontent.com/PolicyEngine/upstream-data/main/", + "https://raw.githubusercontent.com/PolicyEngine/upstream-data/HEAD/", + "https://raw.githubusercontent.com/PolicyEngine/upstream-data/refs/heads/main/", +) +#: Quote marks a file name or a locator may carry. +_QUOTE_MARKS = st.sampled_from(("", "`", "'", '"', "‘", "’", "“", "”")) +#: Locators with quote slots before and after the file ({q1}, {q2}) and +#: before and after its line numbers ({q3}, {q4}). +_QUOTABLE_LOCATORS = ( + "{q1}{file}{q2}:{q3}{a}-{b}{q4}", + "{q1}{file}{q2}:{q3}L{a}{q4}", + "{q1}{file}{q2}#{q3}L{a}{q4}", + "{q1}{file}{q2} {q3}lines {a}-{b}{q4}", + "{q1}{file}{q2}, {q3}lines {a}-{b}{q4}", + "{q1}{file}{q2} @ {q3}L{a}{q4}", + "{q1}{file}{q2}: {q3}lines {a}-{b}{q4}", + "{q3}lines {a}-{b}{q4} of {q1}{file}{q2}", ) #: Clause breaks, each followed by text that opens the next clause. _CLAUSE_BREAKS = ("; ", ";", ". See ", "! See ", "? (", ". `") @@ -414,6 +449,48 @@ def test_an_unpinned_line_citation_is_refused_at_any_depth( assert not resolves_in_repository(file) +@settings(max_examples=300, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + here=st.one_of(st.none(), st.sampled_from(_IN_REPOSITORY_FILES)), + locator=st.sampled_from(_QUOTABLE_LOCATORS), + q1=_QUOTE_MARKS, + q2=_QUOTE_MARKS, + q3=_QUOTE_MARKS, + q4=_QUOTE_MARKS, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + depth=st.integers(0, 3), + as_key=st.booleans(), +) +def test_a_quoted_line_locator_is_refused_as_a_value_or_a_key( + stem: str, + suffix: str, + directory: str, + here: str | None, + locator: str, + q1: str, + q2: str, + q3: str, + q4: str, + a: int, + b: int, + depth: int, + as_key: bool, +) -> None: + file = here or f"{directory}{stem}.{suffix}" + text = locator.format(file=file, q1=q1, q2=q2, q3=q3, q4=q4, a=a, b=b) + document = _plant(text, depth, as_key) + assert line_citations(document) == [LineCitation(file, text)] + assert unrecognised_line_locators(document) == [] + if here is None: + assert LineCitation(file, text) in unpinned_line_citations(document) + else: + assert in_repository_line_citations(document) == [LineCitation(file, text)] + + @settings(max_examples=200, deadline=None) @given( stem=_STEM, @@ -607,6 +684,38 @@ def test_an_earlier_commit_does_not_pin_a_citation_whose_clause_names_a_moving_r assert LineCitation(file, text) in unpinned_line_citations({"note": text}) +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + directory=_FOREIGN_DIRECTORY, + url=st.sampled_from(_MOVING_URLS), + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_SHA, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + clause_break=st.sampled_from(_CLAUSE_BREAKS), +) +def test_an_earlier_commit_does_not_pin_a_cited_url_at_a_moving_ref( + stem: str, + suffix: str, + directory: str, + url: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, + clause_break: str, +) -> None: + file = f"{url}{directory}{stem}.{suffix}" + citation = locator.format(file=file, a=a, b=b) + text = f"{pin.format(sha=sha)}{clause_break}{citation}" + assert [citing_commit(found) for found in line_citations(text)] == [None] + assert LineCitation(file, text) in unpinned_line_citations({"note": text}) + + @settings(max_examples=200, deadline=None) @given( stem=_STEM, @@ -699,6 +808,34 @@ def test_a_decimal_a_word_a_digest_a_truncation_or_a_branch_names_no_commit( ) +@settings(max_examples=200, deadline=None) +@given( + stem=_STEM, + suffix=_SUFFIX, + deeper=_PIN_ROOT, + locator=st.sampled_from(_FORWARD_LOCATORS), + keyword=st.sampled_from(("commit ", "Commit ", "at ", "At ", "uk-data@")), + sha=_SHA, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), +) +def test_a_hex_directory_after_a_pin_word_names_no_commit( + stem: str, + suffix: str, + deeper: str, + locator: str, + keyword: str, + sha: str, + a: int, + b: int, +) -> None: + file = f"{sha}/{deeper}{stem}.{suffix}" + text = locator.format(file=f"{keyword}{file}", a=a, b=b) + assert not names_a_commit(text) + assert LineCitation(file, text) in unpinned_line_citations({"note": text}) + assert misspelled_commit_pins({"note": text}) == [] + + @settings(max_examples=200, deadline=None) @given( stem=_STEM, @@ -1035,6 +1172,28 @@ def test_the_round_one_review_counterexamples_are_refused() -> None: assert line_citations(text) == [LineCitation("x.py", text)], text +def test_the_round_two_review_counterexamples_are_refused() -> None: + tool = "tools/build_us_puf_support_base.py" + for document in ({"note": f'{tool}:"12-19"'}, {f'{tool}:"12-19"': "cited"}): + assert [ + citation.file for citation in in_repository_line_citations(document) + ] == [tool] + cps = ( + "https://raw.githubusercontent.com/PolicyEngine/upstream-data/main/" + "upstream_data/datasets/cps/cps.py" + ) + for text in (f"commit a1b2c3d; {cps}#L12", "commit a1b2c3d; x.py:12 on `main`"): + assert [citing_commit(found) for found in line_citations(text)] == [None] + assert len(unpinned_line_citations({"note": text})) == 1, text + text = "look at a1b2c3d/foo.py:12" + assert unpinned_line_citations({"note": text}) == [ + LineCitation("a1b2c3d/foo.py", text) + ] + # A URL pin is still followed by its path. + pinned = "https://github.com/o/upstream-data/blob/a1b2c3d/foo.py#L12" + assert [citing_commit(found) for found in line_citations(pinned)] == ["a1b2c3d"] + + @settings(max_examples=200, deadline=None) @given( pin=st.sampled_from(_INLINE_PINS), diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 72ce2fc914..679f76825b 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -12,8 +12,8 @@ ``/blame/``, ``/commit/`` or ``/commits/`` URL segment, or a ``raw.githubusercontent.com////`` URL, where ```` is 7-40 lowercase hex digits with at least one digit and one letter, so a - decimal amount ("at 2500000"), a word ("at defaced") or a truncated hash - ("at e2de2b41…") pins nothing; + decimal amount ("at 2500000"), a word ("at defaced"), a truncated hash + ("at e2de2b41…") or a hex directory ("at a1b2c3d/foo.py") pins nothing; - the resource holds a mapping, at any depth, whose ``path`` ends with the cited path and whose ``commit`` is a sha of the same form (``source``-style pins): a pin of ``upstream_data/datasets/cps/cps.py`` covers @@ -22,7 +22,8 @@ Each citation binds to one inline commit (``citing_commit``): the nearest one in the citation's own clause, else the nearest one before it in the string, unless the citation's clause names a moving ref (``on main``, ``@HEAD``, -``/blob/master/``). A clause ends at a ``;`` or a sentence end. So +``on `main` ``, ``/blob/master/``, ``raw.githubusercontent.com///main/``). +A clause ends at a ``;`` or a sentence end. So "obr.py:302 on main; compare us-data at 42ed5d4" and "us-data at 42ed5d4; obr.py:302 on main" pin nothing, and in "x.py:12 at ; y.py:30 at " each file binds its own commit. A commit-like token in no recognised @@ -100,15 +101,15 @@ #: - reverse: ``lines 12-19 of x.py``, ``line 12 in x.py``, ``L12 of x.py``. #: #: The numbers are any ``_NUMBERS``; the file name and the locator may be -#: quoted or backticked (```x.py` lines 12-19``). +#: quoted or backticked (```x.py` lines 12-19``, ``x.py:"12-19"``). _LINE_CITATIONS = ( re.compile( _SOURCE + _QUOTE - + r"(?::(?=L?\d)" + + rf"(?::(?={_QUOTE}L?\d)" + rf"|\s*[:：]\s*{_QUOTE}{_LABEL}" + rf"|\s*@\s*{_QUOTE}(?:{_LABEL}|(?=L\d))" - + r"|#(?=L\d)" + + rf"|#(?={_QUOTE}L\d)" + r"|\s*(?:[,;—–]|\s-(?=\s))?\s*[\[(]?\s*(?:(?i:at|on)\s+)?" + rf"{_QUOTE}(?:{_LABEL}|(?=L\d)))" + _QUOTE @@ -136,24 +137,32 @@ _COMMIT_PIN = re.compile(_SHA) #: A commit named inline. The sha must not run on into a file name #: (``at a1b2c3d.py:12`` names a file), a longer token (``a1b2c3d-x``) or an -#: ellipsis (``e2de2b41…`` and ``e2de2b41...`` are truncated hashes). +#: ellipsis (``e2de2b41…`` and ``e2de2b41...`` are truncated hashes). After a +#: prose spelling (``commit``, ``at``, ``@``) it must not run on into a path +#: either: in ``at a1b2c3d/foo.py:12`` the hex is a directory. A URL segment +#: is always followed by one. INLINE_COMMIT = re.compile( - r"(?:(?i:\bcommit\s+|\bat\s+)|@|/(?:blob|tree|raw|blame|commits?)/" + r"(?:(?P(?i:\bcommit\s+|\bat\s+)|@)|/(?:blob|tree|raw|blame|commits?)/" r"|githubusercontent\.com/[^/\s]+/[^/\s]+/)" + r"(?P" + _SHA - + r")(?![0-9A-Za-z_…-]|\.[0-9A-Za-z_.])" + + r")(?![0-9A-Za-z_…-]|\.[0-9A-Za-z_.])(?(prose)(?!/))" ) #: A commit-like token in any spelling: not part of a path, a URL, an #: identifier, a longer token or a truncated hash. _COMMIT_LIKE = re.compile( r"(?//main/``, either also under +#: ``refs/heads/``. _MOVING_REF = re.compile( - r"\b(?i:on|at)\s+(?:main|master|HEAD)\b|@(?:main|master|HEAD)\b" - r"|/(?:blob|tree|raw|blame)/(?:main|master|HEAD)/" + rf"\b(?i:on|at)\s+{_QUOTE}{_BRANCH}\b|@{_QUOTE}{_BRANCH}\b" + rf"|/(?:blob|tree|raw|blame)/{_BRANCH}/" + rf"|githubusercontent\.com/[^/\s]+/[^/\s]+/{_BRANCH}/" ) #: A clause break: a semicolon, or the space after a sentence end before a #: capital, a bracket or a backtick. From 1840bc92f8d155adb685f1090245058ebb990df0 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 21:49:18 -0700 Subject: [PATCH 06/21] Resolve this repository's URLs at a ref of any depth The in-repository rule removed a GitHub URL's ref as one path segment, so .../microcosm/blob/refs/heads/main/tools/x.py#L12 (and any slash-named branch) left heads/main/tools/x.py unresolved and passed when pinned. Try the path after each segment instead, and add a property over refs of one to four segments plus the round-3 review's exact input. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 61 +++++++++++++++++++ .../microcosm_build/spec_source_citations.py | 35 +++++++---- 3 files changed, 85 insertions(+), 13 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index 722fadc0da..de63e0853d 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 462f87bc2e..674b0df266 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -399,7 +399,29 @@ def test_an_unresolvable_dotted_symbol_raises( "src/microcosm/build/us_runtime/asec_pool.py", "https://raw.githubusercontent.com/PolicyEngine/microcosm/main/tools/" "build_us_puf_support_base.py", + "https://raw.githubusercontent.com/PolicyEngine/microcosm/refs/heads/main/tools/" + "build_us_puf_support_base.py", + "https://github.com/PolicyEngine/microcosm/blob/refs/heads/main/tools/" + "build_us_puf_support_base.py", + "https://github.com/PolicyEngine/microcosm/tree/feature/citations/packages/" + "microcosm-build/src/microcosm/build/us_runtime/asec_pool.py", +) +#: URL prefixes of this repository, up to the ref. +_THIS_REPOSITORY_URLS = ( + "https://github.com/PolicyEngine/microcosm/blob/", + "https://github.com/policyengine/microcosm/tree/", + "github.com/PolicyEngine/microcosm/raw/", + "https://www.github.com/PolicyEngine/microcosm/blame/", + "https://raw.githubusercontent.com/PolicyEngine/microcosm/", ) +#: Files in this checkout, from the repository root. +_ROOTED_FILES = ( + "tools/build_us_puf_support_base.py", + "packages/microcosm-build/src/microcosm/build/us_runtime/asec_pool.py", + "experiments/build_j_recert/buildj_base.sh", +) +#: One segment of a git ref. +_REF_SEGMENT = st.from_regex(r"[A-Za-z0-9_-]{1,12}", fullmatch=True) def _plant(text: str, depth: int, as_key: bool) -> object: @@ -1064,6 +1086,33 @@ def test_a_line_citation_of_this_repository_is_refused_even_when_pinned( assert LineCitation(file, text) in in_repository_line_citations(document) +@settings(max_examples=200, deadline=None) +@given( + url=st.sampled_from(_THIS_REPOSITORY_URLS), + ref=st.lists(_REF_SEGMENT, min_size=1, max_size=4).map("/".join), + path=st.sampled_from(_ROOTED_FILES), + locator=st.sampled_from(_LINE_LOCATORS), + pin=st.sampled_from(_INLINE_PINS), + sha=_SHA, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), +) +def test_a_url_of_this_repository_is_refused_at_a_ref_of_any_depth( + url: str, + ref: str, + path: str, + locator: str, + pin: str, + sha: str, + a: int, + b: int, +) -> None: + file = f"{url}{ref}/{path}" + text = f"{locator.format(file=file, a=a, b=b)} {pin.format(sha=sha)}" + assert resolves_in_repository(file) + assert in_repository_line_citations({"note": text}) == [LineCitation(file, text)] + + def test_in_repository_paths_resolve_from_an_anchor_or_a_repository_prefix() -> None: for file in ( *_IN_REPOSITORY_FILES, @@ -1194,6 +1243,18 @@ def test_the_round_two_review_counterexamples_are_refused() -> None: assert [citing_commit(found) for found in line_citations(pinned)] == ["a1b2c3d"] +def test_the_round_three_review_counterexample_is_refused() -> None: + tool = "tools/build_us_puf_support_base.py" + for url in ( + "https://raw.githubusercontent.com/PolicyEngine/microcosm/refs/heads/main/", + "https://github.com/PolicyEngine/microcosm/blob/refs/heads/main/", + ): + text = f"{url}{tool}#L12 at a1b2c3d" + assert in_repository_line_citations({"note": text}) == [ + LineCitation(f"{url}{tool}", text) + ] + + @settings(max_examples=200, deadline=None) @given( pin=st.sampled_from(_INLINE_PINS), diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 679f76825b..5a661f0c0d 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -33,9 +33,10 @@ that some other commit pins. A cited path is in-repository when, after any prefix naming this repository -is removed (a ``github.com/PolicyEngine/microcosm/blob//`` URL, or -``microcosm/`` before a top-level source directory), it is a file here or one -of the path forms ``_repository_path_tails`` lists. A line citation of it is +is removed (a ``github.com/PolicyEngine/microcosm/blob//`` URL, at a ref +of any depth such as ``refs/heads/main``, or ``microcosm/`` before a +top-level source directory), it is a file here or one of the path forms +``_repository_path_tails`` lists. A line citation of it is refused even when pinned. A bare file name cannot be told apart from another repository's, so it must be pinned. @@ -171,12 +172,13 @@ _SOURCE_ROOTS = ("packages", "tools", "experiments", "test_support", "docs") _SKIPPED_DIRECTORIES = {"__pycache__", ".venv", "node_modules", ".git"} #: A prefix naming this repository before a path in it: a GitHub URL of a -#: file here at any ref, or ``microcosm/`` (after any ``./`` or ``../`` and -#: an optional ``PolicyEngine/``) before a top-level source directory. +#: file here, up to its ref (``url``), or ``microcosm/`` (after any ``./`` or +#: ``../`` and an optional ``PolicyEngine/``) before a top-level source +#: directory. _THIS_REPOSITORY = re.compile( - r"(?:https?:)?/*(?:www\.)?github\.com/(?i:policyengine/microcosm)/" - r"(?:blob|tree|raw|blame)/[^/]+/" - r"|(?:https?:)?/*raw\.githubusercontent\.com/(?i:policyengine/microcosm)/[^/]+/" + r"(?P(?:https?:)?/*(?:www\.)?github\.com/(?i:policyengine/microcosm)/" + r"(?:blob|tree|raw|blame)/" + r"|(?:https?:)?/*raw\.githubusercontent\.com/(?i:policyengine/microcosm)/)" r"|(?:\.\.?/)*(?:(?i:policyengine)/)?microcosm/(?=(?:" + "|".join(_SOURCE_ROOTS) + r")/)" @@ -437,10 +439,19 @@ def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: relative = PurePosixPath(file[named.end() :] if named else file) if relative.is_absolute() or ".." in relative.parts: return False - if named is not None and (root / relative).is_file(): - return True - return len(relative.parts) >= 2 and relative.as_posix() in _repository_path_tails( - root + candidates = [relative] + if named is not None and named["url"]: + # A URL's ref comes first and may span segments (``main``, + # ``refs/heads/main``, ``feature/x``), so try the path after each. + candidates = [ + PurePosixPath(*relative.parts[cut:]) + for cut in range(1, len(relative.parts)) + ] + tails = _repository_path_tails(root) + return any( + (named is not None and (root / candidate).is_file()) + or (len(candidate.parts) >= 2 and candidate.as_posix() in tails) + for candidate in candidates ) From 4229cd8c685ca9a8440be1072d32b003033ca1a5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 22:46:51 -0700 Subject: [PATCH 07/21] Read each URL as GitHub reads it before scanning for citations Round 4 of the PR 1407 review found two valid GitHub URLs that escaped the guard: `%74ools/x.py` lost the repository prefix during extraction, and a `?plain=1&review=...` query longer than the backstop's window hid the `#L12` locator. Each URL in a scanned string is now read the way GitHub reads it: scheme and host lower-cased, a user or port dropped, `%` escapes of unreserved characters and `/` decoded, dot segments resolved, and the query dropped. GitHub reads `%23` and `%3F` as part of a file name, so those stay escaped. The resolver also tried every split point of a URL's ref, so an 8,000-segment ref took 115 s and 1.09 GB. It now tries only the paths no longer than the deepest path form in the checkout. The new reading changes 69 live strings (queries on data-source URLs), none of which carries a citation: classification is unchanged across all 200 resources and 283 citations. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 186 ++++++++++++++++++ .../microcosm_build/spec_source_citations.py | 114 ++++++++++- 3 files changed, 291 insertions(+), 11 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index de63e0853d..c7859566d7 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way GitHub reads it first: its scheme and host lower-cased, a user or port dropped, `%` escapes of unreserved characters and `/` decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 674b0df266..5f28023cd7 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -45,6 +45,9 @@ import ast import inspect import json +import re +import string +from collections.abc import Callable from pathlib import Path, PurePosixPath import pandas as pd @@ -422,6 +425,15 @@ def test_an_unresolvable_dotted_symbol_raises( ) #: One segment of a git ref. _REF_SEGMENT = st.from_regex(r"[A-Za-z0-9_-]{1,12}", fullmatch=True) +#: A URL up to its ref: the scheme, the host, and the owner, repository and +#: route. +_URL_PARTS = re.compile(r"(?P(?:https?://)?)(?P[^/]+)(?P/.*)") +#: Detours in a URL path that GitHub resolves away. +_DOT_DETOURS = ("", "./", "%2e/", "docs/../", "x/%2E%2e/", "a/b/../../", "tools/./../") +#: A URL query, without its ``?``: anything but a fragment, a quote or a space. +_URL_QUERY = st.text( + alphabet=string.ascii_letters + string.digits + "=&.;:,%+-_~?/", max_size=120 +) def _plant(text: str, depth: int, as_key: bool) -> object: @@ -431,6 +443,27 @@ def _plant(text: str, depth: int, as_key: bool) -> object: return planted +def _escape(text: str, chosen: set[int], upper: bool, keep: str) -> str: + """``text`` with its characters at ``chosen`` written as ``%`` escapes.""" + + return "".join( + (f"%{ord(char):02X}" if upper else f"%{ord(char):02x}") + if index in chosen and char not in keep + else char + for index, char in enumerate(text) + ) + + +class _CountingPath(type(REPOSITORY_ROOT)): # type: ignore[misc] + """A path that counts the ``is_file`` checks made through it.""" + + checks = 0 + + def is_file(self, *, follow_symlinks: bool = True) -> bool: + type(self).checks += 1 + return super().is_file(follow_symlinks=follow_symlinks) + + def _is_commit_like(sha: str) -> bool: return any(c.isdigit() for c in sha) and any(c.isalpha() for c in sha) @@ -1255,6 +1288,159 @@ def test_the_round_three_review_counterexample_is_refused() -> None: ] +@settings(max_examples=300, deadline=None) +@given( + url=st.sampled_from(_THIS_REPOSITORY_URLS), + ref=st.one_of(_SHA, st.lists(_REF_SEGMENT, min_size=1, max_size=3).map("/".join)), + detour=st.sampled_from(_DOT_DETOURS), + path=st.sampled_from(_ROOTED_FILES), + chosen=st.sets(st.integers(0, 160), max_size=12), + upper=st.booleans(), + case=st.sampled_from((str.lower, str.upper, str.title)), + authority=st.sampled_from( + ("{host}", "{host}:443", "user@{host}", "u:p@{host}:443") + ), + query=st.one_of(st.just(""), _URL_QUERY.map("?".__add__)), + fragment=st.sampled_from(("#L{a}", "#L{a}-L{b}")), + pin=st.sampled_from(_INLINE_PINS), + sha=_SHA, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), +) +def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( + url: str, + ref: str, + detour: str, + path: str, + chosen: set[int], + upper: bool, + case: Callable[[str], str], + authority: str, + query: str, + fragment: str, + pin: str, + sha: str, + a: int, + b: int, +) -> None: + parts = _URL_PARTS.fullmatch(url) + assert parts is not None + scheme, host, route = parts["scheme"], parts["host"], parts["route"] + # A user or a port needs a scheme; GitHub 404s a ``%2F`` before the ref. + written = ( + case(scheme) + + (authority.format(host=case(host)) if scheme else case(host)) + + _escape(route, chosen, upper, keep="/%") + + _escape( + f"{ref}/{detour}{path}", + {index - len(route) for index in chosen}, + upper, + keep="%", + ) + ) + text = f"{written}{query}{fragment.format(a=a, b=b)} {pin.format(sha=sha)}" + found = in_repository_line_citations({"note": text}) + assert [citation.file for citation in found] == [f"{url}{ref}/{path}"] + # A citation's text is already read: reading it again changes nothing. + assert in_repository_line_citations({"note": found[0].text}) == found + + +@settings(max_examples=200, deadline=None) +@given( + directory=_FOREIGN_DIRECTORY, + stem=_STEM, + suffix=_SUFFIX, + query=_URL_QUERY, + fragment=st.sampled_from(("#L{a}", "#L{a}-L{b}")), + sha=_SHA, + pinned=st.booleans(), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), +) +def test_a_query_does_not_hide_a_line_citation( + directory: str, + stem: str, + suffix: str, + query: str, + fragment: str, + sha: str, + pinned: bool, + a: int, + b: int, +) -> None: + ref = sha if pinned else "main" + url = f"https://github.com/PolicyEngine/upstream-data/blob/{ref}/{directory}{stem}.{suffix}" + document = {"note": f"{url}?{query}{fragment.format(a=a, b=b)}"} + assert [ + (citation.file, citing_commit(citation)) + for citation in line_citations(document) + ] == [(url, sha if pinned else None)] + assert len(unpinned_line_citations(document)) == (0 if pinned else 1) + + +def test_the_round_four_review_counterexamples_are_refused() -> None: + blob = "https://github.com/PolicyEngine/microcosm/blob/" + blob += "1840bc92f8d155adb685f1090245058ebb990df0/" + tool = "tools/build_us_puf_support_base.py" + # GitHub served each of these (HTTP 200) on 2026-10-26. + for written in ( + f"{blob}%74ools/build_us_puf_support_base.py#L12", + f"{blob}{tool}?plain=1&review={'x' * 32}#L12", + f"{blob}tools%2Fbuild_us_puf_support_base.py#L12", + f"{blob}tools/build_us_puf_support_base%2Epy#L12", + f"{blob}docs/../{tool}#L12", + f"{blob}docs/%2e%2e/{tool}#L12", + f"{blob}./{tool}#L12", + blob.replace("github.com", "GitHub.com") + f"{tool}#L12", + blob.replace("github.com", "github.com:443") + f"{tool}#L12", + blob.replace("github.com", "user@github.com") + f"{tool}#L12", + blob.replace("/blob/", "/%62lob/") + f"{tool}#L12", + ): + assert [ + citation.file + for citation in in_repository_line_citations({"note": written}) + ] == [f"{blob}{tool}"], written + path = blob.removeprefix("https://github.com") + for written, file in ( + (f"see GitHub.com{path}{tool}#L12.", f"github.com{path}{tool}"), + ( + f"//github.com{path}{tool}?plain=1&review={'x' * 32}#L12", + f"//github.com{path}{tool}", + ), + ): + assert [ + citation.file + for citation in in_repository_line_citations({"note": written}) + ] == [file], written + # GitHub reads %23 and %3F as part of a file name (HTTP 404). + for written in (f"{blob}{tool}%23L12", f"{blob}{tool}%3Fplain=1"): + assert line_citations({"note": written}) == [], written + # Punctuation after a URL is prose, so the clause still ends at the ``;``. + assert unpinned_line_citations( + {"note": "x.py:12, see https://example.org/doc?page=1; at a1b2c3d"} + ) == [LineCitation("x.py", "x.py:12, see https://example.org/doc; at a1b2c3d")] + gitlab = "https://gitlab.com/o/upstream-data/-/blob/a1b2c3d/x.py" + text = f"{gitlab}?ref_type=heads&view=plain&page=2 some_function, lines 12-19" + assert unrecognised_line_locators({"note": text}) == [ + LineCitation(gitlab, f"{gitlab} some_function, lines 12-19") + ] + + +def test_a_long_ref_is_searched_only_as_deep_as_the_deepest_path_here( + tmp_path: Path, +) -> None: + deepest = "tools/lanes/emtr/check.py" + (tmp_path / deepest).parent.mkdir(parents=True) + (tmp_path / deepest).write_text("", encoding="utf-8") + root = _CountingPath(tmp_path) + url = "https://github.com/PolicyEngine/microcosm/blob/" + "x/" * 500 + assert resolves_in_repository(url + deepest, root) + _CountingPath.checks = 0 + assert not resolves_in_repository(url + "tools/lanes/emtr/missing.py", root) + # One check per suffix of at most four segments, not one per ref segment. + assert _CountingPath.checks == 4 + + @settings(max_examples=200, deadline=None) @given( pin=st.sampled_from(_INLINE_PINS), diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 5a661f0c0d..f96bafbbc5 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -32,6 +32,14 @@ (``misspelled_commit_pins``), so a stray sha cannot sit beside a citation that some other commit pins. +Each URL is read the way GitHub reads it before anything is scanned +(``_read_url``): the scheme and host lower-cased, a user or port dropped, +``%`` escapes of unreserved characters and ``/`` decoded, ``.`` and ``..`` +segments resolved, and the query dropped. So +``HTTPS://GitHub.com/o/r/blob//docs/../%74ools/x%2Epy?plain=1#L12`` +cites ``https://github.com/o/r/blob//tools/x.py`` at line 12. GitHub +reads ``%23`` and ``%3F`` as part of a file name, so they stay escaped. + A cited path is in-repository when, after any prefix naming this repository is removed (a ``github.com/PolicyEngine/microcosm/blob//`` URL, at a ref of any depth such as ``refs/heads/main``, or ``microcosm/`` before a @@ -56,10 +64,12 @@ import json import os import re -from collections.abc import Iterator, Mapping +import string +from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field from functools import cache from pathlib import Path, PurePosixPath +from urllib.parse import urlsplit from microcosm.build.spec_engine import load_yaml12 from test_support.paths import REPOSITORY_ROOT @@ -130,6 +140,23 @@ + _QUOTE + r"[^.;\n]{0,24}?(?]*[^\s\"'`‘’“”<>.,;:!?)\]]" +) +#: The scheme a URL starts with, when it has one. +_SCHEME = re.compile(r"(?i:https?)://") +#: A ``%`` escape in a URL. +_ESCAPE = re.compile(r"%([0-9A-Fa-f]{2})") +#: The characters a ``%`` escape in a URL path is read as: the unreserved ones +#: (RFC 3986 section 2.3), and ``/``, which GitHub reads as a separator after +#: the ref (``tools%2Fx.py``). Any other escape stays as written. +_DECODED = frozenset(string.ascii_letters + string.digits + "-._~/") #: A commit sha: 7-40 lowercase hex with at least one digit and one letter, #: so a decimal amount ("2500000"), a word ("defaced") or a branch name such #: as ``main`` is not one. @@ -202,8 +229,9 @@ class LineCitation: """A line citation: the source as cited, and the string carrying it. - ``start`` and ``end`` locate the citation in ``text`` (-1 when built by - hand); they do not take part in equality. + Both read each URL as ``_read_url`` reads it. ``start`` and ``end`` + locate the citation in ``text`` (-1 when built by hand); they do not take + part in equality. """ file: str @@ -230,6 +258,62 @@ def nested_strings(value: object) -> Iterator[str]: yield value +def _decode(path: str) -> str: + """A URL path with each escape of a ``_DECODED`` character decoded.""" + + def read(escape: re.Match[str]) -> str: + character = chr(int(escape[1], 16)) + return character if character in _DECODED else escape[0] + + return _ESCAPE.sub(read, path) + + +def _remove_dot_segments(path: str) -> str: + """A URL path with its ``.`` and ``..`` segments resolved (RFC 3986 5.2.4).""" + + segments: list[str] = [] + for segment in path.split("/"): + if segment == "..": + if len(segments) > 1: + segments.pop() + elif segment != ".": + segments.append(segment) + if path.endswith(("/.", "/..")): + segments.append("") + return "/".join(segments) + + +def _read_url(url: re.Match[str]) -> str: + """A URL as GitHub reads it, without its query. + + The scheme and host are lower-cased, a user and a port dropped, the + path's ``_DECODED`` escapes decoded and its dot segments resolved; the + fragment (``#L12``) is kept. + """ + + text = url[0] + has_scheme = _SCHEME.match(text) is not None + try: + parts = urlsplit(text if has_scheme or text.startswith("//") else f"//{text}") + host = parts.hostname + except ValueError: + return text + if not host: + return text + authority = "//" if has_scheme or text.startswith("//") else "" + path = _remove_dot_segments(_decode(parts.path)) + fragment = f"#{parts.fragment}" if parts.fragment else "" + scheme = f"{parts.scheme}:" if has_scheme else "" + return f"{scheme}{authority}{host}{path}{fragment}" + + +def _readable_strings(value: object) -> Iterator[str]: + """``nested_strings``, with each URL in them read as ``_read_url`` reads it.""" + + for text in nested_strings(value): + yield _URL.sub(_read_url, text) + + def _citation_matches(text: str) -> list[re.Match[str]]: return sorted( (match for pattern in _LINE_CITATIONS for match in pattern.finditer(text)), @@ -242,7 +326,7 @@ def line_citations(value: object) -> list[LineCitation]: return [ LineCitation(match["file"], text, match.start(), match.end()) - for text in nested_strings(value) + for text in _readable_strings(value) for match in _citation_matches(text) ] @@ -251,7 +335,7 @@ def unrecognised_line_locators(value: object) -> list[LineCitation]: """Source names followed by a line locator in no recognised spelling.""" found: list[LineCitation] = [] - for text in nested_strings(value): + for text in _readable_strings(value): matches = _citation_matches(text) files = {match.start("file") for match in matches} spans = [(match.start(), match.end()) for match in matches] @@ -425,6 +509,15 @@ def _repository_path_tails(root: Path) -> frozenset[str]: return frozenset(tails) +@cache +def _repository_depth(root: Path) -> int: + """The most components a path form in ``root`` has.""" + + return max( + (tail.count("/") + 1 for tail in _repository_path_tails(root)), default=0 + ) + + def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: """Whether a cited path names a file in this checkout. @@ -439,14 +532,15 @@ def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: relative = PurePosixPath(file[named.end() :] if named else file) if relative.is_absolute() or ".." in relative.parts: return False - candidates = [relative] + candidates: Iterable[PurePosixPath] = [relative] if named is not None and named["url"]: # A URL's ref comes first and may span segments (``main``, # ``refs/heads/main``, ``feature/x``), so try the path after each. - candidates = [ - PurePosixPath(*relative.parts[cut:]) - for cut in range(1, len(relative.parts)) - ] + # No path form here is longer than ``_repository_depth`` segments, so + # only the paths that short are tried, however long the ref. + parts = relative.parts + first = max(1, len(parts) - _repository_depth(root)) + candidates = (PurePosixPath(*parts[cut:]) for cut in range(first, len(parts))) tails = _repository_path_tails(root) return any( (named is not None and (root / candidate).is_file()) From c67030f75e07737f6aeb34841d2190a8009ec351 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Mon, 26 Oct 2026 23:08:03 -0700 Subject: [PATCH 08/21] Escape only characters outside an existing escape in the URL property The new URL property could escape the `2` of a `%2E` detour, writing `%%32E`, which names a different path from the one it expects refused. The mutant harness's unmutated baseline caught it (raw URL, ref 000000a, detour `x/%2E%2e/`). Characters already inside an escape now stay as written; the helper is unchanged. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_spec_source_citations.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 5f28023cd7..3b58656592 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -443,12 +443,21 @@ def _plant(text: str, depth: int, as_key: bool) -> object: return planted -def _escape(text: str, chosen: set[int], upper: bool, keep: str) -> str: - """``text`` with its characters at ``chosen`` written as ``%`` escapes.""" +def _escape(text: str, chosen: set[int], upper: bool, keep: str = "") -> str: + """``text`` with its characters at ``chosen`` written as ``%`` escapes. + A character in ``keep``, or in an escape already there, stays as written: + escaping the ``2`` of ``%2E`` would name another path (``%%32E``). + """ + + escaped = { + index + for escape in re.finditer(r"%[0-9A-Fa-f]{2}", text) + for index in range(escape.start(), escape.end()) + } return "".join( (f"%{ord(char):02X}" if upper else f"%{ord(char):02x}") - if index in chosen and char not in keep + if index in chosen and index not in escaped and char not in keep else char for index, char in enumerate(text) ) @@ -1330,12 +1339,11 @@ def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( written = ( case(scheme) + (authority.format(host=case(host)) if scheme else case(host)) - + _escape(route, chosen, upper, keep="/%") + + _escape(route, chosen, upper, keep="/") + _escape( f"{ref}/{detour}{path}", {index - len(route) for index in chosen}, upper, - keep="%", ) ) text = f"{written}{query}{fragment.format(a=a, b=b)} {pin.format(sha=sha)}" From 29983852e5182266c9a4e565731920ea972fe695 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 00:28:13 -0700 Subject: [PATCH 09/21] Read each URL's authority as the WHATWG URL Standard reads it Round 5's review found that a URL of this repository with an escaped host (https://%67ithub.com/...) or a network-path reference with a port or a user (//github.com:443/..., //user@github.com/.../%74ools/...) escaped the in-repository rule. urlsplit kept the host's escapes, and the URL pattern read a scheme-less GitHub URL only when a slash followed its host. The reader now parses the authority as the URL Standard does. Before the query, \ is /. Any number of slashes may follow https: or start a network-path reference. The user (up to the last @) and the port are dropped. The host is percent-decoded, mapped to ASCII as UTS46 maps it (NFKC case folding with Python's Unicode 16, plus Unicode 17's U+A7F1) and lower-cased. A trailing dot is dropped, since GitHub redirects github.com. to github.com. A URL the standard refuses is read as written. A scheme-less GitHub URL may now carry a user and a port. Checked against bun's WHATWG URL with every code point escaped into a host: the two read the same wherever the standard maps it into plain ASCII. Of 26,350 generated URLs, none is read only by the standard and none is read differently. The live packages' 4,482,233 strings read exactly as before. Tests: - the review's three inputs and nine more spellings, plus four authorities the standard refuses; - a table of 20 readings recorded from the standard; - the URL property now writes 12 leads, 7 authorities, full-width and escaped host characters, trailing dots and backslash separators; - a deterministic example of a clause break inside a citation (M27). Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 180 ++++++++++++++++-- .../microcosm_build/spec_source_citations.py | 143 +++++++++++--- 3 files changed, 278 insertions(+), 47 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index c7859566d7..1e9d45f331 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way GitHub reads it first: its scheme and host lower-cased, a user or port dropped, `%` escapes of unreserved characters and `/` decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 3b58656592..8160dcdbcd 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -425,9 +425,34 @@ def test_an_unresolvable_dotted_symbol_raises( ) #: One segment of a git ref. _REF_SEGMENT = st.from_regex(r"[A-Za-z0-9_-]{1,12}", fullmatch=True) -#: A URL up to its ref: the scheme, the host, and the owner, repository and -#: route. -_URL_PARTS = re.compile(r"(?P(?:https?://)?)(?P[^/]+)(?P/.*)") +#: A URL up to its ref: the host, and the owner, repository and route (its +#: scheme is dropped; ``_URL_LEADS`` writes one). +_URL_PARTS = re.compile(r"(?:https?://)?(?P[^/]+)(?P/.*)") +#: Ways to write the start of a URL, each with the start it is read as. +_URL_LEADS = ( + ("https://", "https://"), + ("http://", "http://"), + ("https:", "https://"), + ("https:/", "https://"), + ("https:///", "https://"), + ("https:\\\\", "https://"), + ("http:\\/", "http://"), + ("//", "//"), + ("\\\\", "//"), + ("/\\", "//"), + ("///", "//"), + ("", ""), +) +#: Authorities around a host: a user (up to the last ``@``) and a port. +_AUTHORITIES = ( + "{host}", + "{host}:443", + "{host}:", + "{host}:0443", + "user@{host}", + "u:p@{host}:443", + "a@b@{host}", +) #: Detours in a URL path that GitHub resolves away. _DOT_DETOURS = ("", "./", "%2e/", "docs/../", "x/%2E%2e/", "a/b/../../", "tools/./../") #: A URL query, without its ``?``: anything but a fragment, a quote or a space. @@ -463,6 +488,22 @@ def _escape(text: str, chosen: set[int], upper: bool, keep: str = "") -> str: ) +def _spell_host(host: str, wide: set[int], chosen: set[int], upper: bool) -> str: + """``host`` with its characters at ``wide`` written full-width (``ｇ``) + and those at ``chosen`` as ``%`` escapes of their UTF-8 bytes.""" + + spelled = [] + for index, char in enumerate(host): + if index in wide: + char = chr(ord(char) + 0xFEE0) + if index in chosen: + char = "".join( + f"%{byte:02X}" if upper else f"%{byte:02x}" for byte in char.encode() + ) + spelled.append(char) + return "".join(spelled) + + class _CountingPath(type(REPOSITORY_ROOT)): # type: ignore[misc] """A path that counts the ``is_file`` checks made through it.""" @@ -1306,9 +1347,12 @@ def test_the_round_three_review_counterexample_is_refused() -> None: chosen=st.sets(st.integers(0, 160), max_size=12), upper=st.booleans(), case=st.sampled_from((str.lower, str.upper, str.title)), - authority=st.sampled_from( - ("{host}", "{host}:443", "user@{host}", "u:p@{host}:443") - ), + leads=st.sampled_from(_URL_LEADS), + authority=st.sampled_from(_AUTHORITIES), + wide=st.sets(st.integers(0, 24), max_size=6), + escaped_host=st.sets(st.integers(0, 24), max_size=6), + dot=st.booleans(), + backslashed=st.sets(st.integers(0, 160), max_size=6), query=st.one_of(st.just(""), _URL_QUERY.map("?".__add__)), fragment=st.sampled_from(("#L{a}", "#L{a}-L{b}")), pin=st.sampled_from(_INLINE_PINS), @@ -1324,7 +1368,12 @@ def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( chosen: set[int], upper: bool, case: Callable[[str], str], + leads: tuple[str, str], authority: str, + wide: set[int], + escaped_host: set[int], + dot: bool, + backslashed: set[int], query: str, fragment: str, pin: str, @@ -1334,21 +1383,29 @@ def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( ) -> None: parts = _URL_PARTS.fullmatch(url) assert parts is not None - scheme, host, route = parts["scheme"], parts["host"], parts["route"] - # A user or a port needs a scheme; GitHub 404s a ``%2F`` before the ref. + host, route = parts["host"], parts["route"] + written_lead, lead = leads + # A browser reads a host's escapes and full-width letters; written without + # a scheme or slashes, a host is GitHub's only as spelled. + spelled = case(host) + if written_lead: + spelled = _spell_host(spelled, wide, escaped_host, upper) + # GitHub 404s a ``%2F`` before the ref; a browser reads ``\`` as ``/``. + route_and_path = _escape(route, chosen, upper, keep="/") + _escape( + f"{ref}/{detour}{path}", {index - len(route) for index in chosen}, upper + ) + route_and_path = "".join( + "\\" if char == "/" and index in backslashed else char + for index, char in enumerate(route_and_path) + ) written = ( - case(scheme) - + (authority.format(host=case(host)) if scheme else case(host)) - + _escape(route, chosen, upper, keep="/") - + _escape( - f"{ref}/{detour}{path}", - {index - len(route) for index in chosen}, - upper, - ) + case(written_lead) + + authority.format(host=spelled + ("." if dot else "")) + + route_and_path ) text = f"{written}{query}{fragment.format(a=a, b=b)} {pin.format(sha=sha)}" found = in_repository_line_citations({"note": text}) - assert [citation.file for citation in found] == [f"{url}{ref}/{path}"] + assert [citation.file for citation in found] == [f"{lead}{host}{route}{ref}/{path}"] # A citation's text is already read: reading it again changes nothing. assert in_repository_line_citations({"note": found[0].text}) == found @@ -1434,6 +1491,95 @@ def test_the_round_four_review_counterexamples_are_refused() -> None: ] +def test_the_round_five_review_counterexamples_are_refused() -> None: + route = "/PolicyEngine/microcosm/blob/c67030f75e07737f6aeb34841d2190a8009ec351/" + tool = "tools/ci_test_plan.py" + escaped = "%74ools/ci_test_plan.py" + backslashed = (route + tool).replace("/", "\\") + for written, lead in ( + (f"https://%67ithub.com{route}{tool}#L12", "https://"), + (f"//github.com:443{route}{tool}#L12", "//"), + (f"//user@github.com{route}{escaped}#L12", "//"), + (f"https:github.com{route}{tool}#L12", "https://"), + (f"HTTPS:\\\\GitHub.com{backslashed}#L12", "https://"), + (f"\\\\github.com{route}{tool}#L12", "//"), + (f"///github.com{route}{tool}#L12", "//"), + (f"https://ｇithub.com{route}{tool}#L12", "https://"), + (f"https://raw.githubu\ua7f1ercontent.com{route}{tool}#L12", "https://"), + (f"https://github.com.{route}{tool}#L12", "https://"), + (f"https://u:p@GITHUB.COM:0443{route}{tool}#L12", "https://"), + (f"a@b@github.com:{route}{escaped}#L12", ""), + ): + host = "raw.githubusercontent.com" if "raw." in written else "github.com" + assert [ + citation.file + for citation in in_repository_line_citations({"note": written}) + ] == [f"{lead}{host}{route}{tool}"], written + # The URL Standard refuses each of these authorities, so no browser opens + # them: each is read as written, and its path names no file here. + for written in ( + f"https://%2567ithub.com{route}{tool}#L12", + f"https://user%40github.com{route}{tool}#L12", + f"https://github.com:44x{route}{tool}#L12", + f"https://github.com:65536{route}{tool}#L12", + ): + assert in_repository_line_citations({"note": written}) == [], written + + +#: URLs and how they are read: the WHATWG URL Standard's reading (bun 1.3's +#: ``URL``, against an ``https`` page for a network-path reference) without +#: its query, a network-path reference kept as one. Where the standard refuses +#: a URL it is read as written. +_WHATWG_READINGS = ( + ("https://%67ithub.com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("//github.com:443/o/r/blob/a1b2c3d/x.py#L12", "//github.com"), + ("\\\\user@GitHub.com\\o\\r\\blob\\a1b2c3d\\x.py#L12", "//github.com"), + ("https:github.com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("HTTP:///github.com/o/r/blob/a1b2c3d/x.py?plain=1#L12", "http://github.com"), + ("https://ｇithub．com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://%EF%BD%87ithub.com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://gi%C2%ADthub.com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://github.c\u2092m/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://git\u210cub.com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://a@b@github.com:/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://GITHUB.COM:0443/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://Ex%61mple.ORG:8443/o/r/blob/a1b2c3d/x.py#L12", "https://example.org"), + ("https://[::1]:8080/o/r/blob/a1b2c3d/x.py#L12", "https://[::1]"), + ("https://%2567ithub.com/o/r/blob/a1b2c3d/x.py#L12", None), + ("https://user%40github.com/o/r/blob/a1b2c3d/x.py#L12", None), + ("https://github.com%2F/o/r/blob/a1b2c3d/x.py#L12", None), + ("https://gi%FFthub.com/o/r/blob/a1b2c3d/x.py#L12", None), + ("https://github.com:44x/o/r/blob/a1b2c3d/x.py#L12", None), + ("https://github.com:65536/o/r/blob/a1b2c3d/x.py#L12", None), +) + + +@pytest.mark.parametrize(("written", "authority"), _WHATWG_READINGS) +def test_a_url_is_read_as_the_url_standard_reads_it( + written: str, authority: str | None +) -> None: + reading = written if authority is None else f"{authority}/o/r/blob/a1b2c3d/x.py#L12" + assert [citation.text for citation in line_citations({"note": written})] == [ + reading + ] + + +def test_a_trailing_dot_is_dropped_from_a_host() -> None: + # The standard keeps it, but DNS ignores it and GitHub redirects + # ``github.com.`` to ``github.com`` (HTTP 301 on 2026-10-27). + written = "https://github.com./o/r/blob/a1b2c3d/x.py#L12" + assert [citation.text for citation in line_citations({"note": written})] == [ + "https://github.com/o/r/blob/a1b2c3d/x.py#L12" + ] + + +def test_a_break_inside_a_citation_does_not_end_its_clause() -> None: + for text in ("x.py; lines 12-19 at a1b2c3d", "x.py; lines 12-19, commit a1b2c3d"): + assert [citing_commit(found) for found in line_citations(text)] == [ + "a1b2c3d" + ], text + + def test_a_long_ref_is_searched_only_as_deep_as_the_deepest_path_here( tmp_path: Path, ) -> None: diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index f96bafbbc5..2498031783 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -32,10 +32,14 @@ (``misspelled_commit_pins``), so a stray sha cannot sit beside a citation that some other commit pins. -Each URL is read the way GitHub reads it before anything is scanned -(``_read_url``): the scheme and host lower-cased, a user or port dropped, -``%`` escapes of unreserved characters and ``/`` decoded, ``.`` and ``..`` -segments resolved, and the query dropped. So +Each URL is read the way a browser and then GitHub read it before anything +is scanned (``_read_url``). Its authority is parsed as the WHATWG URL +Standard parses it: ``\\`` read as ``/``, any number of slashes after the +scheme, a user and a port dropped, and the host percent-decoded, mapped to +ASCII (IDNA) and lower-cased, so ``%67ithub.com``, ``ｇithub.com`` and +``user@GitHub.com:443`` are all ``github.com``. Then, as GitHub does, ``%`` +escapes of unreserved characters and ``/`` in its path are decoded, ``.`` and +``..`` segments resolved, and the query dropped. So ``HTTPS://GitHub.com/o/r/blob//docs/../%74ools/x%2Epy?plain=1#L12`` cites ``https://github.com/o/r/blob//tools/x.py`` at line 12. GitHub reads ``%23`` and ``%3F`` as part of a file name, so they stay escaped. @@ -65,11 +69,12 @@ import os import re import string +import unicodedata from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field from functools import cache from pathlib import Path, PurePosixPath -from urllib.parse import urlsplit +from urllib.parse import unquote_to_bytes from microcosm.build.spec_engine import load_yaml12 from test_support.paths import REPOSITORY_ROOT @@ -140,17 +145,38 @@ + _QUOTE + r"[^.;\n]{0,24}?(?/\\?#]*@)?" + r"(?i:(?:www\.)?github\.com|raw\.githubusercontent\.com)\.?(?::[0-9]*)?(?=[/\\]))" r"[^\s\"'`‘’“”<>]*[^\s\"'`‘’“”<>.,;:!?)\]]" ) #: The scheme a URL starts with, when it has one. -_SCHEME = re.compile(r"(?i:https?)://") +_SCHEME = re.compile(r"(?i:https?):") +#: A URL up to its query or fragment. +_BEFORE_QUERY = re.compile(r"[^?#]*") +#: A port after a host: ASCII digits, if any. +_PORT = re.compile(r"(?::(?P[0-9]*))?") +#: The characters the WHATWG URL Standard refuses in a host once it is +#: decoded and mapped to ASCII (its forbidden domain code points). +_FORBIDDEN_IN_HOST = re.compile(r"[\x00-\x20#%/:<>?@\[\\\]^|\x7f]") +#: The default-ignorable code points, which UTS46 drops from a host or +#: refuses in one. +_IGNORABLE_IN_HOST = re.compile( + "[\u00ad\u034f\u061c\u115f\u1160\u17b4\u17b5\u180b-\u180f\u200b-\u200f" + "\u202a-\u202e\u2060-\u206f\u3164\ufe00-\ufe0f\ufeff\uffa0\ufff0-\ufff8" + "\U0001bca0-\U0001bca3\U0001d173-\U0001d17a\U000e0000-\U000e0fff]" +) +#: The code points UTS46 maps into ASCII that this Python's Unicode database +#: (16.0) does not know: Unicode 17's U+A7F1 MODIFIER LETTER CAPITAL S. +_MAPPED_AFTER_UNICODE_16 = str.maketrans({"\ua7f1": "s"}) #: A ``%`` escape in a URL. _ESCAPE = re.compile(r"%([0-9A-Fa-f]{2})") #: The characters a ``%`` escape in a URL path is read as: the unreserved ones @@ -283,28 +309,87 @@ def _remove_dot_segments(path: str) -> str: return "/".join(segments) -def _read_url(url: re.Match[str]) -> str: - """A URL as GitHub reads it, without its query. +def _to_ascii(domain: str) -> str: + """A decoded host mapped to ASCII as UTS46 (IDNA) maps it. - The scheme and host are lower-cased, a user and a port dropped, the - path's ``_DECODED`` escapes decoded and its dot segments resolved; the - fragment (``#L12``) is kept. + Default-ignorable code points are dropped, the rest NFKC case-folded + (``ｇ`` and ``ₒ`` are ``g`` and ``o``), ``。`` read as ``.``, and a label + left outside ASCII written in Punycode. Where UTS46 refuses a character + this reads it anyway, so it never misses a host UTS46 reads. """ - text = url[0] - has_scheme = _SCHEME.match(text) is not None + domain = _IGNORABLE_IN_HOST.sub("", domain).translate(_MAPPED_AFTER_UNICODE_16) + folded = unicodedata.normalize( + "NFKC", unicodedata.normalize("NFKC", domain).casefold() + ).replace("\u3002", ".") + return ".".join( + label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii") + for label in folded.split(".") + ) + + +def _read_host(authority: str) -> str | None: + """The host in a URL's authority, as the WHATWG URL Standard reads it. + + The user (up to the last ``@``) and the port are dropped; the host is + percent-decoded and mapped to ASCII by ``_to_ascii``. A trailing dot, + which DNS ignores, is dropped too: GitHub redirects ``github.com.`` to + ``github.com``. ``None`` when the standard refuses the authority: a port + that is not a number up to 65535, an empty host, or a host that is not + UTF-8 or holds a ``_FORBIDDEN_IN_HOST`` character once read. + """ + + host = authority.rpartition("@")[2] + if host.startswith("["): + # An IPv6 address, kept as written: GitHub serves no URL by address. + address, bracket, port = host.partition("]") + host = address + bracket if bracket else "" + else: + host, colon, port = host.partition(":") + port = colon + port + number = _PORT.fullmatch(port) + if not host or number is None: + return None + digits = (number["number"] or "").lstrip("0") + if len(digits) > 5 or int(digits or 0) > 65535: + return None + if host.startswith("["): + return host.lower() try: - parts = urlsplit(text if has_scheme or text.startswith("//") else f"//{text}") - host = parts.hostname - except ValueError: - return text - if not host: + host = _to_ascii(unquote_to_bytes(host).decode("utf-8")) + except UnicodeError: + return None + if _FORBIDDEN_IN_HOST.search(host): + return None + return host.removesuffix(".") + + +def _read_url(url: re.Match[str]) -> str: + """A URL as a browser and then GitHub read it, without its query. + + An ``http(s)`` URL, or a network-path reference read against an + ``https`` page, is parsed as the WHATWG URL Standard parses it: ``\\`` + before the query is ``/``, any number of slashes may come before the + authority, and ``_read_host`` reads the host. A GitHub URL written + without its scheme is read as if ``https://`` came first. Then, as GitHub + does, the path's ``_DECODED`` escapes are decoded and its dot segments + resolved; the fragment (``#L12``) is kept. A URL the standard refuses is + left as written. + """ + + text = url[0] + scheme = _SCHEME.match(text) + rest = text[scheme.end() :] if scheme else text + before = _BEFORE_QUERY.match(rest)[0].replace("\\", "/") + slashes = len(before) - len(before.lstrip("/")) + authority, slash, path = before[slashes:].partition("/") + host = _read_host(authority) + if host is None: return text - authority = "//" if has_scheme or text.startswith("//") else "" - path = _remove_dot_segments(_decode(parts.path)) - fragment = f"#{parts.fragment}" if parts.fragment else "" - scheme = f"{parts.scheme}:" if has_scheme else "" - return f"{scheme}{authority}{host}{path}{fragment}" + lead = f"{scheme[0].lower()}//" if scheme else "//" if slashes else "" + path = _remove_dot_segments(_decode(slash + path)) + fragment = rest.partition("#")[2] + return f"{lead}{host}{path}{'#' + fragment if fragment else ''}" def _readable_strings(value: object) -> Iterator[str]: From 5f6647b14e612863ebe15bbd03c8da4d7eb1a04d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 02:04:49 -0700 Subject: [PATCH 10/21] Read a quote inside a URL as part of it, and scan long strings in linear time Review round 6 found that a URL of this repository with a quote in its query (`.../tools/ci_test_plan.py?value='xxxx'#L12`) escaped the in-repository rule: the URL token ended at the quote, so its `#L12` was read as prose. A browser reads a quote or a bracket inside a URL as part of it (bun's URL), and GitHub serves each such URL. A URL now runs to whitespace and does not end in a quote, a bracket or sentence punctuation. The guard's scans are now linear in the string: - a GitHub host written without a scheme is matched where it starts, with any user before it left as prose, instead of scanning for a user from every character of a run; - a source-name pattern is tried only where a match can start (the start of a run of path characters, `http(s):`, or the previous match's end), which finds the same matches as `finditer`; - each string's clause breaks, inline commits and commit-like tokens are found once, and each citation's clause, nearest commits and tokens are found by bisection; the moving-ref check is cached per clause; - the backstop checks whether a near match ends inside a citation by bisection. Tests: the reviewer's input with each quote and bracket in a query or a dot segment; the URL property writes quotes and brackets in queries and dot detours, and its full-width, escape and backslash indices always land; a timing test of eight hostile strings (0.16 s; the round-6 helper takes over 90 s on five of them); a differential of the run-start scan against `finditer`, and of the bindings, misspelled pins and backstop against the whole-string scans they replace. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 268 +++++++++++++++++- .../microcosm_build/spec_source_citations.py | 147 +++++++--- 3 files changed, 377 insertions(+), 40 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index 1e9d45f331..b62900304e 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 8160dcdbcd..9e75c64f5d 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -47,12 +47,13 @@ import json import re import string +import time from collections.abc import Callable from pathlib import Path, PurePosixPath import pandas as pd import pytest -from hypothesis import assume, given, settings +from hypothesis import assume, example, given, settings from hypothesis import strategies as st from microcosm.build.us_runtime import asec_pool @@ -60,6 +61,7 @@ ASEC_CENSUS_PERSON_COLUMN_NAMES, ASEC_CENSUS_PERSON_COLUMNS, ) +from test_support.microcosm_build import spec_source_citations as citations from test_support.microcosm_build.spec_source_citations import ( COUNTRY_PACKAGE_ROOT, MICROCOSM_SHARDS, @@ -453,11 +455,28 @@ def test_an_unresolvable_dotted_symbol_raises( "u:p@{host}:443", "a@b@{host}", ) -#: Detours in a URL path that GitHub resolves away. -_DOT_DETOURS = ("", "./", "%2e/", "docs/../", "x/%2E%2e/", "a/b/../../", "tools/./../") -#: A URL query, without its ``?``: anything but a fragment, a quote or a space. +#: Detours in a URL path that GitHub resolves away, quotes and brackets in +#: them included. +_DOT_DETOURS = ( + "", + "./", + "%2e/", + "docs/../", + "x/%2E%2e/", + "a/b/../../", + "tools/./../", + "'x'/../", + '"/../', + "/../", + "`x`/../", +) +#: A URL query, without its ``?``: anything but a fragment or a space, quotes +#: and brackets included. _URL_QUERY = st.text( - alphabet=string.ascii_letters + string.digits + "=&.;:,%+-_~?/", max_size=120 + alphabet=string.ascii_letters + + string.digits + + "=&.;:,%+-_~?/'\"`‘’“”<>!$()*[]{}|^", + max_size=120, ) @@ -1389,13 +1408,20 @@ def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( # a scheme or slashes, a host is GitHub's only as spelled. spelled = case(host) if written_lead: - spelled = _spell_host(spelled, wide, escaped_host, upper) + spelled = _spell_host( + spelled, + {index % len(host) for index in wide}, + {index % len(host) for index in escaped_host}, + upper, + ) # GitHub 404s a ``%2F`` before the ref; a browser reads ``\`` as ``/``. route_and_path = _escape(route, chosen, upper, keep="/") + _escape( f"{ref}/{detour}{path}", {index - len(route) for index in chosen}, upper ) + slashes = [index for index, char in enumerate(route_and_path) if char == "/"] + backslashed = {slashes[index % len(slashes)] for index in backslashed} route_and_path = "".join( - "\\" if char == "/" and index in backslashed else char + "\\" if index in backslashed else char for index, char in enumerate(route_and_path) ) written = ( @@ -1526,6 +1552,234 @@ def test_the_round_five_review_counterexamples_are_refused() -> None: assert in_repository_line_citations({"note": written}) == [], written +def test_the_round_six_review_counterexample_is_refused() -> None: + blob = "https://github.com/PolicyEngine/microcosm/blob/" + blob += "29983852e5182266c9a4e565731920ea972fe695/" + tool = "tools/ci_test_plan.py" + # A browser reads a quote or a bracket inside a URL as part of it (bun + # 1.3's ``URL``), and GitHub served each of these, escaped as a browser + # sends them (HTTP 200 on 2026-10-27). + for written in ( + f"{blob}{tool}?value='{'x' * 40}'#L12", + f'{blob}{tool}?value="x"#L12', + f"{blob}{tool}?value=`x`#L12", + f"{blob}{tool}?value=‘x’“x”#L12", + f"{blob}{tool}?value=#L12", + f"{blob}tools/'x'/../ci_test_plan.py#L12", + f"{blob}tools//../ci_test_plan.py#L12", + f"'{blob}{tool}#L12'", + f"“{blob}{tool}?value='x'#L12”", + f"<{blob}{tool}#L12>", + f"({blob}{tool}?value=(x)#L12).", + ): + assert [ + citation.file + for citation in in_repository_line_citations({"note": written}) + ] == [f"{blob}{tool}"], written + + +def test_a_url_runs_to_whitespace() -> None: + # As the URL Standard reads it (bun 1.3's ``URL``), a URL written straight + # after another, with only a quote between, is part of the first one's + # path: its host is read as written. + written = "https://example.org/a'https://GitHub.com/o/r/blob/a1b2c3d/x.py#L12" + assert [citation.text for citation in line_citations({"note": written})] == [ + written + ] + + +def test_a_long_string_is_scanned_in_linear_time() -> None: + # Each of these once took time quadratic in its length: a run of path + # characters was rescanned from each of its characters, and a user was + # looked for before a GitHub host from each character of a run that can + # be in one. At these lengths the quadratic scans took seconds each. + started = time.process_time() + for text in ( + ":" * 60_000, + "//" * 30_000, + "a" * 60_000, + "github.com/" * 6_000, + "https://a/" + "%74" * 20_000, + ): + line_citations(text) + unrecognised_line_locators(text) + # So did a string holding thousands of citations: each one's clause, + # commits and commit-like tokens were found again from the whole string. + for text in ( + "x.py:1 " * 8_000, + "x.py:1 at a1b2c3d " * 4_000, + "a1b2c3d; " + "x.py:1; " * 8_000, + "x.py:1 y.py foo L1 " * 4_000, + ): + misspelled_commit_pins(text) + unpinned_line_citations(text) + unrecognised_line_locators(text) + assert time.process_time() - started < 2 + + +#: Pieces of text around source names and line locators. +_LOCATOR_PIECES = ( + "x", + "a/b", + "tools", + ".py", + ".json", + ".", + "/", + "-", + "_", + ":", + "12", + "L3", + " ", + "lines ", + "l.", + "https:", + "//", + "'", + "`", + ", ", + "of ", + "#", + "@", + ";", + "\n", + "at ", + "(", + "—", +) + + +@settings(max_examples=500, deadline=None) +@given(text=st.lists(st.sampled_from(_LOCATOR_PIECES), max_size=40).map("".join)) +@example(text="x.py:12/y.py:13") +@example(text="xhttps://a/b.py:12") +def test_source_names_are_found_where_finditer_finds_them(text: str) -> None: + for pattern in (*citations._LINE_CITATIONS, citations._LOCATOR_NEAR_A_SOURCE): + assert [ + (match.span(), match["file"]) + for match in citations._finditer(pattern, text) + ] == [(match.span(), match["file"]) for match in pattern.finditer(text)] + + +def _clause_from_the_whole_string(citation: LineCitation) -> tuple[int, int]: + first, last = 0, len(citation.text) + for clause_break in citations._CLAUSE_BREAK.finditer(citation.text): + if clause_break.end() <= citation.start: + first = clause_break.end() + elif clause_break.start() >= citation.end: + last = clause_break.start() + break + return first, last + + +def _binding_from_the_whole_string(citation: LineCitation) -> re.Match[str] | None: + commits = list(citations.INLINE_COMMIT.finditer(citation.text)) + start, end = _clause_from_the_whole_string(citation) + in_clause = [match for match in commits if start <= match.start() < end] + if in_clause: + return min( + in_clause, + key=lambda match: max( + match.start() - citation.end, citation.start - match.end() + ), + ) + if citations._MOVING_REF.search(citation.text, start, end): + return None + before = [match for match in commits if match.start() < start] + return before[-1] if before else None + + +def _misspelled_pins_from_the_whole_string(text: str) -> list[tuple[LineCitation, str]]: + found: list[tuple[LineCitation, str]] = [] + for citation in line_citations(text): + pinned = { + match.start("sha") for match in citations.INLINE_COMMIT.finditer(text) + } + start, end = _clause_from_the_whole_string(citation) + binding = _binding_from_the_whole_string(citation) + if binding is not None and binding.end() < start: + start = binding.end() + found.extend( + (citation, token.group(0)) + for token in citations._COMMIT_LIKE.finditer(citation.text, start, end) + if token.start() not in pinned + ) + return found + + +def _unrecognised_from_the_whole_string(text: str) -> list[LineCitation]: + found: list[LineCitation] = [] + for readable in citations._readable_strings(text): + matches = citations._citation_matches(readable) + files = {match.start("file") for match in matches} + spans = [(match.start(), match.end()) for match in matches] + for near in citations._LOCATOR_NEAR_A_SOURCE.finditer(readable): + if near.start("file") in files or any( + start <= near.end() < end for start, end in spans + ): + continue + found.append(LineCitation(near["file"], readable, near.start(), near.end())) + return found + + +#: Pieces of text around citations: pins in each spelling, commit-like tokens +#: in none, moving refs and clause breaks. +_BINDING_PIECES = ( + "x.py:12", + "tools/x.py L3", + "lines 4-5 of y.json", + "x.py foo L9", + "x.py, lines 1-2", + "x.py; lines 6-7", + " at a1b2c3d", + " commit 1a2b3c4d", + "@b2c3d4e", + "/blob/c3d4e5f/", + "a1b2c3d", + " 9f8e7d6c", + " on main", + "@HEAD", + "/blob/main/", + "https://github.com/o/r/blob/d4e5f6a/z.py#L7", + "https://github.com/o/r/blob/main/z.py#L7", + "; ", + ";", + ". See ", + "! Then ", + " ", + ", ", + " and ", + "(", + "'", +) + + +@settings(max_examples=500, deadline=None) +@given(text=st.lists(st.sampled_from(_BINDING_PIECES), max_size=30).map("".join)) +@example(text="at a1b2c3d x.py:12 at b2c3d4e") +@example(text=";x.py:12 at a1b2c3d") +@example(text="https://github.com/o/r/blob/d4e5f6a/z.py#L7 at a1b2c3d") +@example(text="a1b2c3d; x.py:12; 9f8e7d6c y.py:3 on main") +@example(text="x.py:1 y.py foo L1 x.py, lines 1-2 z.py foo L2") +def test_bindings_are_found_as_a_scan_of_the_whole_string_finds_them( + text: str, +) -> None: + found = line_citations(text) + assert [citing_commit(citation) for citation in found] == [ + None if binding is None else binding["sha"] + for binding in map(_binding_from_the_whole_string, found) + ] + assert misspelled_commit_pins(text) == _misspelled_pins_from_the_whole_string(text) + assert [ + (citation, citation.start, citation.end) + for citation in unrecognised_line_locators(text) + ] == [ + (citation, citation.start, citation.end) + for citation in _unrecognised_from_the_whole_string(text) + ] + + #: URLs and how they are read: the WHATWG URL Standard's reading (bun 1.3's #: ``URL``, against an ``https`` page for a network-path reference) without #: its query, a network-path reference kept as one. Where the standard refuses diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 2498031783..8d113258bc 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -42,7 +42,9 @@ ``..`` segments resolved, and the query dropped. So ``HTTPS://GitHub.com/o/r/blob//docs/../%74ools/x%2Epy?plain=1#L12`` cites ``https://github.com/o/r/blob//tools/x.py`` at line 12. GitHub -reads ``%23`` and ``%3F`` as part of a file name, so they stay escaped. +reads ``%23`` and ``%3F`` as part of a file name, so they stay escaped. A URL +runs to whitespace: a quote or a bracket inside it is part of it, as it is to +a browser, so ``x.py?q='a'#L12`` still cites line 12 of ``x.py``. A cited path is in-repository when, after any prefix naming this repository is removed (a ``github.com/PolicyEngine/microcosm/blob//`` URL, at a ref @@ -70,9 +72,11 @@ import re import string import unicodedata +from bisect import bisect_left, bisect_right from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field -from functools import cache +from functools import cache, lru_cache +from itertools import accumulate from pathlib import Path, PurePosixPath from urllib.parse import unquote_to_bytes @@ -145,18 +149,27 @@ + _QUOTE + r"[^.;\n]{0,24}?(?``, ``.``, ``,``, ``;``, ``)``, ...). A GitHub host is matched +#: where it starts, so a long run of punctuation is scanned once. _URL = re.compile( r"(?:(?i:https?):" r"|(?/\\?#]*@)?" + r"|(?]*[^\s\"'`‘’“”<>.,;:!?)\]]" + r"\S*[^\s\"'`‘’“”<>.,;:!?)\]]" ) #: The scheme a URL starts with, when it has one. _SCHEME = re.compile(r"(?i:https?):") @@ -399,9 +412,30 @@ def _readable_strings(value: object) -> Iterator[str]: yield _URL.sub(_read_url, text) +def _finditer(pattern: re.Pattern[str], text: str) -> Iterator[re.Match[str]]: + """``pattern.finditer(text)``, reading a long run of path characters once. + + A ``_SOURCE`` match that starts inside a run of the characters a path is + made of (``[A-Za-z0-9_./-]``), other than at ``http(s):``, could start + where the run does, so ``finditer`` starts each match at the start of a + run, at ``http(s):``, or where its previous match ended. Trying only + those starts, rather than every character of a run, finds the same + matches in time linear in the run. + """ + + anchored = _STARTING_A_RUN.get(pattern) + if anchored is None: + yield from pattern.finditer(text) + return + position = 0 + while match := pattern.match(text, position) or anchored.search(text, position): + yield match + position = match.end() + + def _citation_matches(text: str) -> list[re.Match[str]]: return sorted( - (match for pattern in _LINE_CITATIONS for match in pattern.finditer(text)), + (match for pattern in _LINE_CITATIONS for match in _finditer(pattern, text)), key=lambda match: match.start(), ) @@ -423,10 +457,14 @@ def unrecognised_line_locators(value: object) -> list[LineCitation]: for text in _readable_strings(value): matches = _citation_matches(text) files = {match.start("file") for match in matches} - spans = [(match.start(), match.end()) for match in matches] - for near in _LOCATOR_NEAR_A_SOURCE.finditer(text): - if near.start("file") in files or any( - start <= near.end() < end for start, end in spans + # A near match ending inside a citation is part of it: the citations + # starting at or before its end reach past it. + starts = [match.start() for match in matches] + reach = list(accumulate((match.end() for match in matches), max)) + for near in _finditer(_LOCATOR_NEAR_A_SOURCE, text): + started = bisect_right(starts, near.end()) + if near.start("file") in files or ( + started and reach[started - 1] > near.end() ): continue found.append(LineCitation(near["file"], text, near.start(), near.end())) @@ -445,6 +483,44 @@ def named_commits(text: str) -> list[str]: return [match["sha"] for match in INLINE_COMMIT.finditer(text)] +@dataclass(frozen=True) +class _Marks: + """Where a string's clause breaks, inline commits and commit-like tokens + in no pin spelling are, found once for all the citations in it.""" + + break_starts: list[int] + break_ends: list[int] + commits: list[re.Match[str]] + commit_starts: list[int] + commit_ends: list[int] + loose: list[re.Match[str]] + loose_starts: list[int] + + +@lru_cache(maxsize=1024) +def _marks(text: str) -> _Marks: + breaks = list(_CLAUSE_BREAK.finditer(text)) + commits = list(INLINE_COMMIT.finditer(text)) + pinned = {match.start("sha") for match in commits} + loose = [ + token for token in _COMMIT_LIKE.finditer(text) if token.start() not in pinned + ] + return _Marks( + [match.start() for match in breaks], + [match.end() for match in breaks], + commits, + [match.start() for match in commits], + [match.end() for match in commits], + loose, + [token.start() for token in loose], + ) + + +@lru_cache(maxsize=1024) +def _names_a_moving_ref(text: str, start: int, end: int) -> bool: + return _MOVING_REF.search(text, start, end) is not None + + def _clause(citation: LineCitation) -> tuple[int, int]: """The bounds of the clauses of a citation's string that it touches. @@ -453,22 +529,30 @@ def _clause(citation: LineCitation) -> tuple[int, int]: if citation.start < 0: raise ValueError("only a citation from line_citations carries its position") - first, last = 0, len(citation.text) - for clause_break in _CLAUSE_BREAK.finditer(citation.text): - if clause_break.end() <= citation.start: - first = clause_break.end() - elif clause_break.start() >= citation.end: - last = clause_break.start() - break + marks = _marks(citation.text) + before = bisect_right(marks.break_ends, citation.start) + after = bisect_left(marks.break_starts, citation.end) + first = marks.break_ends[before - 1] if before else 0 + last = ( + marks.break_starts[after] + if after < len(marks.break_starts) + else len(citation.text) + ) return first, last def _binding(citation: LineCitation) -> re.Match[str] | None: """The inline commit match a line citation binds to, if any.""" - commits = list(INLINE_COMMIT.finditer(citation.text)) + marks = _marks(citation.text) start, end = _clause(citation) - in_clause = [match for match in commits if start <= match.start() < end] + # Commits do not overlap, so the nearest ones in the clause are among the + # last that ends before the citation, those overlapping it and the first + # that starts after it. + first = bisect_right(marks.commit_ends, citation.start) + after = bisect_left(marks.commit_starts, citation.end) + nearest = marks.commits[max(first - 1, 0) : after + 1] + in_clause = [match for match in nearest if start <= match.start() < end] if in_clause: return min( in_clause, @@ -476,10 +560,10 @@ def _binding(citation: LineCitation) -> re.Match[str] | None: match.start() - citation.end, citation.start - match.end() ), ) - if _MOVING_REF.search(citation.text, start, end): + if _names_a_moving_ref(citation.text, start, end): return None - before = [match for match in commits if match.start() < start] - return before[-1] if before else None + before = bisect_left(marks.commit_starts, start) + return marks.commits[before - 1] if before else None def citing_commit(citation: LineCitation) -> str | None: @@ -503,17 +587,16 @@ def misspelled_commit_pins(value: object) -> list[tuple[LineCitation, str]]: found: list[tuple[LineCitation, str]] = [] for citation in line_citations(value): - text = citation.text - pinned = {match.start("sha") for match in INLINE_COMMIT.finditer(text)} + marks = _marks(citation.text) start, end = _clause(citation) binding = _binding(citation) if binding is not None and binding.end() < start: start = binding.end() - found.extend( - (citation, token.group(0)) - for token in _COMMIT_LIKE.finditer(text, start, end) - if token.start() not in pinned - ) + # A token cannot cross a clause break or end a commit, so the ones in + # the span are the string's that start in it. + first = bisect_left(marks.loose_starts, start) + last = bisect_left(marks.loose_starts, end) + found.extend((citation, token.group(0)) for token in marks.loose[first:last]) return found From ea9120943260bdc89e7b9d52a44f8091063ae912 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 03:56:30 -0700 Subject: [PATCH 11/21] Test that a host's ideographic full stops are read as dots The mutant that stops reading U+3002 as "." in a host survived round 7: no test wrote a host with an ideographic full stop. The URL property now writes a dot at a full-width index as any of the full-width, ideographic and half-width ideographic full stops, the readings table has a raw and an escaped ideographic stop, and the trailing-dot test drops each stop. bun 1.3's URL reads all three as "." in a host. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_spec_source_citations.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 9e75c64f5d..21d51d5ddb 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -507,14 +507,22 @@ def _escape(text: str, chosen: set[int], upper: bool, keep: str = "") -> str: ) -def _spell_host(host: str, wide: set[int], chosen: set[int], upper: bool) -> str: - """``host`` with its characters at ``wide`` written full-width (``ｇ``) - and those at ``chosen`` as ``%`` escapes of their UTF-8 bytes.""" +#: The full stops a host may be written with: full-width, ideographic and +#: half-width ideographic. The standard reads each as ``.`` (bun 1.3). +_FULL_STOPS = ("\uff0e", "\u3002", "\uff61") + + +def _spell_host( + host: str, wide: set[int], chosen: set[int], upper: bool, stop: str +) -> str: + """``host`` with its characters at ``wide`` written full-width (``ｇ``), + a dot there as ``stop``, and those at ``chosen`` as ``%`` escapes of + their UTF-8 bytes.""" spelled = [] for index, char in enumerate(host): if index in wide: - char = chr(ord(char) + 0xFEE0) + char = stop if char == "." else chr(ord(char) + 0xFEE0) if index in chosen: char = "".join( f"%{byte:02X}" if upper else f"%{byte:02x}" for byte in char.encode() @@ -1369,6 +1377,7 @@ def test_the_round_three_review_counterexample_is_refused() -> None: leads=st.sampled_from(_URL_LEADS), authority=st.sampled_from(_AUTHORITIES), wide=st.sets(st.integers(0, 24), max_size=6), + stop=st.sampled_from(_FULL_STOPS), escaped_host=st.sets(st.integers(0, 24), max_size=6), dot=st.booleans(), backslashed=st.sets(st.integers(0, 160), max_size=6), @@ -1390,6 +1399,7 @@ def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( leads: tuple[str, str], authority: str, wide: set[int], + stop: str, escaped_host: set[int], dot: bool, backslashed: set[int], @@ -1413,6 +1423,7 @@ def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( {index % len(host) for index in wide}, {index % len(host) for index in escaped_host}, upper, + stop, ) # GitHub 404s a ``%2F`` before the ref; a browser reads ``\`` as ``/``. route_and_path = _escape(route, chosen, upper, keep="/") + _escape( @@ -1791,6 +1802,9 @@ def test_bindings_are_found_as_a_scan_of_the_whole_string_finds_them( ("https:github.com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), ("HTTP:///github.com/o/r/blob/a1b2c3d/x.py?plain=1#L12", "http://github.com"), ("https://ｇithub．com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://github\u3002com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("https://github\uff61com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), + ("//github%E3%80%82com/o/r/blob/a1b2c3d/x.py#L12", "//github.com"), ("https://%EF%BD%87ithub.com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), ("https://gi%C2%ADthub.com/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), ("https://github.c\u2092m/o/r/blob/a1b2c3d/x.py#L12", "https://github.com"), @@ -1821,10 +1835,11 @@ def test_a_url_is_read_as_the_url_standard_reads_it( def test_a_trailing_dot_is_dropped_from_a_host() -> None: # The standard keeps it, but DNS ignores it and GitHub redirects # ``github.com.`` to ``github.com`` (HTTP 301 on 2026-10-27). - written = "https://github.com./o/r/blob/a1b2c3d/x.py#L12" - assert [citation.text for citation in line_citations({"note": written})] == [ - "https://github.com/o/r/blob/a1b2c3d/x.py#L12" - ] + for stop in (".", *_FULL_STOPS): + written = f"https://github.com{stop}/o/r/blob/a1b2c3d/x.py#L12" + assert [citation.text for citation in line_citations({"note": written})] == [ + "https://github.com/o/r/blob/a1b2c3d/x.py#L12" + ], written def test_a_break_inside_a_citation_does_not_end_its_clause() -> None: From 25b5bbe2efe70426c3bb5e1b58173faf33dca6e5 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 06:31:55 -0700 Subject: [PATCH 12/21] Read Markdown link destinations, and scan line-number lists in linear time A URL's query no longer hides a Markdown link or an autolink written after it: a string is also read with each URL ending at a quote, a parenthesis or an angle bracket, as Markdown and prose delimit one, and every rule applies to both readings. Line-number lists are read with possessive whitespace and an unambiguous separator, a reverse locator is tried once per list rather than once per number, and a host label longer than DNS allows is not Punycode-encoded, so no locator scan backtracks exponentially or quadratically. The patterns as they were are kept in the test file as the reference a differential property checks the new ones against. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 269 +++++++++++++++++- .../microcosm_build/spec_source_citations.py | 149 +++++++--- 3 files changed, 362 insertions(+), 58 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index b62900304e..e0a53b80ef 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs, and a citation either reading finds is checked: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 21d51d5ddb..4ace6b04a7 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -29,6 +29,10 @@ - every dotted ``microcosm.*`` name in a resource resolves through import plus getattr (``resolve_dotted_symbol``). +Each string is read with its URLs as a browser reads them and, where that +differs, as Markdown delimits them (each URL also ending at a quote, a +parenthesis or an angle bracket); every rule applies to both readings. + Differential checks compare two records of one fact: the commit each UK citation binds to with the incumbent commit its resource declares, and the US known gaps' symbol citations with the code they name (the Census person-column @@ -53,7 +57,7 @@ import pandas as pd import pytest -from hypothesis import assume, example, given, settings +from hypothesis import assume, event, example, given, settings from hypothesis import strategies as st from microcosm.build.us_runtime import asec_pool @@ -1441,10 +1445,22 @@ def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( + route_and_path ) text = f"{written}{query}{fragment.format(a=a, b=b)} {pin.format(sha=sha)}" - found = in_repository_line_citations({"note": text}) + # A quote or a parenthesis in the query ends the URL in the string's + # second reading, which may read what follows as a quoted locator of the + # same file (``x.py‘#L12``): the first reading is the URL's. + read = next(citations._readable_strings(text)) + found = [ + citation + for citation in in_repository_line_citations({"note": text}) + if citation.text == read + ] assert [citation.file for citation in found] == [f"{lead}{host}{route}{ref}/{path}"] # A citation's text is already read: reading it again changes nothing. - assert in_repository_line_citations({"note": found[0].text}) == found + assert [ + citation + for citation in in_repository_line_citations({"note": read}) + if citation.text == read + ] == found @settings(max_examples=200, deadline=None) @@ -1473,11 +1489,20 @@ def test_a_query_does_not_hide_a_line_citation( ref = sha if pinned else "main" url = f"https://github.com/PolicyEngine/upstream-data/blob/{ref}/{directory}{stem}.{suffix}" document = {"note": f"{url}?{query}{fragment.format(a=a, b=b)}"} - assert [ - (citation.file, citing_commit(citation)) - for citation in line_citations(document) - ] == [(url, sha if pinned else None)] - assert len(unpinned_line_citations(document)) == (0 if pinned else 1) + # A quote or a parenthesis in the query ends the URL in the string's + # second reading, which reads the rest of the query as prose: only the + # first reading is the URL's. + read = next(citations._readable_strings(document["note"])) + found = [citation for citation in line_citations(document) if citation.text == read] + assert [(citation.file, citing_commit(citation)) for citation in found] == [ + (url, sha if pinned else None) + ] + unpinned = [ + citation + for citation in unpinned_line_citations(document) + if citation.text == read + ] + assert unpinned == ([] if pinned else found) def test_the_round_four_review_counterexamples_are_refused() -> None: @@ -1521,6 +1546,12 @@ def test_the_round_four_review_counterexamples_are_refused() -> None: assert unpinned_line_citations( {"note": "x.py:12, see https://example.org/doc?page=1; at a1b2c3d"} ) == [LineCitation("x.py", "x.py:12, see https://example.org/doc; at a1b2c3d")] + assert ( + misspelled_commit_pins( + {"note": "x.py:12 at a1b2c3d, see https://example.org/doc?page=1; 9f8e7d6c"} + ) + == [] + ) gitlab = "https://gitlab.com/o/upstream-data/-/blob/a1b2c3d/x.py" text = f"{gitlab}?ref_type=heads&view=plain&page=2 some_function, lines 12-19" assert unrecognised_line_locators({"note": text}) == [ @@ -1589,14 +1620,59 @@ def test_the_round_six_review_counterexample_is_refused() -> None: ] == [f"{blob}{tool}"], written -def test_a_url_runs_to_whitespace() -> None: +def test_the_round_seven_review_counterexamples_are_refused() -> None: + blob = "https://github.com/PolicyEngine/microcosm/blob/" + blob += "29983852e5182266c9a4e565731920ea972fe695/" + tool = "tools/ci_test_plan.py" + cited = blob + tool + network = cited.removeprefix("https:") + bare = cited.removeprefix("https://") + # CommonMark renders each of these with a link to the file at line 12, + # however the URL before it ends. + for written, file in ( + (f"[https://example.org/?x](<{cited}#L12>)", cited), + (f"<{cited}#L12>", cited), + (f"[https://example.org/?x]({cited}#L12)", cited), + (f"[https://example.org/?x]({network}#L12)", network), + (f"[https://example.org/?x]({bare}#L12)", bare), + (f"[https://example.org/?x]({tool}#L12)", tool), + (f'{cited}#L12', cited), + (f"'https://example.org/?x','{cited}#L12'", cited), + ): + assert [ + citation.file + for citation in in_repository_line_citations({"note": written}) + ] == [file], written + + +def test_a_string_is_read_as_a_browser_and_as_markdown_delimit_its_urls() -> None: # As the URL Standard reads it (bun 1.3's ``URL``), a URL written straight # after another, with only a quote between, is part of the first one's - # path: its host is read as written. + # path, so its host is read as written; ended at the quote, as prose ends + # it, it is read on its own. written = "https://example.org/a'https://GitHub.com/o/r/blob/a1b2c3d/x.py#L12" assert [citation.text for citation in line_citations({"note": written})] == [ - written + written, + "https://example.org/a'https://github.com/o/r/blob/a1b2c3d/x.py#L12", + ] + # Ended at a quote in its query, a URL leaves the rest of the query to be + # read as prose: a quoted locator (``x.py‘#L1``) of the same file, or, + # with more between, a locator in no recognised spelling. + url = "https://github.com/o/r/blob/a1b2c3d/x.py" + assert [ + (citation.file, citing_commit(citation)) + for citation in line_citations({"note": f"{url}?0‘#L1"}) + ] == [(url, "a1b2c3d"), (url, "a1b2c3d")] + assert unrecognised_line_locators({"note": f"{url}?q='v'#L1"}) == [ + LineCitation(url, f"{url}'v'#L1") ] + # A string read the same way both times is read once. + for written in ( + "https://GitHub.com/o/r/blob/a1b2c3d/x.py#L12", + "(https://github.com/o/r/blob/a1b2c3d/x.py#L12).", + "https://[::1]:8080/o/r/blob/a1b2c3d/x.py#L12", + ): + assert len(line_citations({"note": written})) == 1, written def test_a_long_string_is_scanned_in_linear_time() -> None: @@ -1626,6 +1702,27 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: unpinned_line_citations(text) unrecognised_line_locators(text) assert time.process_time() - started < 2 + # Each of these took 4 to 28 seconds: a list of line numbers that failed + # was retried once for each way to share its whitespace out (exponential + # in its length), a failed locator once for each way to share a run of + # spaces or quotes, a reverse locator was tried from each number of a + # failed list, and a long host label was encoded in Punycode (quadratic). + # The exponential ones come first, so a regression fails before the + # slower ones run. + for text in ( + "L1 and " * 24, + "lines " + "1 and " * 8 + "in x.py", + "lines " + "1 and " * 10 + "in x.py", + "lines " + "1, " * 24 + "in x.py", + "x.py" + " " * 1_024 + "x", + "x.py" + "'" * 16_000 + "L1x", + "line " + " " * 32_768 + "1x in x.py", + "L1 and " * 8_000, + "https://" + "".join(map(chr, range(0x4000, 0x4000 + 8_000))) + "/a.py#L12", + ): + line_citations(text) + unrecognised_line_locators(text) + assert time.process_time() - started < 2, text[:24] #: Pieces of text around source names and line locators. @@ -1658,6 +1755,14 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: "at ", "(", "—", + "x.py:12", + "a/b.json#L3", + "x.py foo L9", + "x.py, lines 4-5", + "lines 4-5 of y.py", + "L1 and L2 in a/b.py", + "L1, ", + " and ", ) @@ -1665,12 +1770,130 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: @given(text=st.lists(st.sampled_from(_LOCATOR_PIECES), max_size=40).map("".join)) @example(text="x.py:12/y.py:13") @example(text="xhttps://a/b.py:12") +@example(text="L1 and L2 x L3 of y.py") +@example(text="lines 1, L2 of x.py") +@example(text="L1 and L2 and L3 in L4 of x.py") def test_source_names_are_found_where_finditer_finds_them(text: str) -> None: - for pattern in (*citations._LINE_CITATIONS, citations._LOCATOR_NEAR_A_SOURCE): - assert [ + for name, pattern in ( + ("forward", citations._FORWARD_LOCATOR), + ("reverse", citations._REVERSE_LOCATOR), + ("near", citations._LOCATOR_NEAR_A_SOURCE), + ): + found = [ (match.span(), match["file"]) for match in citations._finditer(pattern, text) - ] == [(match.span(), match["file"]) for match in pattern.finditer(text)] + ] + assert found == [ + (match.span(), match["file"]) for match in pattern.finditer(text) + ] + event(f"{name} matches: {min(len(found), 3)}") + + +#: The locator patterns as ``ea912094`` wrote them, before their whitespace and +#: quotes were read possessively: the reference the current ones must agree +#: with. +_REFERENCE_QUOTE = r"[`'\"‘’“”]*" +_REFERENCE_NUMBER = r"L?\s*\d+(?!\w)" +_REFERENCE_SPAN = ( + rf"{_REFERENCE_NUMBER}(?:\s*(?:[-–—]|(?i:to|through))\s*{_REFERENCE_NUMBER})?" +) +_REFERENCE_NUMBERS = ( + rf"{_REFERENCE_SPAN}(?:\s*(?:,|,?\s*(?i:and))\s*{_REFERENCE_SPAN})*" +) +_REFERENCE_LABEL = r"(?i:lines?\b\s+|ll?\.\s*)" +_REFERENCE_FORWARD = re.compile( + citations._SOURCE + + _REFERENCE_QUOTE + + rf"(?::(?={_REFERENCE_QUOTE}L?\d)" + + rf"|\s*[:：]\s*{_REFERENCE_QUOTE}{_REFERENCE_LABEL}" + + rf"|\s*@\s*{_REFERENCE_QUOTE}(?:{_REFERENCE_LABEL}|(?=L\d))" + + rf"|#(?={_REFERENCE_QUOTE}L\d)" + + r"|\s*(?:[,;—–]|\s-(?=\s))?\s*[\[(]?\s*(?:(?i:at|on)\s+)?" + + rf"{_REFERENCE_QUOTE}(?:{_REFERENCE_LABEL}|(?=L\d)))" + + _REFERENCE_QUOTE + + _REFERENCE_NUMBERS +) +_REFERENCE_REVERSE = re.compile( + rf"\b(?:{_REFERENCE_LABEL}|(?=L\d)){_REFERENCE_QUOTE}{_REFERENCE_NUMBERS}" + rf"{_REFERENCE_QUOTE}\s+(?i:of|in)\s+{_REFERENCE_QUOTE}" + citations._SOURCE +) +#: Pieces of line locators, heavy in what reading possessively changed: runs +#: of whitespace and quotes, separators, ranges and labels. The reference +#: backtracks exponentially on a long list that fails, so a text holds at most +#: 24 pieces. +_LIST_PIECES = ( + "x.py", + "a/b.json", + ":", + "：", + "#", + "@", + " ", + " ", + "\t", + "\n", + ",", + ";", + "—", + "–", + "-", + " - ", + "(", + "[", + "at ", + "on ", + "line ", + "Lines", + "l.", + "ll. ", + "L", + "L1", + "12", + " to ", + "through", + " and ", + "AND", + ", and ", + " of ", + " in ", + "'", + "`", + "“", + "x", + "1x", + "x.py:12", + "x.py L3", + "x.py, lines 1-2", + "x.py @ L5", + "x.py (ll. 6–7)", + "lines 4 of y.py", + "L1 and L2 in a/b.py", +) + + +@settings(max_examples=1000, deadline=None) +@given(text=st.lists(st.sampled_from(_LIST_PIECES), max_size=24).map("".join)) +@example(text="x.py - L12") +@example(text="x.py- L12") +@example(text="x.py -L12") +@example(text="lines ' 12 , and 13 ' of x.py") +@example(text="x.py ''L1 and L2") +@example(text="x.py: lines 1 through 2, AND 3") +def test_the_locators_match_as_before_their_whitespace_was_read_possessively( + text: str, +) -> None: + for name, current, reference in ( + ("forward", citations._FORWARD_LOCATOR, _REFERENCE_FORWARD), + ("reverse", citations._REVERSE_LOCATOR, _REFERENCE_REVERSE), + ): + found = [ + (match.span(), match.span("file")) + for match in citations._finditer(current, text) + ] + assert found == [ + (match.span(), match.span("file")) for match in reference.finditer(text) + ] + event(f"{name} matches: {min(len(found), 3)}") def _clause_from_the_whole_string(citation: LineCitation) -> tuple[int, int]: @@ -1705,7 +1928,8 @@ def _misspelled_pins_from_the_whole_string(text: str) -> list[tuple[LineCitation found: list[tuple[LineCitation, str]] = [] for citation in line_citations(text): pinned = { - match.start("sha") for match in citations.INLINE_COMMIT.finditer(text) + match.start("sha") + for match in citations.INLINE_COMMIT.finditer(citation.text) } start, end = _clause_from_the_whole_string(citation) binding = _binding_from_the_whole_string(citation) @@ -1773,6 +1997,8 @@ def _unrecognised_from_the_whole_string(text: str) -> list[LineCitation]: @example(text="https://github.com/o/r/blob/d4e5f6a/z.py#L7 at a1b2c3d") @example(text="a1b2c3d; x.py:12; 9f8e7d6c y.py:3 on main") @example(text="x.py:1 y.py foo L1 x.py, lines 1-2 z.py foo L2") +@example(text="at a1b2c3d; on main;x.py:12") +@example(text="9f8e7d6c;x.py:12") def test_bindings_are_found_as_a_scan_of_the_whole_string_finds_them( text: str, ) -> None: @@ -1842,6 +2068,19 @@ def test_a_trailing_dot_is_dropped_from_a_host() -> None: ], written +def test_a_label_longer_than_dns_allows_is_left_unencoded() -> None: + # Up to 63 characters a label is encoded as the standard encodes it (bun + # 1.3); a longer one names no host either way, so it is left as mapped. + for length, reading in ( + (63, "https://xn--td" + "a" * 63 + ".com"), + (64, "https://" + "ü" * 64 + ".com"), + ): + written = f"https://{'Ü' * length}.com/o/r/blob/a1b2c3d/x.py#L12" + assert [citation.text for citation in line_citations({"note": written})] == [ + f"{reading}/o/r/blob/a1b2c3d/x.py#L12" + ], length + + def test_a_break_inside_a_citation_does_not_end_its_clause() -> None: for text in ("x.py; lines 12-19 at a1b2c3d", "x.py; lines 12-19, commit a1b2c3d"): assert [citing_commit(found) for found in line_citations(text)] == [ diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 8d113258bc..55d36e24cf 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -44,7 +44,12 @@ cites ``https://github.com/o/r/blob//tools/x.py`` at line 12. GitHub reads ``%23`` and ``%3F`` as part of a file name, so they stay escaped. A URL runs to whitespace: a quote or a bracket inside it is part of it, as it is to -a browser, so ``x.py?q='a'#L12`` still cites line 12 of ``x.py``. +a browser, so ``x.py?q='a'#L12`` still cites line 12 of ``x.py``. Markdown +and prose end a URL at a quote, a parenthesis or an angle bracket too, so a +string is also read that way where it reads differently +(``_readable_strings``): in ``[https://a/?x](https://github.com/...#L12)`` +the label's query does not hide the link. A citation either reading finds +is checked. A cited path is in-repository when, after any prefix naming this repository is removed (a ``github.com/PolicyEngine/microcosm/blob//`` URL, at a ref @@ -100,15 +105,23 @@ ) #: Quotes or backticks around a file name or a locator. _QUOTE = r"[`'\"‘’“”]*" +#: ``_QUOTE``, never given back (``*+``). Where it is used, what follows it +#: cannot start with a quote, or is another ``_QUOTES`` that would only take +#: the same quotes, so giving one back never helps a match. +_QUOTES = r"[`'\"‘’“”]*+" #: A line number, plain or GitHub-style (``L12``). -_NUMBER = r"L?\s*\d+(?!\w)" +_NUMBER = r"L?\s*+\d++(?!\w)" #: One line or a range of them: ``12-19``, ``L12-L19``, ``12–19``, #: ``12 to 19``, ``12 through 19``. -_SPAN = rf"{_NUMBER}(?:\s*(?:[-–—]|(?i:to|through))\s*{_NUMBER})?" -#: Several lines or ranges: ``12-19, 30`` or ``12-19 and 30``. -_NUMBERS = rf"{_SPAN}(?:\s*(?:,|,?\s*(?i:and))\s*{_SPAN})*" +_SPAN = rf"{_NUMBER}(?:\s*+(?:[-–—]|(?i:to|through))\s*+{_NUMBER})?" +#: Several lines or ranges: ``12-19, 30``, ``12-19 and 30`` or +#: ``12, and 30``. Each run of whitespace is read by one possessive ``\s*+`` +#: and each separator can be read only one way, so a list that fails is given +#: up in time linear in its length, not retried once for each way to share +#: its whitespace out. +_NUMBERS = rf"{_SPAN}(?:\s*+(?:,\s*+(?:(?i:and)\s*+)?|(?i:and)\s*+){_SPAN})*" #: A line label: ``line``, ``lines`` (any case), ``l.`` or ``ll.``. -_LABEL = r"(?i:lines?\b\s+|ll?\.\s*)" +_LABEL = r"(?i:lines?\b\s++|ll?\.\s*+)" #: A line locator into a source, in each spelling the packages have used or #: GitHub produces, followed by its line numbers: #: @@ -121,25 +134,32 @@ #: - reverse: ``lines 12-19 of x.py``, ``line 12 in x.py``, ``L12 of x.py``. #: #: The numbers are any ``_NUMBERS``; the file name and the locator may be -#: quoted or backticked (```x.py` lines 12-19``, ``x.py:"12-19"``). -_LINE_CITATIONS = ( - re.compile( - _SOURCE - + _QUOTE - + rf"(?::(?={_QUOTE}L?\d)" - + rf"|\s*[:：]\s*{_QUOTE}{_LABEL}" - + rf"|\s*@\s*{_QUOTE}(?:{_LABEL}|(?=L\d))" - + rf"|#(?={_QUOTE}L\d)" - + r"|\s*(?:[,;—–]|\s-(?=\s))?\s*[\[(]?\s*(?:(?i:at|on)\s+)?" - + rf"{_QUOTE}(?:{_LABEL}|(?=L\d)))" - + _QUOTE - + _NUMBERS - ), - re.compile( - rf"\b(?:{_LABEL}|(?=L\d)){_QUOTE}{_NUMBERS}{_QUOTE}\s+(?i:of|in)\s+{_QUOTE}" - + _SOURCE - ), +#: quoted or backticked (```x.py` lines 12-19``, ``x.py:"12-19"``). Each run +#: of whitespace is read by one possessive ``\s*+``, and the space before a +#: spaced ``-`` is looked behind for (``(?<=\s)``), so a locator that fails +#: does so in time linear in its length. +_FORWARD_LOCATOR = re.compile( + _SOURCE + + _QUOTES + + rf"(?::(?={_QUOTE}L?\d)" + + rf"|\s*+[:：]\s*+{_QUOTES}{_LABEL}" + + rf"|\s*+@\s*+{_QUOTES}(?:{_LABEL}|(?=L\d))" + + rf"|#(?={_QUOTE}L\d)" + + r"|\s*+(?:[,;—–]|(?<=\s)-(?=\s))?\s*+[\[(]?\s*+(?:(?i:at|on)\s++)?" + + rf"{_QUOTES}(?:{_LABEL}|(?=L\d)))" + + _QUOTES + + _NUMBERS +) +_REVERSE_LOCATOR = re.compile( + rf"\b(?:{_LABEL}|(?=L\d)){_QUOTES}{_NUMBERS}{_QUOTES}\s++(?i:of|in)\s++{_QUOTES}" + + _SOURCE ) +_LINE_CITATIONS = (_FORWARD_LOCATOR, _REVERSE_LOCATOR) +#: How a reverse locator starts: a label or an ``L`` number, and the list of +#: line numbers after it. What the locator needs after its list (`` of ``, +#: `` in ``) cannot continue the list, so it matches after the whole list or +#: not at all. +_REVERSE_HEAD = re.compile(rf"\b(?:{_LABEL}|(?=L\d)){_QUOTES}{_NUMBERS}") #: A source name followed by a labelled line locator (``L12``, ``line 12``, #: ``ll. 12``) within 24 characters, with no ``.``, ``;`` or line break #: between: the shape of every forward spelling, recognised or not. A bare @@ -154,23 +174,30 @@ #: or at ``http(s):``. _STARTING_A_RUN = { pattern: re.compile(r"(?:(?``, ``.``, ``,``, ``;``, ``)``, ...). A GitHub host is matched -#: where it starts, so a long run of punctuation is scanned once. -_URL = re.compile( +#: Where a URL starts: at ``http(s):``, at a network-path reference +#: (``//host/...``, any two of ``/`` and ``\``), or at a GitHub host written +#: without a scheme (``github.com/...``, before an optional port; a user +#: before it is left as prose). A GitHub host is matched where it starts, so +#: a long run of punctuation is scanned once. +_URL_START = ( r"(?:(?i:https?):" r"|(?.,;:!?)\]]" ) +#: A URL as a browser reads it: it runs to whitespace, so a quote or an angle +#: bracket inside it is part of it, and it does not end in a quote, a bracket +#: or punctuation that closes the sentence around it (``'``, ``>``, ``.``, +#: ``,``, ``;``, ``)``, ...). +_URL = re.compile(_URL_START + r"\S*[^\s\"'`‘’“”<>.,;:!?)\]]") +#: A URL as Markdown and prose delimit it: as ``_URL``, but it also ends at a +#: quote, a parenthesis or an angle bracket, so the label and the destination +#: of ``[https://a/?x](https://b/...)``, the two autolinks of +#: ```` and a URL quoted straight after another +#: are each read on their own. +_DELIMITED_URL = re.compile(_URL_START + r"[^\s\"'`‘’“”<>()]*[^\s\"'`‘’“”<>()\].,;:!?]") #: The scheme a URL starts with, when it has one. _SCHEME = re.compile(r"(?i:https?):") #: A URL up to its query or fragment. @@ -190,6 +217,8 @@ #: The code points UTS46 maps into ASCII that this Python's Unicode database #: (16.0) does not know: Unicode 17's U+A7F1 MODIFIER LETTER CAPITAL S. _MAPPED_AFTER_UNICODE_16 = str.maketrans({"\ua7f1": "s"}) +#: The most characters a DNS label holds (RFC 1035 section 2.3.4). +_LONGEST_LABEL = 63 #: A ``%`` escape in a URL. _ESCAPE = re.compile(r"%([0-9A-Fa-f]{2})") #: The characters a ``%`` escape in a URL path is read as: the unreserved ones @@ -328,7 +357,10 @@ def _to_ascii(domain: str) -> str: Default-ignorable code points are dropped, the rest NFKC case-folded (``ｇ`` and ``ₒ`` are ``g`` and ``o``), ``。`` read as ``.``, and a label left outside ASCII written in Punycode. Where UTS46 refuses a character - this reads it anyway, so it never misses a host UTS46 reads. + this reads it anyway, so it never misses a host UTS46 reads. A label + longer than DNS allows (``_LONGEST_LABEL``) is left unencoded: it names + no host GitHub serves either way, and Python's Punycode encoder takes + time quadratic in a label's length. """ domain = _IGNORABLE_IN_HOST.sub("", domain).translate(_MAPPED_AFTER_UNICODE_16) @@ -336,7 +368,9 @@ def _to_ascii(domain: str) -> str: "NFKC", unicodedata.normalize("NFKC", domain).casefold() ).replace("\u3002", ".") return ".".join( - label if label.isascii() else "xn--" + label.encode("punycode").decode("ascii") + label + if label.isascii() or len(label) > _LONGEST_LABEL + else "xn--" + label.encode("punycode").decode("ascii") for label in folded.split(".") ) @@ -406,23 +440,50 @@ def _read_url(url: re.Match[str]) -> str: def _readable_strings(value: object) -> Iterator[str]: - """``nested_strings``, with each URL in them read as ``_read_url`` reads it.""" + """``nested_strings``, with the URLs in them read as ``_read_url`` reads them. + + Each string is read with each URL running to whitespace (``_URL``), as a + browser reads it. Where it reads differently with each URL also ending at + a quote, a parenthesis or an angle bracket (``_DELIMITED_URL``), as Markdown + and prose delimit one, it is read that way too, so a link's destination + is not lost in the query of a URL before it. + """ for text in nested_strings(value): - yield _URL.sub(_read_url, text) + whole = _URL.sub(_read_url, text) + yield whole + delimited = _DELIMITED_URL.sub(_read_url, text) + if delimited != whole: + yield delimited def _finditer(pattern: re.Pattern[str], text: str) -> Iterator[re.Match[str]]: - """``pattern.finditer(text)``, reading a long run of path characters once. + """``pattern.finditer(text)``, in time linear in ``text``. A ``_SOURCE`` match that starts inside a run of the characters a path is made of (``[A-Za-z0-9_./-]``), other than at ``http(s):``, could start where the run does, so ``finditer`` starts each match at the start of a run, at ``http(s):``, or where its previous match ended. Trying only those starts, rather than every character of a run, finds the same - matches in time linear in the run. + matches. + + A reverse locator starts with a label or an ``L`` number and a list of + line numbers (``_REVERSE_HEAD``). Inside a list it can start only at a + number, and from there it reads the rest of the list to the same end, so + when it fails after a list it fails from every number in it. Each list is + therefore read once, rather than once from each of its numbers. """ + if pattern is _REVERSE_LOCATOR: + position = 0 + while head := _REVERSE_HEAD.search(text, position): + match = pattern.match(text, head.start()) + if match is not None: + yield match + position = match.end() + else: + position = head.end() + return anchored = _STARTING_A_RUN.get(pattern) if anchored is None: yield from pattern.finditer(text) @@ -441,7 +502,11 @@ def _citation_matches(text: str) -> list[re.Match[str]]: def line_citations(value: object) -> list[LineCitation]: - """Each line citation of a source in a payload, in string order.""" + """Each line citation of a source in a payload, in string order. + + A string read two ways (``_readable_strings``) gives the citations of + each reading, so one both readings find is listed twice. + """ return [ LineCitation(match["file"], text, match.start(), match.end()) From a0d20514495f861e7f409a83ce3ccc42a155fbc9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 08:51:29 -0700 Subject: [PATCH 13/21] Try a failed reverse locator again from its list's last number, and read Markdown and HTML escapes and URL line breaks Round 8's review found two citations of this repository that no rule saw. `L1-L2-3 of tools/ci_test_plan.py`: a reverse locator was tried only from a list's head. It read `L1-L2`, failed, and the scan moved past the list, but from `L2` the locator reads `L2-3` and succeeds. When a reverse locator fails from a list's head, `_finditer` now tries it again from the list's last number, if that is an `L` number, and only then moves past the list. From any other number it reads the rest of the list as the head did. Each number is read in at most two lists, so the scan stays linear. `[a](https://github.com/PolicyEngine/microcosm/blob//tools/ci_test_plan\.py#L12)`: CommonMark decodes `\.` in a link destination, and character references likewise (`#L12`, `.py`). A string holding a backslash before ASCII punctuation or a character reference is now read again with them decoded, in one pass. The URL Standard also deletes tabs and line breaks from a URL, so each string is also read with them deleted, as written and as decoded. Every rule checks every reading, so a new reading can only refuse more. Tests: the review's inputs, three more chains, 15 escaped Markdown and HTML shapes and 4 broken hrefs; a property that escapes a line link of this repository in six Markdown and HTML settings; chain pieces and examples in both differentials; and timing strings for the round-8 mutant survivors M80 and M81 and for the new code. Three tests that asserted the exact citation list now filter to the first reading. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 300 +++++++++++++++++- .../microcosm_build/spec_source_citations.py | 98 +++++- 3 files changed, 377 insertions(+), 23 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index e0a53b80ef..69ddc8f2d3 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs, and a citation either reading finds is checked: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`), and with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A citation any reading finds is checked. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 4ace6b04a7..15798182a8 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -53,6 +53,7 @@ import string import time from collections.abc import Callable +from html.entities import html5 from pathlib import Path, PurePosixPath import pandas as pd @@ -565,6 +566,20 @@ def _is_commit_like(sha: str) -> bool: depth=st.integers(0, 3), as_key=st.booleans(), ) +# The reverse spellings come last, so generation draws them least: a worded +# reverse range is found only if ``to`` reads as a range. +@example( + stem="x", + suffix="py", + directory="", + locator="lines {a} to {b} of {file}", + a=12, + b=19, + prefix="", + tail="", + depth=0, + as_key=False, +) def test_an_unpinned_line_citation_is_refused_at_any_depth( stem: str, suffix: str, @@ -601,6 +616,22 @@ def test_an_unpinned_line_citation_is_refused_at_any_depth( depth=st.integers(0, 3), as_key=st.booleans(), ) +@example( + stem="x", + suffix="py", + directory="", + here="https://github.com/PolicyEngine/microcosm/blob/main/tools/" + "build_us_puf_support_base.py", + locator="{q1}{file}{q2}#{q3}L{a}{q4}", + q1="", + q2="", + q3="`", + q4="", + a=1, + b=1, + depth=0, + as_key=False, +) def test_a_quoted_line_locator_is_refused_as_a_value_or_a_key( stem: str, suffix: str, @@ -619,12 +650,18 @@ def test_a_quoted_line_locator_is_refused_as_a_value_or_a_key( file = here or f"{directory}{stem}.{suffix}" text = locator.format(file=file, q1=q1, q2=q2, q3=q3, q4=q4, a=a, b=b) document = _plant(text, depth, as_key) - assert line_citations(document) == [LineCitation(file, text)] + # A quote after a URL's ``#`` ends the URL in the string's second reading, + # which finds the same file's locator again (``x.py'L12``). + found = line_citations(document) + assert [citation for citation in found if citation.text == text] == [ + LineCitation(file, text) + ] + assert {citation.file for citation in found} == {file} assert unrecognised_line_locators(document) == [] if here is None: assert LineCitation(file, text) in unpinned_line_citations(document) else: - assert in_repository_line_citations(document) == [LineCitation(file, text)] + assert in_repository_line_citations(document) == found @settings(max_examples=200, deadline=None) @@ -1579,9 +1616,13 @@ def test_the_round_five_review_counterexamples_are_refused() -> None: (f"a@b@github.com:{route}{escaped}#L12", ""), ): host = "raw.githubusercontent.com" if "raw." in written else "github.com" + # To Markdown ``\\`` is an escaped ``\``, so a decoded reading follows + # the URL Standard's, which comes first. + read = next(citations._readable_strings(written)) assert [ citation.file for citation in in_repository_line_citations({"note": written}) + if citation.text == read ] == [f"{lead}{host}{route}{tool}"], written # The URL Standard refuses each of these authorities, so no browser opens # them: each is read as written, and its path names no file here. @@ -1675,6 +1716,215 @@ def test_a_string_is_read_as_a_browser_and_as_markdown_delimit_its_urls() -> Non assert len(line_citations({"note": written})) == 1, written +def test_the_round_eight_review_counterexamples_are_refused() -> None: + blob = "https://github.com/PolicyEngine/microcosm/blob/" + blob += "29983852e5182266c9a4e565731920ea972fe695/" + tool = "tools/ci_test_plan.py" + cited = blob + tool + # A range can end where another starts. Tried only from a list's head, a + # reverse locator missed the one starting at the list's last number + # (``L2-3`` after ``L1-L2``), or found a later, shorter one. + for written in ( + f"L1-L2-3 of {tool}", + f"L1 to L2 through 3 of {tool}", + f"lines 1-L2-3 in {tool}", + f"L1-L2, L3-L4-5 of {tool}", + ): + assert [ + citation.file + for citation in in_repository_line_citations({"note": written}) + ] == [tool], written + assert [ + (citation.start, citation.end) + for citation in line_citations({"note": "L1-L2-L3 of x.py"}) + ] == [(3, 16)] + # CommonMark (markdown-it-py 4.2) renders each Markdown link here, and + # HTML decodes each attribute, as a link to line 12 of the file. + for written in ( + f"[a]({blob}tools/ci_test_plan\\.py#L12)", + f"[a]({blob}tools/ci\\_test\\_plan.py#L12)", + f"[a](<{cited}\\#L12>)", + f"[a]({cited}#L12)", + f"[a]({cited}#L12)", + f"[a]({blob}tools/ci_test_plan.py#L12)", + f"[a]({cited}#L12)", + f"[a]({cited}#L12)", + f"[a]({cited}#L12)", + f"[a]({blob}tools/ci_test_plan.py#L12)", + f"[a][r]\n\n[r]: {cited}#L12", + f"[a]({cited.replace('github.com', 'github.com')}#L12)", + f'a', + f"a", + f'a', + ): + assert { + citation.file + for citation in in_repository_line_citations({"note": written}) + } == {cited}, written + # The URL Standard deletes each ASCII tab and line break from a URL, so + # bun 1.3's ``URL`` reads each of these attributes as the same link. + for written in ( + f'a', + f'a', + f'a', + f'a', + ): + assert { + citation.file + for citation in in_repository_line_citations({"note": written}) + } == {cited}, written + + +def test_a_string_is_read_again_with_its_markup_escapes_decoded() -> None: + def readings(text: str) -> list[str]: + return list(citations._readable_strings(text)) + + # Decoded in one pass, as CommonMark decodes: an escaped ``&`` starts no + # reference, and a decoded one starts no second. + assert readings("x\\&y") == ["x\\&y", "x&y"] + assert readings("x&#35;y") == ["x&#35;y", "x#y"] + # A backslash before anything but ASCII punctuation stays. + assert readings("x\\a\\é\\1") == ["x\\a\\é\\1"] + # HTML decodes a numeric reference without its ``;``. + assert readings("#L1") == ["#L1", "#L1"] + # A numeric reference is read past its leading zeros, and one past + # U+10FFFF is U+FFFD, however many digits it has: converting 5,000 + # digits whole raises. + assert readings("&#" + "0" * 5_000 + "35;L1") == [ + "&#" + "0" * 5_000 + "35;L1", + "#L1", + ] + for written in ("&#" + "9" * 5_000 + ";", "&#x" + "f" * 5_000 + ";", "�"): + assert readings(written) == [written, "\ufffd"], written[:12] + # The escapes are decoded before a URL is read: ``%`` decoded from + # ``%`` starts an escape the URL's path decodes. + url = "https://github.com/o/r/blob/a1b2c3d/x%2Epy#L1" + assert readings(url)[-1] == "https://github.com/o/r/blob/a1b2c3d/x.py#L1" + # A tab or a line break, as written or decoded, is deleted in a reading + # of its own. + assert readings("a\tb\nc\r\nd") == ["a\tb\nc\r\nd", "abcd"] + assert readings("x y") == ["x y", "x\ny", "xy"] + # A string with nothing to decode is read as before. + for written, read in ( + ("x.py:12", "x.py:12"), + ("a & b; c\\d", "a & b; c\\d"), + ("https://example.org/?a=1&b=2", "https://example.org/"), + ): + assert readings(written) == [read], written + + +#: Each ASCII character's named character references (``#`` for ``#``), +#: from the HTML standard's table, which CommonMark reads too. +_NAMED_REFERENCES = { + char: tuple( + sorted( + name + for name, value in html5.items() + if value == char and name.endswith(";") + ) + ) + for char in string.printable +} +#: Markdown and HTML around a link destination, and whether a backslash +#: escape is decoded there (Markdown) or not (HTML). +_MARKUP = ( + ("[a]({})", True), + ("[a](<{}>)", True), + ("[a][r]\n\n[r]: {}", True), + ("See [the plan]({} 'title').", True), + ('a', False), + ("a", False), +) + + +def _escape_markup( + text: str, + chosen: dict[int, int], + backslash: bool, + zeros: int, + breaks: dict[int, str] | None = None, +) -> str: + """``text`` with its characters at ``chosen`` written as Markdown or HTML + escape them: a backslash before ASCII punctuation (where ``backslash``), a + decimal reference with ``zeros`` leading zeros, a hexadecimal one, or a + named one, each chosen by its number, falling back to the next. Each of + ``breaks`` goes before the character at its index.""" + + def spell(char: str, kind: int) -> str: + for option in range(kind, kind + 4): + option %= 4 + if option == 0 and backslash and char in string.punctuation: + return "\\" + char + if option == 1: + return f"&#{'0' * zeros}{ord(char)};" + if option == 2: + return f"&#x{ord(char):X};" + if option == 3 and _NAMED_REFERENCES.get(char): + names = _NAMED_REFERENCES[char] + return "&" + names[zeros % len(names)] + raise AssertionError(char) + + return "".join( + (breaks or {}).get(index, "") + + (spell(char, chosen[index]) if index in chosen else char) + for index, char in enumerate(text) + ) + + +#: An ASCII tab or line break, as written or as an HTML reference. +_URL_BREAK_SPELLINGS = ("\n", "\t", "\r\n", " ", " ", " ") + + +@settings(max_examples=300, deadline=None) +@given( + url=st.sampled_from(_THIS_REPOSITORY_URLS), + sha=_SHA, + path=st.sampled_from(_ROOTED_FILES), + fragment=st.sampled_from(("#L{a}", "#L{a}-L{b}")), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), + markup=st.sampled_from(_MARKUP), + chosen=st.dictionaries( + st.integers(0, 200), st.integers(0, 3), min_size=1, max_size=24 + ), + zeros=st.integers(0, 3), + breaks=st.dictionaries( + st.integers(0, 200), st.sampled_from(_URL_BREAK_SPELLINGS), max_size=3 + ), +) +def test_a_line_link_of_this_repository_is_refused_however_markup_escapes_it( + url: str, + sha: str, + path: str, + fragment: str, + a: int, + b: int, + markup: tuple[str, bool], + chosen: dict[int, int], + zeros: int, + breaks: dict[int, str], +) -> None: + template, backslash = markup + link = f"{url}{sha}/{path}{fragment.format(a=a, b=b)}" + plain = { + citation.file + for citation in in_repository_line_citations({"note": template.format(link)}) + } + assert plain + chosen = {index % len(link): kind for index, kind in chosen.items()} + # A URL in an HTML attribute may break anywhere: the URL Standard deletes + # its tabs and line breaks. A Markdown destination cannot hold one. + breaks = {index % len(link): spelled for index, spelled in breaks.items()} + escaped = template.format( + _escape_markup(link, chosen, backslash, zeros, None if backslash else breaks) + ) + event(f"escapes: {min(len(chosen), 3)}") + event(f"breaks: {0 if backslash else min(len(breaks), 2)}") + assert plain <= { + citation.file for citation in in_repository_line_citations({"note": escaped}) + } + + def test_a_long_string_is_scanned_in_linear_time() -> None: # Each of these once took time quadratic in its length: a run of path # characters was rescanned from each of its characters, and a user was @@ -1705,7 +1955,8 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: # Each of these took 4 to 28 seconds: a list of line numbers that failed # was retried once for each way to share its whitespace out (exponential # in its length), a failed locator once for each way to share a run of - # spaces or quotes, a reverse locator was tried from each number of a + # spaces or quotes, a failed range once for each way to share the spaces + # after its dash, a reverse locator was tried from each number of a # failed list, and a long host label was encoded in Punycode (quadratic). # The exponential ones come first, so a regression fails before the # slower ones run. @@ -1717,12 +1968,28 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: "x.py" + " " * 1_024 + "x", "x.py" + "'" * 16_000 + "L1x", "line " + " " * 32_768 + "1x in x.py", + "lines L1-" + " " * 32_768 + "2x in x.py", + "x.py line" + " " * 65_536 + "1x", + "lines L1 to" + " " * 32_768 + "2x in x.py", "L1 and " * 8_000, "https://" + "".join(map(chr, range(0x4000, 0x4000 + 8_000))) + "/a.py#L12", ): line_citations(text) unrecognised_line_locators(text) assert time.process_time() - started < 2, text[:24] + # Since round 9: ranges that each end where the next starts, a list read + # again from its last number, and runs of what a markup escape starts with. + for text in ( + "L1-" * 16_000, + "L1-L2-L3" + ", 4" * 8_000 + " x", + "&#" + "0" * 100_000 + "35;", + "&#x" * 30_000, + "\\" * 100_000, + "&a" * 50_000, + ): + line_citations(text) + unrecognised_line_locators(text) + assert time.process_time() - started < 2, text[:24] #: Pieces of text around source names and line locators. @@ -1763,6 +2030,12 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: "L1 and L2 in a/b.py", "L1, ", " and ", + "L1-L2", + "-3", + "-L4", + " to L2", + " through 3", + "L1-L2-3 of x.py", ) @@ -1773,6 +2046,10 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: @example(text="L1 and L2 x L3 of y.py") @example(text="lines 1, L2 of x.py") @example(text="L1 and L2 and L3 in L4 of x.py") +@example(text="L1-L2-3 of tools/ci_test_plan.py") +@example(text="L1 to L2 through 3 of x.py") +@example(text="L1-L2-L3 of x.py") +@example(text="L1-L2, L3-L4-5 of x.py") def test_source_names_are_found_where_finditer_finds_them(text: str) -> None: for name, pattern in ( ("forward", citations._FORWARD_LOCATOR), @@ -1868,6 +2145,10 @@ def test_source_names_are_found_where_finditer_finds_them(text: str) -> None: "x.py (ll. 6–7)", "lines 4 of y.py", "L1 and L2 in a/b.py", + "L1-L2", + "-3", + "-L4", + "L1-L2-3 of x.py", ) @@ -1879,6 +2160,8 @@ def test_source_names_are_found_where_finditer_finds_them(text: str) -> None: @example(text="lines ' 12 , and 13 ' of x.py") @example(text="x.py ''L1 and L2") @example(text="x.py: lines 1 through 2, AND 3") +@example(text="L1-L2-3 of x.py") +@example(text="L1 to L2 through 3 of x.py") def test_the_locators_match_as_before_their_whitespace_was_read_possessively( text: str, ) -> None: @@ -2053,9 +2336,14 @@ def test_a_url_is_read_as_the_url_standard_reads_it( written: str, authority: str | None ) -> None: reading = written if authority is None else f"{authority}/o/r/blob/a1b2c3d/x.py#L12" - assert [citation.text for citation in line_citations({"note": written})] == [ - reading - ] + # The URL Standard's reading comes first. To Markdown ``\\`` is an escaped + # ``\``, so a decoded reading may follow. + assert next(citations._readable_strings(written)) == reading + assert [ + citation.text + for citation in line_citations({"note": written}) + if citation.text == reading + ] == [reading] def test_a_trailing_dot_is_dropped_from_a_host() -> None: diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 55d36e24cf..67bb230edd 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -48,8 +48,14 @@ and prose end a URL at a quote, a parenthesis or an angle bracket too, so a string is also read that way where it reads differently (``_readable_strings``): in ``[https://a/?x](https://github.com/...#L12)`` -the label's query does not hide the link. A citation either reading finds -is checked. +the label's query does not hide the link. Markdown and HTML decode a +backslash before punctuation and a character reference before a link is +followed, so a string holding one is read both ways again with them decoded: +``[a](https://github.com/.../x\\.py#L12)`` links to line 12 of +``x.py``. The URL Standard deletes ASCII tabs and line breaks from a URL +wherever they are, so a string holding one is read again without them: in +an HTML attribute, ``.../x.p y#L12`` links to line 12 of ``x.py``. A +citation any reading finds is checked. A cited path is in-repository when, after any prefix naming this repository is removed (a ``github.com/PolicyEngine/microcosm/blob//`` URL, at a ref @@ -71,6 +77,7 @@ from __future__ import annotations +import html import importlib import json import os @@ -160,6 +167,10 @@ #: `` in ``) cannot continue the list, so it matches after the whole list or #: not at all. _REVERSE_HEAD = re.compile(rf"\b(?:{_LABEL}|(?=L\d)){_QUOTES}{_NUMBERS}") +#: The last number of a list, where it is an ``L`` number a reverse locator +#: can start at (``L2`` in ``L1-L2``), searched for between a list's first +#: character and its end. +_LAST_L_NUMBER = re.compile(r"(?`` and a URL quoted straight after another #: are each read on their own. _DELIMITED_URL = re.compile(_URL_START + r"[^\s\"'`‘’“”<>()]*[^\s\"'`‘’“”<>()\].,;:!?]") +#: What Markdown and HTML decode before a link is followed: a backslash +#: before ASCII punctuation (``x\.py``, Markdown) and a character reference, +#: decimal (``#``), hexadecimal (``#``) or named (``#``), with +#: or without its ``;`` (HTML decodes some without one). +_MARKUP_ESCAPE = re.compile( + r"\\(?P[!-/:-@\[-`{-~])" + r"|&#(?:[xX](?P[0-9A-Fa-f]++)|(?P[0-9]++));?" + r"|&[A-Za-z][A-Za-z0-9]{0,31};?" +) +#: What the URL Standard deletes from a URL, wherever it is, before reading +#: it: an ASCII tab or line break. +_URL_BREAKS = re.compile(r"[\t\n\r]") #: The scheme a URL starts with, when it has one. _SCHEME = re.compile(r"(?i:https?):") #: A URL up to its query or fragment. @@ -439,6 +462,25 @@ def _read_url(url: re.Match[str]) -> str: return f"{lead}{host}{path}{'#' + fragment if fragment else ''}" +def _decode_escape(escape: re.Match[str]) -> str: + """One ``_MARKUP_ESCAPE``, decoded as Markdown or HTML decodes it. + + A numeric reference is decoded from its value with its leading zeros + dropped, so a long run of digits is never converted whole; one past + U+10FFFF is U+FFFD, as HTML reads it. + """ + + if escape["punctuation"] is not None: + return escape["punctuation"] + hexadecimal, decimal = escape["hexadecimal"], escape["decimal"] + if hexadecimal is None and decimal is None: + return html.unescape(escape[0]) + digits = (hexadecimal or decimal).lstrip("0") or "0" + if len(digits) > (6 if hexadecimal is not None else 7): + return "\ufffd" + return html.unescape(f"&#{'x' if hexadecimal is not None else ''}{digits};") + + def _readable_strings(value: object) -> Iterator[str]: """``nested_strings``, with the URLs in them read as ``_read_url`` reads them. @@ -446,15 +488,29 @@ def _readable_strings(value: object) -> Iterator[str]: browser reads it. Where it reads differently with each URL also ending at a quote, a parenthesis or an angle bracket (``_DELIMITED_URL``), as Markdown and prose delimit one, it is read that way too, so a link's destination - is not lost in the query of a URL before it. + is not lost in the query of a URL before it. Where it holds a + ``_MARKUP_ESCAPE``, it is read both ways again with each escape decoded, + as Markdown and HTML decode a link's destination before it is followed, + so ``[a](https://github.com/.../x\\.py#L12)`` is read as linking to + line 12 of ``x.py``. Where it holds a ``_URL_BREAKS``, as written or + decoded, it is read again without them, as the URL Standard reads a URL + in an HTML attribute (``href=".../x.p y#L12"``). Each reading that + differs from the others is yielded once. """ for text in nested_strings(value): - whole = _URL.sub(_read_url, text) - yield whole - delimited = _DELIMITED_URL.sub(_read_url, text) - if delimited != whole: - yield delimited + decoded = _MARKUP_ESCAPE.sub(_decode_escape, text) + sources = ( + text, + decoded, + _URL_BREAKS.sub("", text), + _URL_BREAKS.sub("", decoded), + ) + yield from dict.fromkeys( + pattern.sub(_read_url, source) + for source in dict.fromkeys(sources) + for pattern in (_URL, _DELIMITED_URL) + ) def _finditer(pattern: re.Pattern[str], text: str) -> Iterator[re.Match[str]]: @@ -468,10 +524,18 @@ def _finditer(pattern: re.Pattern[str], text: str) -> Iterator[re.Match[str]]: matches. A reverse locator starts with a label or an ``L`` number and a list of - line numbers (``_REVERSE_HEAD``). Inside a list it can start only at a - number, and from there it reads the rest of the list to the same end, so - when it fails after a list it fails from every number in it. Each list is - therefore read once, rather than once from each of its numbers. + line numbers (``_REVERSE_HEAD``). Inside a list it can start only at an + ``L`` number. From one that starts a span (``L12`` in ``1, L12-19``), or + ends a range a separator follows (``L19`` in ``L12-L19, 30``), it reads + the rest of the list as the list's head did, to the same end, and what + ends a reverse locator (`` of ``, `` in ``) cannot continue a list, so + when it fails from the head it fails from there too. Only the list's last + number can be read another way: when it ends a range and another dash + follows (``L1-L2-3 of x.py``), from there it starts a range of its own + (``L2-3``). So when a reverse locator fails from a list's head, it is + tried next from the list's last number (``_LAST_L_NUMBER``), and + otherwise after the list. Each number is read in at most two lists, + rather than once from each number of a list. """ if pattern is _REVERSE_LOCATOR: @@ -481,8 +545,9 @@ def _finditer(pattern: re.Pattern[str], text: str) -> Iterator[re.Match[str]]: if match is not None: yield match position = match.end() - else: - position = head.end() + continue + last = _LAST_L_NUMBER.search(text, head.start() + 1, head.end()) + position = head.end() if last is None else last.start() return anchored = _STARTING_A_RUN.get(pattern) if anchored is None: @@ -504,8 +569,9 @@ def _citation_matches(text: str) -> list[re.Match[str]]: def line_citations(value: object) -> list[LineCitation]: """Each line citation of a source in a payload, in string order. - A string read two ways (``_readable_strings``) gives the citations of - each reading, so one both readings find is listed twice. + A string read more than one way (``_readable_strings``) gives the + citations of each reading, so one that several readings find is listed + once for each. """ return [ From fa7601cbbd00314c5bf2d74458217b562689fda9 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 10:47:49 -0700 Subject: [PATCH 14/21] Read HTML references apart from Markdown escapes, and read a URL through the whitespace a link holds Round 9's review found an href this guard did not read as a browser does: ``. HTML decodes the reference and leaves the backslash, which the URL Standard reads as `/`; Markdown's one pass took `\&` as an escaped `&`, so no reading decoded the reference. A string holding a `&` or a backslash is now decoded as Markdown decodes it and, apart, as HTML does, each once as text and once as a URL holds what it decodes to (whitespace percent-encoded, a tab or line break deleted). References are decoded as the HTML standard decodes them: a reference to a control is the control (Python's `html.unescape` drops it, so `..` read as `..`), one to a C1 control is read as windows-1252 reads it, one to 0 or a surrogate is U+FFFD, and in a URL a legacy name without `;` before a letter, a digit or `=` stays as written. Beyond the review: whitespace in a segment that a later `..` removes. A quoted attribute value and a Markdown destination in angle brackets hold any whitespace, so each that starts a URL is now read as the browser takes it, its ends stripped, its tabs and line breaks deleted and its spaces percent-encoded. A Unicode space such as U+00A0 ends no URL in the URL Standard, so where a string holds one its URLs are also read through it. And an in-repository path's inner `..` segments are now resolved (`tools/x/../ci_test_plan.py#L12 at ` is refused); a path that climbs out above where it starts names nothing here. Every rule checks every reading, so a new reading can only refuse more. Over all 4,482,233 live strings, every rule's output is unchanged and no string is read a new way. Tests: 25 counterexamples, each confirmed by parse5 7.3, markdown-it-py 4.2 and bun 1.3's URL; exact readings for the new decoders; inner and leading `..` paths; three plain strings the new readings must not refuse; the markup property now also inserts a whitespace detour before a `..` and turns slashes into backslashes in HTML; and timing strings for each new pattern. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 210 ++++++++++++- .../microcosm_build/spec_source_citations.py | 289 ++++++++++++++---- 3 files changed, 429 insertions(+), 72 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index 69ddc8f2d3..a9dae76bb4 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`), and with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A citation any reading finds is checked. In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its ends stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A citation any reading finds is checked. An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 15798182a8..1415565b09 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -1268,6 +1268,8 @@ def test_in_repository_paths_resolve_from_an_anchor_or_a_repository_prefix() -> for file in ( *_IN_REPOSITORY_FILES, "github.com/PolicyEngine/microcosm/blob/x/pyproject.toml", + "tools/x/../build_us_puf_support_base.py", + "a/../b/../tools/build_us_puf_support_base.py", ): assert resolves_in_repository(file), file for file in ( @@ -1279,6 +1281,8 @@ def test_in_repository_paths_resolve_from_an_anchor_or_a_repository_prefix() -> "targets/sources/obr.py", "build_j_recert/buildj_base.sh", "../tools/build_us_puf_support_base.py", + "tools/../../tools/build_us_puf_support_base.py", + "tools/build_us_puf_support_base.py/..", "/tools/build_us_puf_support_base.py", "https://github.com/PolicyEngine/upstream-data/blob/main/tools/" "build_us_puf_support_base.py", @@ -1775,13 +1779,87 @@ def test_the_round_eight_review_counterexamples_are_refused() -> None: } == {cited}, written +def test_the_round_nine_review_counterexamples_are_refused() -> None: + blob = "https://github.com/PolicyEngine/microcosm/blob/" + blob += "29983852e5182266c9a4e565731920ea972fe695/" + cited = blob + "tools/ci_test_plan.py" + # parse5 7.3 decodes each attribute, markdown-it-py 4.2 renders each + # Markdown link, and bun 1.3's ``URL`` then reads each as a link to line + # 12 of the file. + for written in ( + # HTML decodes the reference and leaves the backslash, which the URL + # Standard reads as ``/``. Markdown's one pass took ``\&`` as an + # escaped ``&``, so no reading decoded the reference. + f'a', + f"a", + f"a", + f'a', + f'a', + f'a', + # Whitespace in a segment that a later ``..`` removes. An attribute's + # quoted value and a destination in angle brackets hold any; a plain + # destination and an unquoted value hold a Unicode space, or one a + # reference decodes to. Each URL was read only up to it. + f'a', + f"a", + f'a', + f'a', + f"[a](<{blob}tools/ x/../ci_test_plan.py#L12>)", + f"[a]: <{blob}tools/ x/../ci_test_plan.py#L12>\n\n[a]", + f"[a]({blob}tools/\u00a0x/../ci_test_plan.py#L12)", + f"[a]({blob}tools/ x/../ci_test_plan.py#L12)", + f"[a]({blob}tools/ x/../ci\\_test\\_plan.py#L12)", + f"[https://example.org/?x]({blob}tools/"/../ci_test_plan.py#L12)", + f"a", + f"a", + f"<{blob}tools/x\u00a0/../ci_test_plan.py#L12>", + # An attribute's value is read after whitespace around its ``=``, and + # with the tabs and line breaks in it deleted before it is stripped. + f'a', + f'a', + f'a', + f'a', + # HTML reads ```` as U+0001. Python's ``html.unescape`` dropped it, + # so ``..`` became a ``..`` that removed ``tools``. + f'a', + # In an attribute, a legacy name before a letter is left as written. + f'a', + ): + assert { + citation.file + for citation in in_repository_line_citations({"note": written}) + } == {cited}, written + # A relative path's inner ``..`` is resolved too. + pinned = ( + "tools/x/../ci_test_plan.py#L12 at 29983852e5182266c9a4e565731920ea972fe695" + ) + assert [ + citation.file for citation in in_repository_line_citations({"note": pinned}) + ] == ["tools/x/../ci_test_plan.py"] + # Reading through a Unicode space or an attribute's value refuses + # nothing written plainly: a pinned citation with a no-break space, a + # title holding a URL and a space, and prose after a URL. + sha = "29983852e5182266c9a4e565731920ea972fe695" + for written in ( + f"https://github.com/o/r/blob/{sha}/x.py\u00a0line 12", + f'', + f"see https://example.org/a\u00a0b and x.py line 12 at {sha}", + ): + document = {"note": written} + assert in_repository_line_citations(document) == [], written + assert unpinned_line_citations(document) == [], written + assert misspelled_commit_pins(document) == [], written + assert unrecognised_line_locators(document) == [], written + + def test_a_string_is_read_again_with_its_markup_escapes_decoded() -> None: def readings(text: str) -> list[str]: return list(citations._readable_strings(text)) # Decoded in one pass, as CommonMark decodes: an escaped ``&`` starts no - # reference, and a decoded one starts no second. - assert readings("x\\&y") == ["x\\&y", "x&y"] + # reference, and a decoded one starts no second. HTML leaves the + # backslash as written and decodes the reference after it. + assert readings("x\\&y") == ["x\\&y", "x&y", "x\\&y"] assert readings("x&#35;y") == ["x&#35;y", "x#y"] # A backslash before anything but ASCII punctuation stays. assert readings("x\\a\\é\\1") == ["x\\a\\é\\1"] @@ -1794,8 +1872,32 @@ def readings(text: str) -> list[str]: "&#" + "0" * 5_000 + "35;L1", "#L1", ] - for written in ("&#" + "9" * 5_000 + ";", "&#x" + "f" * 5_000 + ";", "�"): + # So is one to 0 or to a surrogate. + for written in ( + "&#" + "9" * 5_000 + ";", + "&#x" + "f" * 5_000 + ";", + "�", + "�", + "�", + ): assert readings(written) == [written, "\ufffd"], written[:12] + # HTML reads a reference to a control as the control (Python's + # ``html.unescape`` drops it) and one to a C1 control as windows-1252 + # does. A legacy name before a letter or ``=`` is decoded in text; in a + # URL, as in an attribute, it is left as written. + assert readings("€¬it;&=x") == [ + "€¬it;&=x", + "\x01\u20ac\u00acit;&=x", + "\x01\u20ac¬it;&=x", + ] + # In a URL, whitespace a reference decodes to is held as a browser holds + # it: percent-encoded, or deleted where it is a tab or a line break. + assert readings("x y z ") == [ + "x y z ", + "x y\xa0z\t", + "x y\xa0z", + "x%20y%C2%A0z", + ] # The escapes are decoded before a URL is read: ``%`` decoded from # ``%`` starts an escape the URL's path decodes. url = "https://github.com/o/r/blob/a1b2c3d/x%2Epy#L1" @@ -1804,6 +1906,23 @@ def readings(text: str) -> list[str]: # of its own. assert readings("a\tb\nc\r\nd") == ["a\tb\nc\r\nd", "abcd"] assert readings("x y") == ["x y", "x\ny", "xy"] + # An attribute's quoted value and a destination in angle brackets are + # read as the URL a browser follows: stripped at its ends, with what it + # holds held. A value that is no URL is left as written. + assert readings('') == [ + '', + '', + ] + assert readings("[a]()") == [ + "[a]()", + "[a]()", + "[a]()", + ] + # Where a string holds a Unicode space, its URLs are read through it too. + assert readings("https://example.org/a\xa0b/../x.py#L1 c") == [ + "https://example.org/a\xa0b/../x.py#L1 c", + "https://example.org/x.py#L1 c", + ] # A string with nothing to decode is read as before. for written, read in ( ("x.py:12", "x.py:12"), @@ -1823,17 +1942,20 @@ def readings(text: str) -> list[str]: if value == char and name.endswith(";") ) ) - for char in string.printable + for char in (*string.printable, "\u00a0") } -#: Markdown and HTML around a link destination, and whether a backslash -#: escape is decoded there (Markdown) or not (HTML). +#: Markdown and HTML around a link destination: whether a backslash escape is +#: decoded there (Markdown) or not (HTML), and the whitespace a URL there can +#: hold as written. An HTML attribute's URL may hold a tab or a line break, +#: which the URL Standard deletes; a Markdown destination may not. _MARKUP = ( - ("[a]({})", True), - ("[a](<{}>)", True), - ("[a][r]\n\n[r]: {}", True), - ("See [the plan]({} 'title').", True), - ('a', False), - ("a", False), + ("[a]({})", True, "\u00a0"), + ("[a](<{}>)", True, " \u00a0"), + ("[a][r]\n\n[r]: {}", True, "\u00a0"), + ("See [the plan]({} 'title').", True, "\u00a0"), + ('a', False, " \u00a0\x0c\t\n\r"), + ("a", False, " \u00a0\x0c\t\n\r"), + ("a", False, "\u00a0"), ) @@ -1891,6 +2013,22 @@ def spell(char: str, kind: int) -> str: breaks=st.dictionaries( st.integers(0, 200), st.sampled_from(_URL_BREAK_SPELLINGS), max_size=3 ), + detour=st.integers(-1, 5), + backslashes=st.sets(st.integers(0, 20), max_size=2), +) +@example( + url=_THIS_REPOSITORY_URLS[0], + sha="a1b2c3d", + path=_ROOTED_FILES[0], + fragment="#L{a}", + a=12, + b=12, + markup=_MARKUP[4], + chosen={0: 0}, + zeros=0, + breaks={}, + detour=0, + backslashes={7}, ) def test_a_line_link_of_this_repository_is_refused_however_markup_escapes_it( url: str, @@ -1899,27 +2037,52 @@ def test_a_line_link_of_this_repository_is_refused_however_markup_escapes_it( fragment: str, a: int, b: int, - markup: tuple[str, bool], + markup: tuple[str, bool, str], chosen: dict[int, int], zeros: int, breaks: dict[int, str], + detour: int, + backslashes: set[int], ) -> None: - template, backslash = markup + template, backslash, holds = markup link = f"{url}{sha}/{path}{fragment.format(a=a, b=b)}" plain = { citation.file for citation in in_repository_line_citations({"note": template.format(link)}) } assert plain + # The link may go through a segment holding whitespace that a ``..`` + # then removes. + if detour >= 0: + link = f"{url}{sha}/{holds[detour % len(holds)]}x/../{path}" + link += fragment.format(a=a, b=b) chosen = {index % len(link): kind for index, kind in chosen.items()} + # In HTML a slash after the authority may be written as a backslash, + # which the URL Standard reads as a slash, with a reference after it. + slashes = [ + index + for index, char in enumerate(link) + if char == "/" and index > link.find("//") + 1 + ] + if not backslash: + for index in {slashes[choice % len(slashes)] for choice in backslashes}: + link = link[:index] + "\\" + link[index + 1 :] + chosen.setdefault(index + 1, 1) # A URL in an HTML attribute may break anywhere: the URL Standard deletes - # its tabs and line breaks. A Markdown destination cannot hold one. - breaks = {index % len(link): spelled for index, spelled in breaks.items()} + # its tabs and line breaks. A Markdown destination cannot hold one, and an + # unquoted value only one a reference decodes to. + breaks = { + index % len(link): spelled + for index, spelled in breaks.items() + if spelled.startswith("&") or set(spelled) <= set(holds) + } escaped = template.format( _escape_markup(link, chosen, backslash, zeros, None if backslash else breaks) ) event(f"escapes: {min(len(chosen), 3)}") event(f"breaks: {0 if backslash else min(len(breaks), 2)}") + event(f"detour: {'none' if detour < 0 else repr(holds[detour % len(holds)])}") + event(f"backslashes: {0 if backslash else len(backslashes)}") assert plain <= { citation.file for citation in in_repository_line_citations({"note": escaped}) } @@ -1990,6 +2153,21 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: line_citations(text) unrecognised_line_locators(text) assert time.process_time() - started < 2, text[:24] + # Since round 10: runs of what an attribute's value, a destination in + # angle brackets, a long legacy-looking name and a URL read through + # Unicode spaces start with. + for text in ( + '="https:' * 20_000, + "='" + "x" * 100_000, + "](/`` URL, at a ref of any depth such as ``refs/heads/main``, or ``microcosm/`` before a -top-level source directory), it is a file here or one of the path forms -``_repository_path_tails`` lists. A line citation of it is +top-level source directory) and its inner ``..`` segments resolved +(``tools/x/../y.py`` is ``tools/y.py``), it is a file here or one of the path +forms ``_repository_path_tails`` lists. A line citation of it is refused even when pinned. A bare file name cannot be told apart from another repository's, so it must be pinned. @@ -88,6 +104,7 @@ from collections.abc import Iterable, Iterator, Mapping from dataclasses import dataclass, field from functools import cache, lru_cache +from html.entities import html5 from itertools import accumulate from pathlib import Path, PurePosixPath from urllib.parse import unquote_to_bytes @@ -209,15 +226,56 @@ #: ```` and a URL quoted straight after another #: are each read on their own. _DELIMITED_URL = re.compile(_URL_START + r"[^\s\"'`‘’“”<>()]*[^\s\"'`‘’“”<>()\].,;:!?]") -#: What Markdown and HTML decode before a link is followed: a backslash -#: before ASCII punctuation (``x\.py``, Markdown) and a character reference, -#: decimal (``#``), hexadecimal (``#``) or named (``#``), with -#: or without its ``;`` (HTML decodes some without one). -_MARKUP_ESCAPE = re.compile( - r"\\(?P[!-/:-@\[-`{-~])" - r"|&#(?:[xX](?P[0-9A-Fa-f]++)|(?P[0-9]++));?" - r"|&[A-Za-z][A-Za-z0-9]{0,31};?" +_URLS = (_URL, _DELIMITED_URL) +#: ``_URLS`` as a Markdown destination, an HTML attribute and the URL Standard +#: hold a URL: only ASCII whitespace (``[\t\n\f\r ]``) ends one, so a no-break +#: space or another Unicode space inside it is part of it +#: (``.../tools/\u00a0x/../x.py``). They are tried where a string holds one. +_URLS_THROUGH_UNICODE_SPACES = ( + re.compile(_URL_START + r"[^\t\n\f\r ]*[^\s\"'`‘’“”<>.,;:!?)\]]"), + re.compile(_URL_START + r"[^\t\n\f\r \"'`‘’“”<>()]*[^\s\"'`‘’“”<>()\].,;:!?]"), ) +#: Whitespace other than ASCII whitespace: what ``_URLS_THROUGH_UNICODE_SPACES`` +#: reads a URL through. +_UNICODE_SPACE = re.compile(r"[^\S\t\n\f\r ]") +#: A character reference: decimal (``#``) or hexadecimal (``#``), +#: with or without its ``;``, or named (``#``, or a legacy name such as +#: ``&`` without one). +_REFERENCE = ( + r"&#(?:[xX](?P[0-9A-Fa-f]++)|(?P[0-9]++));?" + r"|&(?P[A-Za-z][A-Za-z0-9]{0,31};?)" +) +#: What Markdown decodes in text and in a link destination: a backslash +#: before ASCII punctuation (``x\.py``) and a character reference. +_MARKDOWN_ESCAPE = re.compile(r"\\(?P[!-/:-@\[-`{-~])|" + _REFERENCE) +#: What HTML decodes in text and in an attribute: a character reference. A +#: backslash is left as written, so in ``tools\ci.py`` the reference is +#: decoded and the URL Standard then reads the backslash as ``/``. +_HTML_REFERENCE = re.compile(_REFERENCE) +#: What a numeric reference to a C1 control reads as in HTML: the character +#: windows-1252 puts there, where it puts one (``€`` is ``€``). +_C1_REPLACEMENTS = { + number: bytes([number]).decode("cp1252") + for number in range(0x80, 0xA0) + if number not in (0x81, 0x8D, 0x8F, 0x90, 0x9D) +} +#: What a URL can hold but a reading would end it at: whitespace, a quote, an +#: angle bracket or a parenthesis. A browser deletes an ASCII tab or line +#: break from a URL and percent-encodes the rest (``_hold``). +_HELD_IN_A_URL = re.compile(r"[\s\"'`‘’“”<>()]") +#: Where a URL can hold any whitespace: an HTML attribute's quoted value +#: (``href="..."``, ``href='...'``), to its closing quote, and a Markdown +#: destination in angle brackets (``[a](<...>)``, ``[a]: <...>``), to an +#: unescaped ``>``. +_ENCLOSED_URL = re.compile( + r"=[\t\n\f\r ]*+(?:\"(?P[^\"]*+)\"|'(?P[^']*+)')" + r"|\](?:\(|:[\t ]*+(?:\r\n?|\n)?[\t ]*+)<(?P(?:\\[^\n\r]|[^<>\n\r\\])*+)>" +) +#: A URL start at the start of a string. +_STARTS_A_URL = re.compile(_URL_START) +#: What the URL Standard strips from both ends of a URL: C0 controls and +#: spaces. +_C0_OR_SPACE = "".join(map(chr, range(0x21))) #: What the URL Standard deletes from a URL, wherever it is, before reading #: it: an ASCII tab or line break. _URL_BREAKS = re.compile(r"[\t\n\r]") @@ -462,54 +520,160 @@ def _read_url(url: re.Match[str]) -> str: return f"{lead}{host}{path}{'#' + fragment if fragment else ''}" -def _decode_escape(escape: re.Match[str]) -> str: - """One ``_MARKUP_ESCAPE``, decoded as Markdown or HTML decodes it. +def _code_point(digits: str, base: int) -> str: + """The character a numeric reference names, as HTML reads it. - A numeric reference is decoded from its value with its leading zeros - dropped, so a long run of digits is never converted whole; one past - U+10FFFF is U+FFFD, as HTML reads it. + Leading zeros are dropped first, so a long run of digits is never + converted whole. Zero, a surrogate and a number past U+10FFFF are U+FFFD, + a C1 control is its ``_C1_REPLACEMENTS`` character where it has one, and + any other code point, a control included, is itself. """ - if escape["punctuation"] is not None: - return escape["punctuation"] - hexadecimal, decimal = escape["hexadecimal"], escape["decimal"] - if hexadecimal is None and decimal is None: - return html.unescape(escape[0]) - digits = (hexadecimal or decimal).lstrip("0") or "0" - if len(digits) > (6 if hexadecimal is not None else 7): + digits = digits.lstrip("0") + if len(digits) > (6 if base == 16 else 7): + return "\ufffd" + number = int(digits or "0", base) + if number == 0 or number > 0x10FFFF or 0xD800 <= number <= 0xDFFF: return "\ufffd" - return html.unescape(f"&#{'x' if hexadecimal is not None else ''}{digits};") + return _C1_REPLACEMENTS.get(number, chr(number)) + + +def _named_in_an_attribute(reference: re.Match[str]) -> str: + """A named reference as HTML decodes it in an attribute. + + The longest name in HTML's table that the reference starts with is read, + as in text, but a legacy name read without its ``;`` and followed by a + letter, a digit or ``=`` is left as written (``¬it;``, ``&=``). + """ + + name = reference["name"] + for length in range(len(name), 1, -1): + if name[:length] in html5: + following = ( + name[length : length + 1] + or reference.string[reference.end() : reference.end() + 1] + ) + if name[length - 1] == ";" or not ( + following == "=" or following.isascii() and following.isalnum() + ): + return html5[name[:length]] + name[length:] + break + return reference[0] + + +def _decode_in_text(escape: re.Match[str]) -> str: + """A ``_MARKDOWN_ESCAPE`` or ``_HTML_REFERENCE`` as text reads it. + + A named reference is read as ``html.unescape`` reads one in text, and a + numeric one as ``_code_point`` reads it. + """ + + kind = escape.lastgroup + if kind == "punctuation": + return escape[kind] + if kind == "name": + return html.unescape(escape[0]) + return _code_point(escape[kind], 16 if kind == "hexadecimal" else 10) + + +def _hold(character: re.Match[str]) -> str: + """A ``_HELD_IN_A_URL`` character as a browser holds it in a URL. + + An ASCII tab or line break is deleted and anything else percent-encoded. + """ + + if character[0] in "\t\n\r": + return "" + return "".join(f"%{byte:02X}" for byte in character[0].encode()) + + +def _decode_in_a_url(escape: re.Match[str]) -> str: + """A ``_MARKDOWN_ESCAPE`` or ``_HTML_REFERENCE`` inside a URL. + + A named reference is read as ``_named_in_an_attribute`` reads it, and what + the escape stands for is held as a browser holds it (``_hold``): in + ``href=https://.../x /../y.py`` the space neither ends the URL nor + stops ``..`` from removing its segment. + """ + + if escape.lastgroup == "name": + decoded = _named_in_an_attribute(escape) + else: + decoded = _decode_in_text(escape) + return _HELD_IN_A_URL.sub(_hold, decoded) + + +def _read_enclosed(enclosed: re.Match[str]) -> str: + """An ``_ENCLOSED_URL`` with its URL as a browser takes it. + + An attribute's value has its references decoded (``_decode_in_a_url``), + the C0 controls and spaces at its ends stripped and its tabs and line + breaks deleted, as HTML and then the URL Standard read it. A + destination's backslash escapes and references are decoded, as Markdown + decodes them. What the URL then still holds that a reading would end it + at is held (``_hold``). A value that does not then start a URL is left as + written. + """ + + kind = enclosed.lastgroup + if kind == "angle": + url = _MARKDOWN_ESCAPE.sub(_decode_in_a_url, enclosed[kind]) + else: + url = _HTML_REFERENCE.sub(_decode_in_a_url, enclosed[kind]) + url = _URL_BREAKS.sub("", url.strip(_C0_OR_SPACE)) + if not _STARTS_A_URL.match(url): + return enclosed[0] + start, end = (index - enclosed.start() for index in enclosed.span(kind)) + return enclosed[0][:start] + _HELD_IN_A_URL.sub(_hold, url) + enclosed[0][end:] def _readable_strings(value: object) -> Iterator[str]: """``nested_strings``, with the URLs in them read as ``_read_url`` reads them. - Each string is read with each URL running to whitespace (``_URL``), as a - browser reads it. Where it reads differently with each URL also ending at - a quote, a parenthesis or an angle bracket (``_DELIMITED_URL``), as Markdown - and prose delimit one, it is read that way too, so a link's destination - is not lost in the query of a URL before it. Where it holds a - ``_MARKUP_ESCAPE``, it is read both ways again with each escape decoded, - as Markdown and HTML decode a link's destination before it is followed, - so ``[a](https://github.com/.../x\\.py#L12)`` is read as linking to - line 12 of ``x.py``. Where it holds a ``_URL_BREAKS``, as written or - decoded, it is read again without them, as the URL Standard reads a URL - in an HTML attribute (``href=".../x.p y#L12"``). Each reading that - differs from the others is yielded once. + Each string is read from several sources, each the string as one kind of + reader takes it before following a link: + + - as written; + - with each HTML attribute's quoted value and each Markdown destination + in angle brackets read as a browser takes its URL (``_read_enclosed``), + so a space inside one (``href=".../tools/ x/../y.py#L12"``) neither ends + it nor stops ``..`` from removing its segment; + - where it holds a backslash or a ``&``, decoded as Markdown decodes it + (``_MARKDOWN_ESCAPE``) and as HTML decodes it (``_HTML_REFERENCE``), + each as text reads it (``_decode_in_text``) and as a URL holds it + (``_decode_in_a_url``): ``[a](https://github.com/.../x\\.py#L12)`` + links to line 12 of ``x.py``, and so does + ````, where HTML decodes + the reference and leaves the backslash. + + Each source is read again with its ``_URL_BREAKS`` deleted, as the URL + Standard reads a URL (``href=".../x.p y#L12"``). Each is read with each + URL running to whitespace (``_URL``), as a browser reads it, and with each + URL also ending at a quote, a parenthesis or an angle bracket + (``_DELIMITED_URL``), as Markdown and prose delimit one, so a link's + destination is not lost in the query of a URL before it. Where a source + holds a Unicode space, its URLs are also read through Unicode spaces + (``_URLS_THROUGH_UNICODE_SPACES``). Each reading that differs from the + others is yielded once. """ for text in nested_strings(value): - decoded = _MARKUP_ESCAPE.sub(_decode_escape, text) - sources = ( - text, - decoded, - _URL_BREAKS.sub("", text), - _URL_BREAKS.sub("", decoded), - ) + sources = [text, _ENCLOSED_URL.sub(_read_enclosed, text)] + if "&" in text or "\\" in text: + sources += ( + escapes.sub(decode, text) + for escapes in (_MARKDOWN_ESCAPE, _HTML_REFERENCE) + for decode in (_decode_in_text, _decode_in_a_url) + ) yield from dict.fromkeys( - pattern.sub(_read_url, source) + pattern.sub(_read_url, joined) for source in dict.fromkeys(sources) - for pattern in (_URL, _DELIMITED_URL) + for joined in dict.fromkeys((source, _URL_BREAKS.sub("", source))) + for pattern in ( + _URLS + _URLS_THROUGH_UNICODE_SPACES + if _UNICODE_SPACE.search(joined) + else _URLS + ) ) @@ -820,17 +984,32 @@ def _repository_depth(root: Path) -> int: def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: """Whether a cited path names a file in this checkout. - A prefix naming this repository is removed first; what is left resolves - when it is a file here (after such a prefix) or one of the path forms - ``_repository_path_tails`` lists: ``tools/x.py``, - ``microcosm/tools/x.py``, ``packages/microcosm-build/src/...``, - ``microcosm/graph/executor.py`` and ``us_runtime/asec_pool.py`` all do. + A prefix naming this repository is removed first, and each ``..`` with a + segment before it removes that segment (``tools/x/../y.py`` is + ``tools/y.py``); a path that climbs above where it starts (``../y.py``) + names nothing here. What is left resolves when it is a file here (after + such a prefix) or one of the path forms ``_repository_path_tails`` lists: + ``tools/x.py``, ``microcosm/tools/x.py``, + ``packages/microcosm-build/src/...``, ``microcosm/graph/executor.py`` and + ``us_runtime/asec_pool.py`` all do. """ named = _THIS_REPOSITORY.match(file) relative = PurePosixPath(file[named.end() :] if named else file) - if relative.is_absolute() or ".." in relative.parts: + if relative.is_absolute(): + return False + parts: list[str] = [] + for part in relative.parts: + if part != "..": + parts.append(part) + elif parts: + parts.pop() + else: + # It climbs out of a directory it does not name. + return False + if not parts: return False + relative = PurePosixPath(*parts) candidates: Iterable[PurePosixPath] = [relative] if named is not None and named["url"]: # A URL's ref comes first and may span segments (``main``, From c36671e61da9eee896323311d52248948fe1aa8d Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Tue, 27 Oct 2026 11:15:50 -0700 Subject: [PATCH 15/21] Strip an enclosed URL's ends after decoding it, as HTML and markdown-it do A fuzz of round 10's generation against markdown-it-py 4.2, parse5 7.3 and bun 1.3 found five links no reading refused. HTML decodes an attribute's references before the URL Standard strips its C0 controls and spaces, so `href=" https://..."` and `href=" https://..."` are links; the enclosed reading held the decoded space as `%20` first, and the value no longer started a URL. markdown-it strips whitespace from a destination's ends before following it (`str.strip`, in mdurl), so `[a](< https://... >)`, a destination padded with U+3000 or U+00A0, and `[a](< https://...>)` are links too; the angle reading did not strip at all. Each enclosed value is now decoded without holding, then stripped as its renderer strips it, then held. Over 200,000 generated links, 113,469 render to a line link of this repository and each is refused as written. The five shapes are in the counterexample test. The round-10 timing strings get 2 s of their own: all grow linearly, but under load they pushed the shared budget over. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 15 ++++-- .../microcosm_build/spec_source_citations.py | 52 ++++++++++++------- 3 files changed, 45 insertions(+), 24 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index a9dae76bb4..61d6ec986a 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its ends stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A citation any reading finds is checked. An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A citation any reading finds is checked. An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 1415565b09..b972a97c7a 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -1819,6 +1819,14 @@ def test_the_round_nine_review_counterexamples_are_refused() -> None: f'a', f'a', f'a', + # HTML decodes a reference before the URL Standard strips the value's + # ends, and markdown-it strips whitespace from a destination's ends + # before following it. + f'a', + f'a', + f"[a](< {blob}tools/x /../ci_test_plan.py#L12 >)", + f"[a](<\u3000{blob}tools/x /../ci_test_plan.py#L12\u00a0>)", + f"[a](< {blob}tools/x /../ci_test_plan.py#L12>)", # HTML reads ```` as U+0001. Python's ``html.unescape`` dropped it, # so ``..`` became a ``..`` that removed ``tools``. f'a', @@ -2153,9 +2161,10 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: line_citations(text) unrecognised_line_locators(text) assert time.process_time() - started < 2, text[:24] - # Since round 10: runs of what an attribute's value, a destination in - # angle brackets, a long legacy-looking name and a URL read through - # Unicode spaces start with. + # Since round 10, in 2 s of their own: runs of what an attribute's value, + # a destination in angle brackets, a long legacy-looking name and a URL + # read through Unicode spaces start with. + started = time.process_time() for text in ( '="https:' * 20_000, "='" + "x" * 100_000, diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 78a3ed4872..b7f435f4b1 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -65,10 +65,11 @@ wherever they are, so each reading is read again without them: in an HTML attribute, ``.../x.p y#L12`` links to line 12 of ``x.py``. A browser follows a quoted attribute value, and a Markdown destination in angle -brackets, as one URL, its ends stripped and the spaces inside it -percent-encoded, so a string holding one that starts a URL is read again -with it read so: ``href=" https://github.com/.../a b/../x.py#L12 "`` links to -line 12 of ``x.py``. The URL Standard ends no URL at a Unicode space such as +brackets, as one URL: its escapes decoded, then its ends stripped (as the +URL Standard strips a URL, and markdown-it a destination) and the spaces +inside it percent-encoded. So a string holding one that starts a URL is read +again with it read so: ``href=" https://github.com/.../a b/../x.py#L12"`` +links to line 12 of ``x.py``. The URL Standard ends no URL at a Unicode space such as U+00A0 (it percent-encodes one), so where a string holds one, its URLs are read through it too. A citation any reading finds is checked. @@ -587,39 +588,50 @@ def _hold(character: re.Match[str]) -> str: return "".join(f"%{byte:02X}" for byte in character[0].encode()) +def _decode_in_an_attribute(escape: re.Match[str]) -> str: + """A ``_MARKDOWN_ESCAPE`` or ``_HTML_REFERENCE`` as an attribute decodes it. + + A named reference is read as ``_named_in_an_attribute`` reads it, and any + other escape as ``_decode_in_text`` reads it. + """ + + if escape.lastgroup == "name": + return _named_in_an_attribute(escape) + return _decode_in_text(escape) + + def _decode_in_a_url(escape: re.Match[str]) -> str: """A ``_MARKDOWN_ESCAPE`` or ``_HTML_REFERENCE`` inside a URL. - A named reference is read as ``_named_in_an_attribute`` reads it, and what - the escape stands for is held as a browser holds it (``_hold``): in + It is decoded as ``_decode_in_an_attribute`` decodes it, and what it + stands for is held as a browser holds it (``_hold``): in ``href=https://.../x /../y.py`` the space neither ends the URL nor stops ``..`` from removing its segment. """ - if escape.lastgroup == "name": - decoded = _named_in_an_attribute(escape) - else: - decoded = _decode_in_text(escape) - return _HELD_IN_A_URL.sub(_hold, decoded) + return _HELD_IN_A_URL.sub(_hold, _decode_in_an_attribute(escape)) def _read_enclosed(enclosed: re.Match[str]) -> str: """An ``_ENCLOSED_URL`` with its URL as a browser takes it. - An attribute's value has its references decoded (``_decode_in_a_url``), - the C0 controls and spaces at its ends stripped and its tabs and line - breaks deleted, as HTML and then the URL Standard read it. A - destination's backslash escapes and references are decoded, as Markdown - decodes them. What the URL then still holds that a reading would end it - at is held (``_hold``). A value that does not then start a URL is left as - written. + An attribute's value has its references decoded + (``_decode_in_an_attribute``), then the C0 controls and spaces at its ends + stripped and its tabs and line breaks deleted, as HTML and then the URL + Standard read it: ``href=" https://..."`` is a URL. A destination's + backslash escapes and references are decoded, as Markdown decodes them, + and then the whitespace at its ends stripped, as markdown-it strips it + (Python's ``str.strip``, in mdurl) before following a link: + ``[a](< https://... >)`` is one. What the URL then still holds that a + reading would end it at is held (``_hold``). A value that does not then + start a URL is left as written. """ kind = enclosed.lastgroup if kind == "angle": - url = _MARKDOWN_ESCAPE.sub(_decode_in_a_url, enclosed[kind]) + url = _MARKDOWN_ESCAPE.sub(_decode_in_an_attribute, enclosed[kind]).strip() else: - url = _HTML_REFERENCE.sub(_decode_in_a_url, enclosed[kind]) + url = _HTML_REFERENCE.sub(_decode_in_an_attribute, enclosed[kind]) url = _URL_BREAKS.sub("", url.strip(_C0_OR_SPACE)) if not _STARTS_A_URL.match(url): return enclosed[0] From 2822aff1ce521b2db42593ac38c7d81b44ca98a2 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 01:39:23 -0400 Subject: [PATCH 16/21] Refuse line citations beside a dot segment, and read a URL through its query's whitespace Round 10's review found four links of this repository that no rule refused, each confirmed with markdown-it-py 4.2, parse5 7.3 and bun 1.3: an angle destination after the space CommonMark allows before its `<`, an attribute whose opening quote a code span's `="` had consumed, a backslash that Markdown writes as `%5C` and a reading took as `/`, and an ordinary destination padded with ` `. In each, a `..` removes a segment that holds whitespace or a backslash, and the renderer removes another segment than a reading does. Every line citation in a string that holds a dot segment is now refused, pinned or not (`dot_segment_line_citations`): `..` as a whole segment beside `/` or `\`, either dot perhaps `%2e`, in any source a reading starts from. Without one, whitespace or a backslash stays in the segment a renderer follows, and a test checks that no source file here has either in its path. No live string holds a dot segment. GitHub ignores a query, so whitespace in one ended the URL wherever no enclosure held it, and the line link was refused only where its `#L` fell within the backstop's 24 characters. Where a string holds a `?` and a `#`, its URLs are now also read through their query's whitespace, ending the query at an angle bracket, a `"` or a `'`. An angle destination is also found after spaces, tabs and one line break. The counterexample test runs the four findings and their variants (11 inputs) and 14 query shapes, each rendered by the three oracles. Two properties cover the rule and the query reading, and a test checks the premise against this tree's file names. The review's 679 misses are now refused, its 27,625-input matrix is unchanged, and the 288 over-refusal probes without a dot segment gain no refusal. Each mutant that only random generation killed at `c36671e6`'s tests (M12 survived round 10's run that way) now has an explicit example. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 766 +++++++++++++++++- .../microcosm_build/spec_source_citations.py | 143 +++- 3 files changed, 877 insertions(+), 34 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index 61d6ec986a..3741ab11b0 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A citation any reading finds is checked. An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A destination in angle brackets is found after the spaces, tabs and one line break CommonMark allows before its `<`. Where a string holds a `?` and a `#`, its URLs are also read through whitespace in their query, which GitHub ignores (`href=".../x.py?a b#L12"`). A citation any reading finds is checked. Every line citation in a string that holds a dot segment (`..` as a whole path segment, either dot perhaps `%2e`) is refused, pinned or not: whitespace, a backslash or markup in the segment it removes can make a renderer remove another one than any reading does, as in `[a]( )` or `[a](https://github.com/.../tools/a\b/../x.py#L12)`, where Markdown writes the backslash `%5C`. No resource cites a line in such a string. An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index b972a97c7a..b0bf686a65 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -16,6 +16,11 @@ distribution directory, from ``src/``, from the ``microcosm`` namespace, from its shard, or from a first-level subpackage of its shard. A bare file name cannot be resolved and falls to the next rule; +- no resource cites a line in a string that holds a dot segment (``..`` as + a path segment, either dot perhaps ``%2e``), pinned or not + (``dot_segment_line_citations``): whitespace, a backslash or markup in the + segment it removes can make a renderer remove another one than a reading + does, so which file the citation names is unknown; - no resource cites a line of any other source unless the citation binds to an inline commit (the nearest one in its own clause, else the nearest one before it in its string; ``citing_commit``), or the resource pins a path @@ -75,6 +80,7 @@ citing_commit, commit_pinned_sources, country_packages, + dot_segment_line_citations, dotted_microcosm_symbols, in_repository_line_citations, line_citations, @@ -123,6 +129,16 @@ def test_no_resource_cites_a_line_of_this_repository(country: str) -> None: assert offenders == {} +@pytest.mark.parametrize("country", PACKAGES) +def test_no_resource_cites_a_line_in_a_string_with_a_dot_segment(country: str) -> None: + offenders = { + name: hits + for name, payload in package_payloads(country).items() + if (hits := dot_segment_line_citations(payload)) + } + assert offenders == {} + + @pytest.mark.parametrize("country", PACKAGES) def test_no_resource_misspells_a_commit_pin(country: str) -> None: offenders = { @@ -580,6 +596,30 @@ def _is_commit_like(sha: str) -> bool: depth=0, as_key=False, ) +@example( + stem="x", + suffix="py", + directory="datasets/cps/", + locator="{file}#L{a}", + a=12, + b=12, + prefix="", + tail="", + depth=2, + as_key=True, +) +@example( + stem="x", + suffix="py", + directory="", + locator="ll. {a}-{b} of {file}", + a=12, + b=19, + prefix="", + tail="", + depth=0, + as_key=False, +) def test_an_unpinned_line_citation_is_refused_at_any_depth( stem: str, suffix: str, @@ -714,6 +754,17 @@ def test_a_line_citation_of_a_file_pinned_at_a_commit_passes( b=st.integers(1, 99_999), commit=_SHA, ) +@example( + stem="x", + suffix="py", + cited_directory="archived_data/datasets/cps/", + pin_directory="", + root="", + locator="{file}:{a}", + a=12, + b=12, + commit="a1b2c3d", +) def test_a_path_pin_covers_exactly_the_citations_its_path_ends_in( stem: str, suffix: str, @@ -752,6 +803,42 @@ def test_a_path_pin_covers_exactly_the_citations_its_path_ends_in( depth=st.integers(0, 3), as_key=st.booleans(), ) +# Only a commit in an earlier clause tests the fallback to the nearest commit +# before a citation, and a whole run can draw no ``;`` between them. +@example( + stem="x", + suffix="py", + directory="", + locator="{file}:{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, + prefix="", + middle=";", + clause_middle="", + tail="", + pin_first=True, + depth=0, + as_key=False, +) +@example( + stem="x", + suffix="py", + directory="", + locator="{file}:{a}", + pin="https://github.com/PolicyEngine/archived-data/raw/{sha}/x", + sha="a1b2c3d", + a=12, + b=12, + prefix="", + middle="", + clause_middle="", + tail="", + pin_first=False, + depth=0, + as_key=False, +) def test_a_commit_before_a_citation_or_later_in_its_clause_pins_it( stem: str, suffix: str, @@ -796,6 +883,34 @@ def test_a_commit_before_a_citation_or_later_in_its_clause_pins_it( clause_break=st.sampled_from(_CLAUSE_BREAKS), tail=_PROSE, ) +@example( + stem="x", + suffix="py", + directory="", + locator="{file}:{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, + prefix="", + middle="", + clause_break="; ", + tail="", +) +@example( + stem="x", + suffix="py", + directory="", + locator="{file}:{a}", + pin="Commit {sha}", + sha="a1b2c3d", + a=12, + b=12, + prefix="", + middle="", + clause_break=". See ", + tail="", +) def test_a_commit_only_in_a_later_clause_does_not_pin_a_citation( stem: str, suffix: str, @@ -832,6 +947,19 @@ def test_a_commit_only_in_a_later_clause_does_not_pin_a_citation( middle=_CLAUSE_PROSE, ref_first=st.booleans(), ) +@example( + stem="x", + suffix="py", + directory="", + locator="{file}:{a}", + pin="commit {sha}", + sha="a1b2c3d", + moving="on refs/heads/main", + a=12, + b=12, + middle="", + ref_first=False, +) def test_an_earlier_commit_does_not_pin_a_citation_whose_clause_names_a_moving_ref( stem: str, suffix: str, @@ -931,6 +1059,15 @@ def test_each_citation_binds_the_commit_in_its_own_clause( a=st.integers(1, 99_999), b=st.integers(1, 99_999), ) +@example( + stem="x", + suffix="py", + locator="{file}:{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) def test_a_commit_named_in_another_string_does_not_pin_a_citation( stem: str, suffix: str, @@ -965,6 +1102,22 @@ def test_a_commit_named_in_another_string_does_not_pin_a_citation( ), a=st.integers(1, 99_999), ) +@example( + stem="x", + suffix="py", + locator="{file}:{a}", + keyword="commit ", + not_a_commit="1234567", + a=12, +) +@example( + stem="x", + suffix="py", + locator="{file}:{a}", + keyword="commit ", + not_a_commit="e2de2b41…", + a=12, +) def test_a_decimal_a_word_a_digest_a_truncation_or_a_branch_names_no_commit( stem: str, suffix: str, @@ -1021,6 +1174,28 @@ def test_a_hex_directory_after_a_pin_word_names_no_commit( branch=st.sampled_from(_BRANCH_NAMES), pin_other_file=st.booleans(), ) +@example( + stem="x", + other="y", + suffix="py", + locator="{file}:{a}", + a=12, + b=12, + commit="a1b2c3d", + branch="main", + pin_other_file=True, +) +@example( + stem="x", + other="y", + suffix="py", + locator="{file}:{a}", + a=12, + b=12, + commit="a1b2c3d", + branch="main", + pin_other_file=False, +) def test_a_pin_of_another_file_or_at_a_branch_does_not_cover_a_citation( stem: str, other: str, @@ -1060,6 +1235,7 @@ def test_a_pin_of_another_file_or_at_a_branch_does_not_cover_a_citation( ), a=st.integers(1, 99_999), ) +@example(stem="x", suffix="py", locator="{file}:{a}", not_a_commit="1234567", a=12) def test_a_path_pin_whose_commit_is_no_sha_pins_nothing( stem: str, suffix: str, locator: str, not_a_commit: str, a: int ) -> None: @@ -1085,6 +1261,32 @@ def test_a_path_pin_whose_commit_is_no_sha_pins_nothing( middle=_CLAUSE_PROSE, place=st.sampled_from(("gap", "clause_start", "clause_end")), ) +@example( + stem="x", + suffix="py", + locator="{file}:{a}", + pin="commit {sha}", + sha="a1b2c3d", + stray="e4f5a6b", + word="rev ", + a=12, + b=12, + middle="", + place="gap", +) +@example( + stem="x", + suffix="py", + locator="{file}:{a}", + pin="commit {sha}", + sha="a1b2c3d", + stray="e4f5a6b", + word="rev ", + a=12, + b=12, + middle="", + place="clause_end", +) def test_a_commit_in_no_pin_spelling_beside_a_citation_is_refused( stem: str, suffix: str, @@ -1126,6 +1328,17 @@ def test_a_commit_in_no_pin_spelling_beside_a_citation_is_refused( middle=_CLAUSE_PROSE, stray_first=st.booleans(), ) +@example( + stem="x", + suffix="py", + locator="{file}:{a}", + sha="a1b2c3d", + stray="e4f5a6b", + a=12, + b=12, + middle="", + stray_first=False, +) def test_a_commit_like_token_in_another_clause_is_no_misspelled_pin( stem: str, suffix: str, @@ -1185,6 +1398,19 @@ def test_a_symbol_citation_is_not_a_line_citation( depth=st.integers(0, 3), as_key=st.booleans(), ) +@example( + stem="x", + suffix="py", + directory="", + locator="{file} {symbol}, lines {a}-{b}", + symbol="helper", + a=12, + b=19, + prefix="", + tail="", + depth=0, + as_key=False, +) def test_a_line_locator_in_an_unrecognised_spelling_is_refused( stem: str, suffix: str, @@ -1217,6 +1443,16 @@ def test_a_line_locator_in_an_unrecognised_spelling_is_refused( pinned_inline=st.booleans(), pinned_by_path=st.booleans(), ) +@example( + file=_IN_REPOSITORY_FILES[0], + locator="{file}:{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, + pinned_inline=True, + pinned_by_path=True, +) def test_a_line_citation_of_this_repository_is_refused_even_when_pinned( file: str, locator: str, @@ -1294,6 +1530,15 @@ def test_in_repository_paths_resolve_from_an_anchor_or_a_repository_prefix() -> assert COUNTRY_PACKAGE_ROOT.is_dir() +def test_no_source_file_here_has_whitespace_or_a_backslash_in_its_path() -> None: + # The dot-segment rule relies on it: without a ``..``, a renderer keeps + # whitespace or a backslash in the segment it follows (``%20``, ``%5C``), + # so its link names no source file here. + tails = citations._repository_path_tails(REPOSITORY_ROOT) + assert "tools/ci_test_plan.py" in tails + assert sorted(tail for tail in tails if re.search(r"[\s\\]", tail)) == [] + + def test_only_anchored_tails_name_a_file_here(tmp_path: Path) -> None: files = ( "packages/microcosm-build/src/microcosm/build/us_runtime/utils/loss.py", @@ -1433,6 +1678,207 @@ def test_the_round_three_review_counterexample_is_refused() -> None: a=st.integers(1, 99_999), b=st.integers(1, 99_999), ) +# One spelling each: an escape in the route, an escaped slash after the ref, +# a ``./`` and a ``..`` detour, an escaped, a backslashed and a full-width +# host, a user, and a port on a host written without a scheme. +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="", + path="tools/build_us_puf_support_base.py", + chosen={1}, + upper=False, + case=str.lower, + leads=("https://", "https://"), + authority="{host}", + wide=set(), + stop=_FULL_STOPS[0], + escaped_host=set(), + dot=False, + backslashed=set(), + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="", + path="tools/build_us_puf_support_base.py", + chosen={36}, + upper=False, + case=str.lower, + leads=("https://", "https://"), + authority="{host}", + wide=set(), + stop=_FULL_STOPS[0], + escaped_host=set(), + dot=False, + backslashed=set(), + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="./", + path="tools/build_us_puf_support_base.py", + chosen=set(), + upper=False, + case=str.lower, + leads=("https://", "https://"), + authority="{host}", + wide=set(), + stop=_FULL_STOPS[0], + escaped_host=set(), + dot=False, + backslashed=set(), + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="docs/../", + path="tools/build_us_puf_support_base.py", + chosen=set(), + upper=False, + case=str.lower, + leads=("https://", "https://"), + authority="{host}", + wide=set(), + stop=_FULL_STOPS[0], + escaped_host=set(), + dot=False, + backslashed=set(), + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="", + path="tools/build_us_puf_support_base.py", + chosen=set(), + upper=False, + case=str.lower, + leads=("https://", "https://"), + authority="{host}", + wide=set(), + stop=_FULL_STOPS[0], + escaped_host={0}, + dot=False, + backslashed=set(), + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="", + path="tools/build_us_puf_support_base.py", + chosen=set(), + upper=False, + case=str.lower, + leads=("https://", "https://"), + authority="{host}", + wide=set(), + stop=_FULL_STOPS[0], + escaped_host=set(), + dot=False, + backslashed={0}, + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="", + path="tools/build_us_puf_support_base.py", + chosen=set(), + upper=False, + case=str.lower, + leads=("https://", "https://"), + authority="{host}", + wide={0}, + stop=_FULL_STOPS[0], + escaped_host=set(), + dot=False, + backslashed=set(), + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="", + path="tools/build_us_puf_support_base.py", + chosen=set(), + upper=False, + case=str.lower, + leads=("https://", "https://"), + authority="user@{host}", + wide=set(), + stop=_FULL_STOPS[0], + escaped_host=set(), + dot=False, + backslashed=set(), + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + url="https://github.com/PolicyEngine/microcosm/blob/", + ref="a1b2c3d", + detour="", + path="tools/build_us_puf_support_base.py", + chosen=set(), + upper=False, + case=str.lower, + leads=("", ""), + authority="{host}:443", + wide=set(), + stop=_FULL_STOPS[0], + escaped_host=set(), + dot=False, + backslashed=set(), + query="", + fragment="#L{a}", + pin="commit {sha}", + sha="a1b2c3d", + a=12, + b=12, +) def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( url: str, ref: str, @@ -1516,6 +1962,17 @@ def test_a_url_of_this_repository_is_refused_however_github_lets_it_be_written( a=st.integers(1, 99_999), b=st.integers(1, 99_999), ) +@example( + directory="", + stem="x", + suffix="py", + query="plain=1", + fragment="#L{a}", + sha="a1b2c3d", + pinned=False, + a=12, + b=12, +) def test_a_query_does_not_hide_a_line_citation( directory: str, stem: str, @@ -1546,6 +2003,199 @@ def test_a_query_does_not_hide_a_line_citation( assert unpinned == ([] if pinned else found) +#: A dot segment's two dots: either perhaps ``%2e``, written as a Markdown or +#: an HTML escape, or split by a tab, which the URL Standard deletes. +_DOT_DOTS = ( + "..", + ".%2e", + "%2E.", + "%2e%2E", + "..", + "..", + "\\.\\.", + ".\t.", +) +#: What is no dot segment where one would be. +_NOT_DOT_DOTS = (".", "...", "x", "%2e", ".x", "x..", ".") +#: Strings with a dot segment ({dots} beside a {separator}) and a line +#: citation ({url}), in prose, in markup, or in the cited URL's own path +#: ({inside}). +_DOT_SEGMENT_PLACES = ( + "{url} see docs{separator}{dots}", + "{url} see {dots}{separator}docs", + "{dots}{separator}docs{separator}notes: {url}", + "{url} (docs{separator}{dots}{separator}notes)", + '{url}', + "[a](<{dots}{separator}x>) {url}", + "{inside}", +) + + +@settings(max_examples=300, deadline=None) +@given( + place=st.sampled_from(_DOT_SEGMENT_PLACES), + dots=st.sampled_from(_DOT_DOTS), + not_dots=st.sampled_from(_NOT_DOT_DOTS), + separator=st.sampled_from(("/", "\\")), + directory=_FOREIGN_DIRECTORY, + stem=_STEM, + suffix=_SUFFIX, + pinned=st.booleans(), + sha=_SHA, + a=st.integers(1, 99_999), + depth=st.integers(0, 2), + as_key=st.booleans(), +) +@example( + place="{inside}", + dots="..", + not_dots=".", + separator="/", + directory="", + stem="x", + suffix="py", + pinned=True, + sha="a1b2c3d", + a=12, + depth=0, + as_key=False, +) +@example( + place="{url} see docs{separator}{dots}", + dots="%2e%2E", + not_dots="...", + separator="\\", + directory="datasets/cps/", + stem="x", + suffix="py", + pinned=False, + sha="a1b2c3d", + a=12, + depth=1, + as_key=True, +) +@example( + place='{url}', + dots="..", + not_dots=".", + separator="/", + directory="", + stem="x", + suffix="py", + pinned=True, + sha="a1b2c3d", + a=12, + depth=0, + as_key=False, +) +@example( + place="{dots}{separator}docs{separator}notes: {url}", + dots=".\t.", + not_dots="x..", + separator="/", + directory="", + stem="x", + suffix="py", + pinned=True, + sha="a1b2c3d", + a=12, + depth=0, + as_key=False, +) +def test_a_dot_segment_refuses_every_line_citation_in_its_string_pinned_or_not( + place: str, + dots: str, + not_dots: str, + separator: str, + directory: str, + stem: str, + suffix: str, + pinned: bool, + sha: str, + a: int, + depth: int, + as_key: bool, +) -> None: + blob = "https://github.com/PolicyEngine/upstream-data/blob/" + blob += f"{sha if pinned else 'main'}/{directory}" + + def written(dots: str) -> str: + return place.format( + url=f"{blob}{stem}.{suffix}#L{a}", + # A letter after the separator: ``\_`` would be an escaped ``_``. + inside=f"{blob}x{separator}{dots}{separator}docs/{stem}.{suffix}#L{a}", + dots=dots, + separator=separator, + ) + + document = _plant(written(dots), depth, as_key) + assert dot_segment_line_citations(document) == line_citations(document) != [] + # Without the dot segment, the same string's citations pass this rule. + document = _plant(written(not_dots), depth, as_key) + assert line_citations(document) + assert dot_segment_line_citations(document) == [] + + +#: Markup that holds a URL ({url}) with whitespace in it. +_URL_ENCLOSURES = ( + "[a](<{url}>)", + "[a]( <{url}>)", + "[a](\t<{url}>)", + "[a](\n<{url}>)", + "[a]\n\n[a]: <{url}>", + 'a', + "a", + 'Use `prefix="` here.\n\na', +) + + +@settings(max_examples=200, deadline=None) +@given( + enclosure=st.sampled_from(_URL_ENCLOSURES), + path=st.sampled_from(_ROOTED_FILES), + sha=_SHA, + query=st.text(alphabet="ab=&+% \t\u00a0\u3000", max_size=60), + fragment=st.sampled_from(("#L{a}", "#L{a}-L{b}")), + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), +) +@example( + enclosure='Use `prefix="` here.\n\na', + path="tools/build_us_puf_support_base.py", + sha="a1b2c3d", + query="a" + " " * 30 + "b", + fragment="#L{a}", + a=12, + b=12, +) +@example( + enclosure="[a]( <{url}>)", + path="tools/build_us_puf_support_base.py", + sha="a1b2c3d", + query="a\tb" * 15, + fragment="#L{a}-L{b}", + a=3, + b=9, +) +def test_a_line_link_of_this_repository_is_refused_whatever_whitespace_its_query_holds( + enclosure: str, + path: str, + sha: str, + query: str, + fragment: str, + a: int, + b: int, +) -> None: + # Each renders as a link to the file at its lines: GitHub ignores the + # query, however long a run of spaces, tabs or Unicode spaces it holds. + cited = f"https://github.com/PolicyEngine/microcosm/blob/{sha}/{path}" + url = f"{cited}?{query}{fragment.format(a=a, b=b)}" + document = {"note": enclosure.format(url=url)} + assert cited in { + citation.file for citation in in_repository_line_citations(document) + } + + def test_the_round_four_review_counterexamples_are_refused() -> None: blob = "https://github.com/PolicyEngine/microcosm/blob/" blob += "1840bc92f8d155adb685f1090245058ebb990df0/" @@ -1677,6 +2327,9 @@ def test_the_round_seven_review_counterexamples_are_refused() -> None: for written, file in ( (f"[https://example.org/?x](<{cited}#L12>)", cited), (f"<{cited}#L12>", cited), + # With no query, only a reading that ends the first URL at its ``>`` + # reads the host a browser lower-cases. + (f"<{cited.replace('github', 'GitHub')}#L12>", cited), (f"[https://example.org/?x]({cited}#L12)", cited), (f"[https://example.org/?x]({network}#L12)", network), (f"[https://example.org/?x]({bare}#L12)", bare), @@ -1684,10 +2337,11 @@ def test_the_round_seven_review_counterexamples_are_refused() -> None: (f'{cited}#L12', cited), (f"'https://example.org/?x','{cited}#L12'", cited), ): - assert [ + # Read through the first URL's query, the string cites the file again. + assert { citation.file for citation in in_repository_line_citations({"note": written}) - ] == [file], written + } == {file}, written def test_a_string_is_read_as_a_browser_and_as_markdown_delimit_its_urls() -> None: @@ -1860,6 +2514,96 @@ def test_the_round_nine_review_counterexamples_are_refused() -> None: assert unrecognised_line_locators(document) == [], written +def test_the_round_ten_review_counterexamples_are_refused() -> None: + blob = "https://github.com/PolicyEngine/microcosm/blob/" + blob += "29983852e5182266c9a4e565731920ea972fe695/" + tools = blob + "tools/" + cited = tools + "ci_test_plan.py" + network = tools.removeprefix("https:") + code_span = 'Use `prefix="` before the value.\n\n' + # markdown-it-py 4.2 and parse5 7.3 render each of these, and bun 1.3's + # ``URL`` reads each, as a link to line 12 of the file: the ``..`` + # removes a segment that holds whitespace or a backslash. Before round 11 + # no rule refused them. + for written, in_repository in ( + # CommonMark allows spaces, tabs and one line break before ``<``. + (f"[a]( <{tools}a b/../ci_test_plan.py#L12>)", True), + (f"[a](\t<{tools}a b/../ci_test_plan.py#L12>)", True), + (f"[a](\n<{tools}a b/../ci_test_plan.py#L12>)", True), + (f"[a]( \t\n \t<{tools}a b/../ci_test_plan.py#L12>)", True), + (f"[a]\n\n[a]: \n<{tools}a b/../ci_test_plan.py#L12>", True), + # The ``="`` in the code span starts an enclosure that ends at the + # attribute's opening quote, so no reading holds its value. + (f'{code_span}a', False), + (f"{code_span}a", True), + # Markdown writes a backslash it leaves as ``%5C``, so ``a\b`` is one + # segment; a reading takes the backslash as ``/``. + (f"[a](<{tools}a\\b/../ci_test_plan.py#L12>)", False), + (f"[a]({tools}a\\b/../ci_test_plan.py#L12)", False), + # markdown-it decodes the references, then strips the destination. + (f"[a]( {network}a b/../ci_test_plan.py#L12)", False), + (f"[a]( {tools}a b/../ci_test_plan.py#L12)", True), + ): + document = {"note": written} + assert dot_segment_line_citations(document), written + assert { + citation.file for citation in in_repository_line_citations(document) + } == ({cited} if in_repository else set()), written + # GitHub ignores a query, so whitespace in one changes no file: each of + # these links to line 12 (or 3-9) of the file, and a reading through the + # query finds it however far the whitespace runs. + query = "?" + "a b=c d&" * 4 + no_break, tab = "a\u00a0b" * 10, "a\tb" * 10 + for written in ( + f"[a]( <{cited}{query}#L12>)", + f"[a](\t<{cited}{query}#L12>)", + f"[a](\n<{cited}{query}#L12>)", + f"[a](<{cited}{query}#L12>)", + f'{code_span}a', + f"a", + f'a', + f"[a](<{cited}?{'a b' * 6}#L12>)", + f"[a](<{cited}?{no_break}#L12>)", + f"[a](<{cited}?{tab}#L12>)", + f"[a](<{cited}?{'a b' * 10}#L3-L9>)", + # Where no enclosure holds the link, a query read through to a ``#`` + # swallows a later URL, so one reading ends a query at each + # enclosure's end: a ``"``, a ``'`` and an angle bracket. + f'Use `prefix="` here. a', + f"Use `prefix='` here. a", + f'Use `prefix="` here [x]() [a](<{cited}{query}#L12>) "', + ): + assert { + citation.file + for citation in in_repository_line_citations({"note": written}) + } == {cited}, written + + +def test_a_dot_segment_refuses_the_line_citations_of_its_string_only() -> None: + pinned = "https://github.com/PolicyEngine/upstream-data/blob/a1b2c3d/x.py#L12" + # A run of dots that is no whole segment, and a lone ``.``, remove nothing. + for written in ( + f"{pinned}; rows 1 .. 9", + f"{pinned} via x/./y", + f"{pinned} at sha...", + f"{pinned}; a.b..c", + "x.py:12 at a1b2c3d; see docs/.../notes", + "see ../docs and x/../y; no line here", + ): + assert dot_segment_line_citations({"note": written}) == [], written + # A dot segment anywhere in the string refuses each citation in it, and + # none in another string. + for written, count in ( + (f"{pinned}; see ../docs", 1), + ("x.py:12 at a1b2c3d; see docs/%2e%2E/notes", 1), + ('x.py:12 at a1b2c3d; x', 1), + ("x.py:12 and y.py:30 at a1b2c3d; see ../docs", 2), + ): + document = {"note": written, "other": pinned} + assert dot_segment_line_citations(document) == line_citations(written) + assert len(dot_segment_line_citations(document)) == count, written + + def test_a_string_is_read_again_with_its_markup_escapes_decoded() -> None: def readings(text: str) -> list[str]: return list(citations._readable_strings(text)) @@ -2177,6 +2921,24 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: line_citations(text) unrecognised_line_locators(text) assert time.process_time() - started < 2, text[:24] + # Since round 11, in 2 s of their own: runs of what a dot segment, a URL + # read through its query and a destination after whitespace start with. + started = time.process_time() + for text in ( + "/.." * 30_000, + "=%2e." * 20_000, + "\\..\\" * 25_000, + "https://a?" + " " * 100_000 + "#", + "https:" * 20_000 + " ?#", + "https://a?b #" * 10_000, + "?#https://a#" + "." * 100_000, + "](" + " " * 100_000 + "\n<", + "](\t \t/`` URL, at a ref @@ -82,6 +86,19 @@ refused even when pinned. A bare file name cannot be told apart from another repository's, so it must be pinned. +A dot segment (``..`` as a whole path segment, either dot perhaps ``%2e``) +removes the segment before it, and whitespace, a backslash or markup in that +segment can make a renderer remove another one than a reading does: in +``[a]( )`` the destination +follows a space, and in ``[a](https://github.com/.../tools/a\\b/../x.py#L12)`` +Markdown writes the backslash as ``%5C``, so both link to line 12 of +``tools/x.py``. So every line citation in a string that holds a dot segment, +in any of its readings' sources, is refused, pinned or not +(``dot_segment_line_citations``). Without one, whitespace or a backslash that +a reading takes otherwise than a renderer stays in the segment the renderer +follows (``%20``, ``%5C``), and no file under ``_SOURCE_ROOTS`` has either in +its path, so that link names no source file here. + The recognised spellings are the ones ``_LINE_CITATIONS`` documents. A source name followed by a line locator in no recognised spelling, with no ``.``, ``;`` or line break between (``x.py some_function, lines 12-19``), is refused @@ -236,6 +253,23 @@ re.compile(_URL_START + r"[^\t\n\f\r ]*[^\s\"'`‘’“”<>.,;:!?)\]]"), re.compile(_URL_START + r"[^\t\n\f\r \"'`‘’“”<>()]*[^\s\"'`‘’“”<>()\].,;:!?]"), ) +#: A URL whose query holds whitespace, as an HTML attribute or a Markdown +#: destination in angle brackets holds one: as ``_DELIMITED_URL`` up to its +#: query, which then runs through whitespace to its fragment or a line break. +#: GitHub ignores a query, so ``.../x.py?a b#L12`` opens line 12 of ``x.py`` +#: (``_read_url`` drops the query). One reading stops the query at an angle +#: bracket, one at ``"`` and one at ``'``: what can end each enclosure. A query +#: is optional, so each start is read once. Tried where a string holds a +#: ``?`` and a ``#``. +_URLS_THROUGH_A_QUERY = tuple( + re.compile( + _URL_START + + r"[^\s?#]*+(?:\?[^" + + stop + + r"#\n\r]*+)?(?:#[^\s\"'`‘’“”<>()]*[^\s\"'`‘’“”<>()\].,;:!?])?" + ) + for stop in ("<>", '"', "'") +) #: Whitespace other than ASCII whitespace: what ``_URLS_THROUGH_UNICODE_SPACES`` #: reads a URL through. _UNICODE_SPACE = re.compile(r"[^\S\t\n\f\r ]") @@ -266,14 +300,22 @@ _HELD_IN_A_URL = re.compile(r"[\s\"'`‘’“”<>()]") #: Where a URL can hold any whitespace: an HTML attribute's quoted value #: (``href="..."``, ``href='...'``), to its closing quote, and a Markdown -#: destination in angle brackets (``[a](<...>)``, ``[a]: <...>``), to an +#: destination in angle brackets (``[a](<...>)``, ``[a]: <...>``, after any +#: spaces or tabs and at most one line break, as CommonMark allows), to an #: unescaped ``>``. _ENCLOSED_URL = re.compile( r"=[\t\n\f\r ]*+(?:\"(?P[^\"]*+)\"|'(?P[^']*+)')" - r"|\](?:\(|:[\t ]*+(?:\r\n?|\n)?[\t ]*+)<(?P(?:\\[^\n\r]|[^<>\n\r\\])*+)>" + r"|\][(:][\t ]*+(?:\r\n?|\n)?[\t ]*+<(?P(?:\\[^\n\r]|[^<>\n\r\\])*+)>" ) #: A URL start at the start of a string. _STARTS_A_URL = re.compile(_URL_START) +#: A dot segment: ``..`` as a whole path segment beside a slash or a +#: backslash (``x/../y``, ``../y``, ``x\\..``), either ``.`` perhaps written +#: ``%2e`` (``.%2E``, ``%2e%2e``), as the URL Standard reads one. +_DOT_SEGMENT = re.compile( + r"(?<=[/\\])(?:\.|%2[eE]){2}(?![^/\\?#\s\"'`‘’“”<>()\[\]])" + r"|(?()\[\]=])(?:\.|%2[eE]){2}(?=[/\\])" +) #: What the URL Standard strips from both ends of a URL: C0 controls and #: spaces. _C0_OR_SPACE = "".join(map(chr, range(0x21))) @@ -639,11 +681,8 @@ def _read_enclosed(enclosed: re.Match[str]) -> str: return enclosed[0][:start] + _HELD_IN_A_URL.sub(_hold, url) + enclosed[0][end:] -def _readable_strings(value: object) -> Iterator[str]: - """``nested_strings``, with the URLs in them read as ``_read_url`` reads them. - - Each string is read from several sources, each the string as one kind of - reader takes it before following a link: +def _sources(text: str) -> dict[str, None]: + """A string as each kind of reader takes it before following a link. - as written; - with each HTML attribute's quoted value and each Markdown destination @@ -658,34 +697,56 @@ def _readable_strings(value: object) -> Iterator[str]: ````, where HTML decodes the reference and leaves the backslash. - Each source is read again with its ``_URL_BREAKS`` deleted, as the URL - Standard reads a URL (``href=".../x.p y#L12"``). Each is read with each - URL running to whitespace (``_URL``), as a browser reads it, and with each - URL also ending at a quote, a parenthesis or an angle bracket - (``_DELIMITED_URL``), as Markdown and prose delimit one, so a link's - destination is not lost in the query of a URL before it. Where a source - holds a Unicode space, its URLs are also read through Unicode spaces - (``_URLS_THROUGH_UNICODE_SPACES``). Each reading that differs from the + Each is also taken with its ``_URL_BREAKS`` deleted, as the URL Standard + reads a URL (``href=".../x.p y#L12"``). Each source that differs from + the others is kept once, in that order. + """ + + sources = [text, _ENCLOSED_URL.sub(_read_enclosed, text)] + if "&" in text or "\\" in text: + sources += ( + escapes.sub(decode, text) + for escapes in (_MARKDOWN_ESCAPE, _HTML_REFERENCE) + for decode in (_decode_in_text, _decode_in_a_url) + ) + return dict.fromkeys( + joined + for source in dict.fromkeys(sources) + for joined in dict.fromkeys((source, _URL_BREAKS.sub("", source))) + ) + + +def _url_patterns(source: str) -> tuple[re.Pattern[str], ...]: + """The URL patterns a source is read with: ``_URLS``, and where it holds + a Unicode space or a ``?`` and a ``#``, the ones that read through them.""" + + patterns = _URLS + if _UNICODE_SPACE.search(source): + patterns += _URLS_THROUGH_UNICODE_SPACES + if "?" in source and "#" in source: + patterns += _URLS_THROUGH_A_QUERY + return patterns + + +def _readable_strings(value: object) -> Iterator[str]: + """``nested_strings``, with the URLs in them read as ``_read_url`` reads them. + + Each string is read from each of its ``_sources``: with each URL running + to whitespace (``_URL``), as a browser reads it, and with each URL also + ending at a quote, a parenthesis or an angle bracket (``_DELIMITED_URL``), + as Markdown and prose delimit one, so a link's destination is not lost in + the query of a URL before it. Where a source holds a Unicode space, its + URLs are also read through Unicode spaces (``_URLS_THROUGH_UNICODE_SPACES``), + and where it holds a ``?`` and a ``#``, through the whitespace in their + query (``_URLS_THROUGH_A_QUERY``). Each reading that differs from the others is yielded once. """ for text in nested_strings(value): - sources = [text, _ENCLOSED_URL.sub(_read_enclosed, text)] - if "&" in text or "\\" in text: - sources += ( - escapes.sub(decode, text) - for escapes in (_MARKDOWN_ESCAPE, _HTML_REFERENCE) - for decode in (_decode_in_text, _decode_in_a_url) - ) yield from dict.fromkeys( - pattern.sub(_read_url, joined) - for source in dict.fromkeys(sources) - for joined in dict.fromkeys((source, _URL_BREAKS.sub("", source))) - for pattern in ( - _URLS + _URLS_THROUGH_UNICODE_SPACES - if _UNICODE_SPACE.search(joined) - else _URLS - ) + pattern.sub(_read_url, source) + for source in _sources(text) + for pattern in _url_patterns(source) ) @@ -1051,6 +1112,26 @@ def in_repository_line_citations( ] +def dot_segment_line_citations(value: object) -> list[LineCitation]: + """Line citations in a string that holds a dot segment, pinned or not. + + A dot segment (``_DOT_SEGMENT``) is removed with the segment before it, + and whitespace, a backslash or markup in that segment can make a renderer + remove another one than a reading does: + ``[a]( )`` links to line 12 + of ``tools/x.py``, though no reading takes ``a b`` as one segment. So + where any of a string's ``_sources`` holds one, every line citation in + the string is refused: the guard cannot tell which file it names. + """ + + return [ + citation + for text in nested_strings(value) + if any(map(_DOT_SEGMENT.search, _sources(text))) + for citation in line_citations(text) + ] + + def dotted_microcosm_symbols(value: object) -> set[str]: """Every dotted ``microcosm..*`` name in a payload's strings.""" From 8a62c9380858d16d94f58146c23cb8daaf8bb70c Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 06:16:07 -0400 Subject: [PATCH 17/21] Refuse a line link of this repository by its path, and read %2f dot segments and paths that hold = Round 11's review found two shapes of link to a line of this repository that no rule refused. markdown-it-py 4.2, parse5 7.3 and bun 1.3 render each as a link to line 12 of `tools/ci_test_plan.py` on github.com. In the first, the ref holds punctuation at which a reading starts a name (`blob/feature+foo/...`, `release!1`, `v1,2;3=4`, `a=b`), so the citation a reading finds (`foo/tools/ci_test_plan.py`) names no file here. In the second, a code span's `="` has taken an attribute's opening quote, and whitespace in the URL's user (`https://a b@%67ithub.com/...`) ends every reading's URL before its host. The review also named `%2f` beside `..`, which GitHub reads as a dot segment. A string is now refused where any reading's source holds this repository's GitHub path, `PolicyEngine/microcosm/`, with a line fragment at the first `#` after it (`line_links_of_this_repository`), whatever file the link names. The path is read as GitHub reads one: escapes decoded, a backslash as `/`, a run of slashes as one, `.` segments removed, and the owner and name in any case. A renderer's link to a line here names that path. Between a source and its URL lie the decodings the readings already apply, these normalizations, percent-encoding of characters that are neither in the path nor a `#`, and dot segments, which the dot-segment rule refuses. Relative links and redirects are out of scope. Prose that holds such a path and fragment without linking is refused too, by intent. A `..` beside `%2f` is now a dot segment. A path here can hold an `=` (`docs/evidence/margins/period=2025-01.json`), so the run of path characters before a cited name is also tried as an in-repository path, and a reverse locator reads a name after an `=` (`lines 3 of a/b=c.py`). The counterexample test runs 22 inputs: all render as links to line 12 here, no rule at 2822aff1 refused any, and the new rule refuses all 22. A property draws refs with punctuation, five spellings of github.com's authority, five of this repository's path and eight enclosures, and checks that a link here is refused and its twin in another repository is not. Rendered by the oracles, 2,880 inputs from that space and 2,000 random draws agree: every input here is refused, and none elsewhere. Across 4,482,233 live strings, the earlier rules' results are unchanged, the new rule refuses none, and no string holds a dot segment. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 327 +++++++++++++++++- .../microcosm_build/spec_source_citations.py | 148 +++++++- 3 files changed, 448 insertions(+), 29 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index 3741ab11b0..2f2cc4419f 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A destination in angle brackets is found after the spaces, tabs and one line break CommonMark allows before its `<`. Where a string holds a `?` and a `#`, its URLs are also read through whitespace in their query, which GitHub ignores (`href=".../x.py?a b#L12"`). A citation any reading finds is checked. Every line citation in a string that holds a dot segment (`..` as a whole path segment, either dot perhaps `%2e`) is refused, pinned or not: whitespace, a backslash or markup in the segment it removes can make a renderer remove another one than any reading does, as in `[a]( )` or `[a](https://github.com/.../tools/a\b/../x.py#L12)`, where Markdown writes the backslash `%5C`. No resource cites a line in such a string. An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A destination in angle brackets is found after the spaces, tabs and one line break CommonMark allows before its `<`. Where a string holds a `?` and a `#`, its URLs are also read through whitespace in their query, which GitHub ignores (`href=".../x.py?a b#L12"`). A citation any reading finds is checked. Every line citation in a string that holds a dot segment (`..` as a whole path segment, either dot perhaps `%2e`) is refused, pinned or not: whitespace, a backslash or markup in the segment it removes can make a renderer remove another one than any reading does, as in `[a]( )` or `[a](https://github.com/.../tools/a\b/../x.py#L12)`, where Markdown writes the backslash `%5C`. No resource cites a line in such a string. GitHub reads `%2f` as `/` before it resolves a path, so a `..` beside one is a dot segment too (`tools/a b%2f..%2fx.py`). A string is also refused where any of its readings' sources holds this repository's GitHub path, `PolicyEngine/microcosm/`, with a line fragment (`#L12`) at the first `#` after it, whatever file the link names and whether or not a reading finds the citation. The path is read as GitHub reads one: escapes decoded, a backslash as `/`, a run of slashes as one, `.` segments removed and the owner and name in any case. So a ref holding punctuation (`blob/feature+foo/tools/x.py#L12`), or whitespace before the host where markup has taken an attribute's opening quote (`href="https://a b@%67ithub.com/..."`), no longer hides a link of this repository. No resource holds such a link. A path here can hold an `=` (`docs/.../period=2025-01.json`), so the run of path characters before a cited name is tried as an in-repository path too, and a reverse locator reads a name after an `=` (`lines 3 of a/b=c.py`). An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index b0bf686a65..e28dc1bf3c 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -14,13 +14,19 @@ GitHub URL, or ``microcosm/`` before a top-level source directory), it is a file here or an anchored path form of one: its path from the root, from its distribution directory, from ``src/``, from the ``microcosm`` namespace, - from its shard, or from a first-level subpackage of its shard. A bare file - name cannot be resolved and falls to the next rule; + from its shard, or from a first-level subpackage of its shard, the run of + path characters before a cited name included (``period=2025-01.json``). + A bare file name cannot be resolved and falls to the next rule; - no resource cites a line in a string that holds a dot segment (``..`` as - a path segment, either dot perhaps ``%2e``), pinned or not - (``dot_segment_line_citations``): whitespace, a backslash or markup in the - segment it removes can make a renderer remove another one than a reading - does, so which file the citation names is unknown; + a path segment beside ``/``, ``\\`` or ``%2f``, either dot perhaps + ``%2e``), pinned or not (``dot_segment_line_citations``): whitespace, a + backslash or markup in the segment it removes can make a renderer remove + another one than a reading does, so which file the citation names is + unknown; +- no resource links a line of this repository by its GitHub path, however a + reading takes the link (``line_links_of_this_repository``): no string's + source holds ``PolicyEngine/microcosm/``, read as GitHub reads a path, + with ``#L`` at the first ``#`` after it; - no resource cites a line of any other source unless the citation binds to an inline commit (the nearest one in its own clause, else the nearest one before it in its string; ``citing_commit``), or the resource pins a path @@ -84,6 +90,7 @@ dotted_microcosm_symbols, in_repository_line_citations, line_citations, + line_links_of_this_repository, misspelled_commit_pins, named_commits, names_a_commit, @@ -139,6 +146,16 @@ def test_no_resource_cites_a_line_in_a_string_with_a_dot_segment(country: str) - assert offenders == {} +@pytest.mark.parametrize("country", PACKAGES) +def test_no_resource_links_a_line_of_this_repository(country: str) -> None: + offenders = { + name: hits + for name, payload in package_payloads(country).items() + if (hits := line_links_of_this_repository(payload)) + } + assert offenders == {} + + @pytest.mark.parametrize("country", PACKAGES) def test_no_resource_misspells_a_commit_pin(country: str) -> None: offenders = { @@ -1530,13 +1547,51 @@ def test_in_repository_paths_resolve_from_an_anchor_or_a_repository_prefix() -> assert COUNTRY_PACKAGE_ROOT.is_dir() -def test_no_source_file_here_has_whitespace_or_a_backslash_in_its_path() -> None: - # The dot-segment rule relies on it: without a ``..``, a renderer keeps - # whitespace or a backslash in the segment it follows (``%20``, ``%5C``), - # so its link names no source file here. +def test_every_source_path_here_is_read_whole() -> None: + # The in-repository rule relies on it: a reading ends a path at any + # character outside ``_PATH_HERE`` (whitespace, a backslash, a ``+``), so a + # path here holding one could be cited by a name no reading resolves. An + # ``=``, which evidence files here hold, is read through by a citation's + # ``path``. tails = citations._repository_path_tails(REPOSITORY_ROOT) assert "tools/ci_test_plan.py" in tails - assert sorted(tail for tail in tails if re.search(r"[\s\\]", tail)) == [] + assert sorted(tail for tail in tails if not set(tail) <= citations._PATH_HERE) == [] + + +def test_a_path_here_that_holds_an_equals_sign_is_read_whole(tmp_path: Path) -> None: + # ``_SOURCE`` starts a name after an ``=`` (``href=x.py`` names ``x.py``), + # so a cited name's run of path characters is tried too, forward and + # reverse, in prose and in a URL. + cited = "docs/evidence/margins/period=2025-01.json" + (tmp_path / cited).parent.mkdir(parents=True) + (tmp_path / cited).write_text("{}", encoding="utf-8") + (tmp_path / "tools").mkdir() + (tmp_path / "tools/x.py").write_text("", encoding="utf-8") + for written in ( + f"{cited} line 3 at a1b2c3d", + f"microcosm/{cited}:3 at a1b2c3d", + f"lines 3-4 of {cited} at a1b2c3d", + f"https://github.com/PolicyEngine/microcosm/blob/a1b2c3d/{cited}#L3", + ): + found = in_repository_line_citations({"note": written}, tmp_path) + assert {citation.file for citation in found} == {"2025-01.json"}, written + found = in_repository_line_citations( + {"note": "x"}, tmp_path + ) + assert [(citation.file, citation.path) for citation in found] == [ + ("tools/x.py", "href=tools/x.py") + ] + # Another path's run names nothing here, and a reverse locator reads a + # name after an ``=`` as a forward one does, pinned or not. + assert ( + in_repository_line_citations({"note": f"x={cited[5:]} line 3"}, tmp_path) == [] + ) + assert [ + citation.file + for citation in unpinned_line_citations( + {"note": "lines 3 of vendor/foo=bar.py"} + ) + ] == ["bar.py"] def test_only_anchored_tails_name_a_file_here(tmp_path: Path) -> None: @@ -2036,7 +2091,8 @@ def test_a_query_does_not_hide_a_line_citation( place=st.sampled_from(_DOT_SEGMENT_PLACES), dots=st.sampled_from(_DOT_DOTS), not_dots=st.sampled_from(_NOT_DOT_DOTS), - separator=st.sampled_from(("/", "\\")), + # GitHub reads ``%2f`` as ``/`` before it resolves a path. + separator=st.sampled_from(("/", "\\", "%2f", "%2F")), directory=_FOREIGN_DIRECTORY, stem=_STEM, suffix=_SUFFIX, @@ -2046,6 +2102,34 @@ def test_a_query_does_not_hide_a_line_citation( depth=st.integers(0, 2), as_key=st.booleans(), ) +@example( + place="{inside}", + dots="..", + not_dots=".", + separator="%2f", + directory="", + stem="x", + suffix="py", + pinned=True, + sha="a1b2c3d", + a=12, + depth=0, + as_key=False, +) +@example( + place="{dots}{separator}docs{separator}notes: {url}", + dots="..", + not_dots=".", + separator="%2F", + directory="", + stem="x", + suffix="py", + pinned=False, + sha="a1b2c3d", + a=12, + depth=0, + as_key=False, +) @example( place="{inside}", dots="..", @@ -2598,12 +2682,210 @@ def test_a_dot_segment_refuses_the_line_citations_of_its_string_only() -> None: ("x.py:12 at a1b2c3d; see docs/%2e%2E/notes", 1), ('x.py:12 at a1b2c3d; x', 1), ("x.py:12 and y.py:30 at a1b2c3d; see ../docs", 2), + ("x.py:12 at a1b2c3d; see ..%2fdocs", 1), + ("x.py:12 at a1b2c3d; see docs%2F.%2e%2fnotes", 1), ): document = {"note": written, "other": pinned} assert dot_segment_line_citations(document) == line_citations(written) assert len(dot_segment_line_citations(document)) == count, written +def test_the_round_eleven_review_counterexamples_are_refused() -> None: + sha = "2822aff1ce521b2db42593ac38c7d81b44ca98a2" + blob = "https://github.com/PolicyEngine/microcosm/blob/" + path = f"/PolicyEngine/microcosm/blob/{sha}/tools/ci_test_plan.py" + code_span = 'Use `prefix="` before the value.\n\n' + # markdown-it-py 4.2 and parse5 7.3 render each of these, and bun 1.3's + # ``URL`` reads each, as a link to line 12 of ``tools/ci_test_plan.py`` on + # github.com; at ``2822aff1`` no rule refused them. A ref can hold + # punctuation at which a reading starts a name, so the citation it finds + # (``foo/tools/ci_test_plan.py``) names no file here. + written = [] + for ref in ( + "feature+foo", + "feature@foo", + "release!1", + "v1,2;3=4", + "it's", + "a$b", + "a=b", + ): + written.append(f"[a](<{blob}{ref}/tools/ci_test_plan.py#L12>) at {sha}") + written.append( + f'a at {sha}' + ) + # Where a code span's ``="`` has taken an attribute's opening quote, a + # reading ends the URL at the whitespace in its user, and finds no host + # in what follows; so too with a query longer than the backstop's + # window, and with the owner escaped and its slash doubled, which GitHub + # reads through. + for authority in ( + "a b@%67ithub.com", + "a b@ｇithub.com", + "a b@github。com", + "a b@%67ithub.com:443", + "a\tb c@%47ITHUB.com", + ): + written.append(f'{code_span}a') + written += ( + f'{code_span}a', + f'{code_span}a', + ) + for text in written: + assert line_links_of_this_repository({"note": text}) == [text], text + # GitHub reads ``%2f`` as ``/`` and then resolves the path: github.com + # serves ``tools/ci_test_plan.py`` for this, though no source held a dot + # segment by round 11's definition. + text = ( + f'{code_span}a' + ) + assert line_links_of_this_repository({"note": text}) == [text] + assert dot_segment_line_citations({"note": text}) == line_citations(text) != [] + + +def test_a_line_link_of_this_repository_is_refused_by_its_path_alone() -> None: + # Whatever file it names: through markup and ``%`` escapes, backslashes, + # repeated slashes, ``.`` segments, a tab and the owner's case, as a + # browser and then GitHub read a path. A Makefile or a workflow outside + # the source directories drifts too. + for written in ( + "https://github.com/PolicyEngine/microcosm/blob/main/Makefile#L3", + "https://github.com/PolicyEngine/microcosm/blob/a1b2c3d/.github/workflows/ci.yml#L30", + "github.com/PolicyEngine/microcosm/blob/x/a.py#L12", + "PolicyEngine\\microcosm\\blob\\x\\a.py#L12", + "policyengine//MICROCOSM/blob/x/a.py#L3-L9", + "PolicyEngine/./microcosm/blob/x/a.py#L3", + "PolicyEngine/micro\tcosm/blob/x/a.py#L3", + "%50olicyEngine/%6Dicrocosm/blob/x/a.py#L3", + # Prose between the path and a later line fragment is read as part of + # the link: refusing the string is intended, as a dot segment's is. + "see github.com/PolicyEngine/microcosm/tree/main/docs and cps.py#L197 at a1b2c3d", + ): + assert line_links_of_this_repository({"note": written}) == [written], written + # An issue, a comment, a heading or a bare ``L`` is no line; another + # repository's path is not this one's; a ``#`` between the path and a line + # fragment starts the link's fragment first; and the path in another + # string links nothing in this one. + for written in ( + "PolicyEngine/microcosm#38 and x.py#L12 at a1b2c3d", + "https://github.com/PolicyEngine/microcosm/pull/939#issuecomment-1; x.py#L12 at a1b2c3d", + "https://github.com/PolicyEngine/microcosm/blob/main/README.md#readme", + "https://github.com/PolicyEngine/microcosm/blob/main/a.py#L", + "https://github.com/PolicyEngine/microcosm-dynamics/blob/a1b2c3d/x.py#L12", + "https://github.com/NotPolicyEngine/microcosm/blob/a1b2c3d/x.py#L12", + "github.com/PolicyEngine/microcosm/issues/1 (#2) and x.py#L12 at a1b2c3d", + "x.py#L12 at a1b2c3d; see github.com/PolicyEngine/microcosm/", + ): + assert line_links_of_this_repository({"note": written}) == [], written + document = {"a": "github.com/PolicyEngine/microcosm/", "b": "x.py#L12 at a1b2c3d"} + assert line_links_of_this_repository(document) == [] + + +#: A ref segment holding punctuation at which a reading starts a name, though +#: a URL's path holds it: ``feature+foo``, ``release!1``, ``v1,2;3=4``. +_PUNCTUATED_REF_SEGMENT = st.from_regex( + r"[A-Za-z0-9_+@!$,;=()-]{1,12}", fullmatch=True +).filter(lambda segment: "microcosm" not in segment.lower()) +#: A user and a host of a link to github.com: whitespace in a user ends a URL +#: in a reading, and a browser reads a host's escapes, full-width letters and +#: ideographic full stop. +_GITHUB_AUTHORITIES = ( + "github.com", + "user@github.com", + "a b@%67ithub.com", + "a\tb c@ｇithub.com", + "x y@github。com:443", +) +#: This repository's owner and name as GitHub routes them, and other +#: repositories' of the same shape. +_THIS_REPOSITORY_PATHS = ( + "PolicyEngine/microcosm", + "policyengine/MICROCOSM", + "%50olicyEngine/%6Dicrocosm", + "PolicyEngine//microcosm", + "PolicyEngine/./microcosm", +) +_OTHER_REPOSITORY_PATHS = ( + "PolicyEngine/microcosm-dynamics", + "PolicyEngine/microcosms", + "NotPolicyEngine/microcosm", + "PolicyEngine/micro", +) + + +@settings(max_examples=300, deadline=None) +@given( + enclosure=st.sampled_from(_URL_ENCLOSURES), + authority=st.sampled_from(_GITHUB_AUTHORITIES), + repository=st.sampled_from(_THIS_REPOSITORY_PATHS), + other=st.sampled_from(_OTHER_REPOSITORY_PATHS), + route=st.sampled_from(("blob", "tree", "blame")), + ref=st.lists(_PUNCTUATED_REF_SEGMENT, min_size=1, max_size=3).map("/".join), + path=st.sampled_from(_ROOTED_FILES), + query=st.sampled_from(("", "?plain=1", "?" + "a" * 30, "?a b=c")), + fragment=st.sampled_from(("#L{a}", "#L{a}-L{b}")), + sha=_SHA, + a=st.integers(1, 99_999), + b=st.integers(1, 99_999), +) +@example( + enclosure="[a](<{url}>)", + authority="github.com", + repository="PolicyEngine/microcosm", + other="PolicyEngine/microcosm-dynamics", + route="blob", + ref="feature+foo", + path="tools/build_us_puf_support_base.py", + query="", + fragment="#L{a}", + sha="a1b2c3d", + a=12, + b=12, +) +@example( + enclosure='Use `prefix="` here.\n\na', + authority="a b@%67ithub.com", + repository="%50olicyEngine/%6Dicrocosm", + other="NotPolicyEngine/microcosm", + route="tree", + ref="release!1/x", + path="tools/build_us_puf_support_base.py", + query="?" + "a" * 30, + fragment="#L{a}-L{b}", + sha="a1b2c3d", + a=3, + b=9, +) +def test_a_line_link_of_this_repository_is_refused_whatever_its_ref_or_authority_holds( + enclosure: str, + authority: str, + repository: str, + other: str, + route: str, + ref: str, + path: str, + query: str, + fragment: str, + sha: str, + a: int, + b: int, +) -> None: + # markdown-it-py 4.2, parse5 7.3 and bun 1.3 render each of these as a + # link to the file's lines on github.com, which routes the path, except a + # full-width host in a Markdown destination, which markdown-it writes as + # no URL bun reads; each is refused, pinned or not, and another + # repository's link of the same shape is not. + def written(repository: str) -> str: + line = fragment.format(a=a, b=b) + url = f"https://{authority}/{repository}/{route}/{ref}/{path}{query}{line}" + return f"Pinned at {sha}. " + enclosure.format(url=url) + + here = written(repository) + assert line_links_of_this_repository({"note": here}) == [here] + assert line_links_of_this_repository({"note": written(other)}) == [] + + def test_a_string_is_read_again_with_its_markup_escapes_decoded() -> None: def readings(text: str) -> list[str]: return list(citations._readable_strings(text)) @@ -2939,6 +3221,27 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: unrecognised_line_locators(text) dot_segment_line_citations(text) assert time.process_time() - started < 2, text[:24] + # Since round 12, in 2 s of their own: runs of what this repository's path, + # a line fragment, a run of slashes or ``.`` segments, a dot segment beside + # a ``%2f``, and the run of path characters before a name start with. + started = time.process_time() + for text in ( + "policyengine/microcosm" * 2_500 + "#L1", + "PolicyEngine/microcosm/#L1" * 2_000, + "#L1" * 20_000, + "/./" * 20_000 + "#L1", + "/" * 60_000 + "#L1", + "%2f..%2f" * 8_000, + "=..%2f" * 10_000, + "a=" * 30_000 + "x.py:1", + "=x.py:1" * 8_000, + "lines 1 of " + "a=" * 30_000, + ): + line_citations(text) + in_repository_line_citations(text) + dot_segment_line_citations(text) + line_links_of_this_repository(text) + assert time.process_time() - started < 2, text[:24] #: Pieces of text around source names and line locators. diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index 89e4bac66b..ae2814d28c 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -83,10 +83,15 @@ top-level source directory) and its inner ``..`` segments resolved (``tools/x/../y.py`` is ``tools/y.py``), it is a file here or one of the path forms ``_repository_path_tails`` lists. A line citation of it is -refused even when pinned. A bare file name cannot be told apart from another -repository's, so it must be pinned. - -A dot segment (``..`` as a whole path segment, either dot perhaps ``%2e``) +refused even when pinned. A path here can hold an ``=``, at which ``_SOURCE`` +starts a name (``href=x.py`` names ``x.py``), so the run of path characters +before a cited file is tried with it too: ``docs/.../period=2025-01.json +line 3`` cites a file here. Every path here is made of ``_PATH_HERE`` +characters, so a path here that prose cites is read whole. A bare file name +cannot be told apart from another repository's, so it must be pinned. + +A dot segment (``..`` as a whole path segment, either dot perhaps ``%2e``, +beside a slash, a backslash or a ``%2f``, which GitHub reads as a slash) removes the segment before it, and whitespace, a backslash or markup in that segment can make a renderer remove another one than a reading does: in ``[a]( )`` the destination @@ -94,10 +99,27 @@ Markdown writes the backslash as ``%5C``, so both link to line 12 of ``tools/x.py``. So every line citation in a string that holds a dot segment, in any of its readings' sources, is refused, pinned or not -(``dot_segment_line_citations``). Without one, whitespace or a backslash that -a reading takes otherwise than a renderer stays in the segment the renderer -follows (``%20``, ``%5C``), and no file under ``_SOURCE_ROOTS`` has either in -its path, so that link names no source file here. +(``dot_segment_line_citations``). + +A reading can lose a link of this repository without a dot segment too: a +ref such as ``feature+foo`` holds a character no path here does, so the +citation a reading finds starts after it, and where a code span's ``="`` has +taken an attribute's opening quote, whitespace before the host +(``href="https://a b@%67ithub.com/..."``) ends the URL. So a string is also +refused where any of its sources holds this repository's path on GitHub +(``PolicyEngine/microcosm/``) with a line fragment (``#L12``) at the first +``#`` after it, whatever file the link names and whether or not a reading +finds a citation in it (``line_links_of_this_repository``). The source is +read as GitHub reads a path: its escapes of unreserved characters and of +``/`` decoded, a backslash read as ``/``, a run of slashes as one, ``.`` +segments removed, and the owner and the name in any case. A renderer takes a +source to the URL it follows by the decodings ``_sources`` makes, the steps +this rule reads through, percent-encoding characters that neither the path +nor a ``#`` holds, and dot segments, which the rule before refuses; and the +first ``#`` after the path in a URL starts its fragment. So a link that +names this repository's path on GitHub and a line is refused however a +reading takes it. A relative link names no repository, and a redirect (a +short link, a renamed repository) names another path: neither is read. The recognised spellings are the ones ``_LINE_CITATIONS`` documents. A source name followed by a line locator in no recognised spelling, with no ``.``, @@ -123,7 +145,7 @@ from dataclasses import dataclass, field from functools import cache, lru_cache from html.entities import html5 -from itertools import accumulate +from itertools import accumulate, pairwise from pathlib import Path, PurePosixPath from urllib.parse import unquote_to_bytes @@ -192,8 +214,14 @@ + _QUOTES + _NUMBERS ) +#: A run of path characters up to an ``=``, which a reverse locator's file may +#: follow, as a forward locator's may: ``lines 3 of docs/period=2025-01.json`` +#: cites ``2025-01.json``. From the run's last ``=`` back, each is tried once, +#: so a run is read in time linear in its length. +_UP_TO_AN_EQUALS = r"(?:[A-Za-z0-9_./=-]*=)?" _REVERSE_LOCATOR = re.compile( rf"\b(?:{_LABEL}|(?=L\d)){_QUOTES}{_NUMBERS}{_QUOTES}\s++(?i:of|in)\s++{_QUOTES}" + + _UP_TO_AN_EQUALS + _SOURCE ) _LINE_CITATIONS = (_FORWARD_LOCATOR, _REVERSE_LOCATOR) @@ -311,10 +339,13 @@ _STARTS_A_URL = re.compile(_URL_START) #: A dot segment: ``..`` as a whole path segment beside a slash or a #: backslash (``x/../y``, ``../y``, ``x\\..``), either ``.`` perhaps written -#: ``%2e`` (``.%2E``, ``%2e%2e``), as the URL Standard reads one. +#: ``%2e`` (``.%2E``, ``%2e%2e``), as the URL Standard reads one, or beside a +#: ``%2f``, which GitHub reads as a slash before it resolves the path +#: (``x%2f..%2fy``). _DOT_SEGMENT = re.compile( - r"(?<=[/\\])(?:\.|%2[eE]){2}(?![^/\\?#\s\"'`‘’“”<>()\[\]])" - r"|(?()\[\]=])(?:\.|%2[eE]){2}(?=[/\\])" + r"(?:(?<=[/\\])|(?<=%2[fF]))(?:\.|%2[eE]){2}" + r"(?:(?![^/\\?#\s\"'`‘’“”<>()\[\]])|(?=%2[fF]))" + r"|(?()\[\]=])(?:\.|%2[eE]){2}(?=[/\\]|%2[fF])" ) #: What the URL Standard strips from both ends of a URL: C0 controls and #: spaces. @@ -389,6 +420,10 @@ _CLAUSE_BREAK = re.compile(r";|(?<=[.!?])\s+(?=[A-Z(\[`])") #: Top-level directories whose files are this repository's sources. _SOURCE_ROOTS = ("packages", "tools", "experiments", "test_support", "docs") +#: The characters every path under ``_SOURCE_ROOTS`` is made of: ``_SOURCE``'s, +#: and ``=`` (``docs/.../period=2025-01.json``), at which ``_SOURCE`` starts a +#: name so that ``href=x.py`` names ``x.py``. +_PATH_HERE = frozenset(string.ascii_letters + string.digits + "_./=-") _SKIPPED_DIRECTORIES = {"__pycache__", ".venv", "node_modules", ".git"} #: A prefix naming this repository before a path in it: a GitHub URL of a #: file here, up to its ref (``url``), or ``microcosm/`` (after any ``./`` or @@ -402,6 +437,15 @@ + "|".join(_SOURCE_ROOTS) + r")/)" ) +#: This repository's path on GitHub: its owner and name, in any case, as +#: GitHub routes them, the owner not the end of a longer name. +_REPOSITORY_ON_GITHUB = re.compile(r"(? str: @@ -803,6 +850,22 @@ def _citation_matches(text: str) -> list[re.Match[str]]: ) +def _path_run(text: str, end: int) -> str: + """The run of ``_PATH_HERE`` characters in ``text`` that ends at ``end``. + + A forward locator holds a character outside ``_PATH_HERE`` after its file + and a reverse one holds whitespace before its file's run, so the runs + before a string's citations overlap only where a reverse locator's file + runs on into the next citation's, and they are read in time linear in the + string's length. + """ + + start = end + while start and text[start - 1] in _PATH_HERE: + start -= 1 + return text[start:end] + + def line_citations(value: object) -> list[LineCitation]: """Each line citation of a source in a payload, in string order. @@ -812,7 +875,13 @@ def line_citations(value: object) -> list[LineCitation]: """ return [ - LineCitation(match["file"], text, match.start(), match.end()) + LineCitation( + match["file"], + text, + match.start(), + match.end(), + _path_run(text, match.start("file")) + match["file"], + ) for text in _readable_strings(value) for match in _citation_matches(text) ] @@ -1103,12 +1172,19 @@ def resolves_in_repository(file: str, root: Path = REPOSITORY_ROOT) -> bool: def in_repository_line_citations( document: object, root: Path = REPOSITORY_ROOT ) -> list[LineCitation]: - """Line citations of this repository's files, pinned or not.""" + """Line citations of this repository's files, pinned or not. + + A citation's ``path`` is tried as well as its ``file``: a path here can + hold an ``=``, where ``_SOURCE`` starts a name + (``docs/.../period=2025-01.json line 3`` cites ``2025-01.json``). + """ return [ citation for citation in line_citations(document) if resolves_in_repository(citation.file, root) + or citation.path != citation.file + and resolves_in_repository(citation.path, root) ] @@ -1132,6 +1208,46 @@ def dot_segment_line_citations(value: object) -> list[LineCitation]: ] +def _links_a_line_here(source: str) -> bool: + """Whether a source holds this repository's path on GitHub with a line + fragment at the first ``#`` after it. + + The source is read as GitHub reads a URL's path, with no URL needed to + start before it: its ``_DECODED`` escapes decoded, a backslash read as + ``/``, a run of slashes as one and its ``.`` segments removed. + """ + + path = _SLASH_RUN.sub("/", _decode(source).replace("\\", "/")) + pieces = _SINGLE_DOT_SEGMENTS.sub("", path).split("#") + return any( + _LINE_FRAGMENT.match(fragment) and _REPOSITORY_ON_GITHUB.search(before) + for before, fragment in pairwise(pieces) + ) + + +def line_links_of_this_repository(value: object) -> list[str]: + """Strings that link a line of this repository on GitHub, however read. + + A link a renderer follows to a line of a file here names this + repository's path (``PolicyEngine/microcosm/``), and its fragment + (``L12``) starts at the first ``#`` after it. A reading can lose the + link: in ``[a]()`` the citation it finds starts after the + ``+`` of the ref, and in ``href="https://a b@%67ithub.com/PolicyEngine/ + microcosm/..."`` whose opening quote a code span's ``="`` took, the URL + ends at the space before the host. So a string any of whose ``_sources`` + holds the path with a line fragment after it (``_links_a_line_here``) is + refused, whatever file it names and whether or not a reading finds a + citation in it. + """ + + return [ + text + for text in nested_strings(value) + if any(map(_links_a_line_here, _sources(text))) + ] + + def dotted_microcosm_symbols(value: object) -> set[str]: """Every dotted ``microcosm..*`` name in a payload's strings.""" From 4dc0734e1b7b382b7a2a2cb16ee471cf83f6de5b Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 07:08:46 -0400 Subject: [PATCH 18/21] Refuse a line link here whose path a dot segment forms A dot segment can form this repository's GitHub path in a link to a file whose name no reading takes for a source: `https://github.com/PolicyEngine/x/../microcosm/blob//LICENSE#L3`. markdown-it-py 4.2, parse5 7.3 and bun 1.3 render it as a link to line 3 of LICENSE here; so do `x/%2E%2e/`, `//x/../../`, `a%2fb/../`, `x\..\` and `..` in its place, and the same with this tree's Dockerfile and SHA256SUMS. No reading finds a citation, so the dot-segment rule never sees one, and the repository-link rule read the path without resolving dot segments. Of 3,960 inputs (4 files x 22 detours x 5 enclosures x 3 authorities x 3 queries), 1,920 render as a link to a line here, and at 8a62c938 1,242 of those passed every rule. The repository-link rule now reads each source as a browser and then GitHub read a URL's path: a backslash as `/` and the dot segments before each `?` or `#` resolved, either dot perhaps `%2e` and an empty segment counted; then escapes decoded and a run of slashes read as one. This single reading replaces the earlier one, whose refusals it covers. The earlier reading also refused links that a `..` sends out of this repository (`PolicyEngine/microcosm/../other/...`); this one does not. A browser reads `%2f` as no separator. GitHub decodes none in the owner or the name (404 for `x%2f..%2fmicrocosm`, measured with curl --path-as-is), so decoding one after resolving can only add a refusal. GitHub also returns 404 for `%252e%252e` and `%5c` there, and 200 for `x/%2e%2e/` and `x/.%2e/`, which a browser has already resolved. All 1,920 here-links in the matrix are refused now, and no other-repository detour is. Rendered by the same oracles, the property's space (11,200 inputs, expanded below) and 2,000 random draws agree: every input naming this repository is refused, and none naming another. Tests: `test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its_path` (8 inputs refused; 5 that link no line here, such as `x/../../microcosm` and `x/.../microcosm`, not refused). The property draws three more spellings of this repository's path through a dot segment, two detours that leave it, the three files, and a query of `..` segments, with two explicit examples. The timing test gains long runs of dot segments, escaped dots, backslashes and segments after a `?`. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 104 ++++++++++++++++-- .../microcosm_build/spec_source_citations.py | 62 ++++++++--- 3 files changed, 142 insertions(+), 26 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index 2f2cc4419f..21ebfb8781 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A destination in angle brackets is found after the spaces, tabs and one line break CommonMark allows before its `<`. Where a string holds a `?` and a `#`, its URLs are also read through whitespace in their query, which GitHub ignores (`href=".../x.py?a b#L12"`). A citation any reading finds is checked. Every line citation in a string that holds a dot segment (`..` as a whole path segment, either dot perhaps `%2e`) is refused, pinned or not: whitespace, a backslash or markup in the segment it removes can make a renderer remove another one than any reading does, as in `[a]( )` or `[a](https://github.com/.../tools/a\b/../x.py#L12)`, where Markdown writes the backslash `%5C`. No resource cites a line in such a string. GitHub reads `%2f` as `/` before it resolves a path, so a `..` beside one is a dot segment too (`tools/a b%2f..%2fx.py`). A string is also refused where any of its readings' sources holds this repository's GitHub path, `PolicyEngine/microcosm/`, with a line fragment (`#L12`) at the first `#` after it, whatever file the link names and whether or not a reading finds the citation. The path is read as GitHub reads one: escapes decoded, a backslash as `/`, a run of slashes as one, `.` segments removed and the owner and name in any case. So a ref holding punctuation (`blob/feature+foo/tools/x.py#L12`), or whitespace before the host where markup has taken an attribute's opening quote (`href="https://a b@%67ithub.com/..."`), no longer hides a link of this repository. No resource holds such a link. A path here can hold an `=` (`docs/.../period=2025-01.json`), so the run of path characters before a cited name is tried as an in-repository path too, and a reverse locator reads a name after an `=` (`lines 3 of a/b=c.py`). An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A destination in angle brackets is found after the spaces, tabs and one line break CommonMark allows before its `<`. Where a string holds a `?` and a `#`, its URLs are also read through whitespace in their query, which GitHub ignores (`href=".../x.py?a b#L12"`). A citation any reading finds is checked. Every line citation in a string that holds a dot segment (`..` as a whole path segment, either dot perhaps `%2e`) is refused, pinned or not: whitespace, a backslash or markup in the segment it removes can make a renderer remove another one than any reading does, as in `[a]( )` or `[a](https://github.com/.../tools/a\b/../x.py#L12)`, where Markdown writes the backslash `%5C`. No resource cites a line in such a string. GitHub reads `%2f` as `/` before it resolves a path, so a `..` beside one is a dot segment too (`tools/a b%2f..%2fx.py`). A string is also refused where any of its readings' sources holds this repository's GitHub path, `PolicyEngine/microcosm/`, with a line fragment (`#L12`) at the first `#` after it, whatever file the link names and whether or not a reading finds the citation. The path is read as a browser and then GitHub read one: a backslash as `/` and dot segments resolved first (either dot perhaps `%2e`, an empty segment counted), then escapes decoded, a run of slashes read as one and the owner and name matched in any case. So a dot segment that forms the path (`PolicyEngine/x/../microcosm/blob//LICENSE#L3`) is refused even where the file's name is not one a reading cites. So a ref holding punctuation (`blob/feature+foo/tools/x.py#L12`), or whitespace before the host where markup has taken an attribute's opening quote (`href="https://a b@%67ithub.com/..."`), no longer hides a link of this repository. No resource holds such a link. A path here can hold an `=` (`docs/.../period=2025-01.json`), so the run of path characters before a cited name is tried as an in-repository path too, and a reverse locator reads a name after an `=` (`lines 3 of a/b=c.py`). An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index e28dc1bf3c..708c4b18db 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -25,8 +25,9 @@ unknown; - no resource links a line of this repository by its GitHub path, however a reading takes the link (``line_links_of_this_repository``): no string's - source holds ``PolicyEngine/microcosm/``, read as GitHub reads a path, - with ``#L`` at the first ``#`` after it; + source holds ``PolicyEngine/microcosm/``, read as a browser and then + GitHub read a path (dot segments resolved first, then escapes decoded and + slash runs read as one), with ``#L`` at the first ``#`` after it; - no resource cites a line of any other source unless the citation binds to an inline commit (the nearest one in its own clause, else the nearest one before it in its string; ``citing_commit``), or the resource pins a path @@ -2782,6 +2783,45 @@ def test_a_line_link_of_this_repository_is_refused_by_its_path_alone() -> None: assert line_links_of_this_repository(document) == [] +def test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its_path() -> ( + None +): + sha = "8a62c9380858d16d94f58146c23cb8daaf8bb70c" + dockerfile = ( + "docs/evidence/us-release-reimputation-1191/scripts/platform/Dockerfile" + ) + sums = "packages/microcosm-build/tests/fixtures/staging/v1/SHA256SUMS" + code_span = 'Use `prefix="` before the value.\n\n' + # markdown-it-py 4.2 and parse5 7.3 render each of these, and bun 1.3's + # ``URL`` resolves each, to a link to line 3 of a file here, and github.com + # serves the file; at ``8a62c938`` no rule refused them. A browser + # resolves the dot segment before GitHub reads the path, and no reading + # takes these files' names for a source, so no citation is found. + for written in ( + f"[a](https://github.com/PolicyEngine/x/../microcosm/blob/{sha}/LICENSE#L3)", + f'a', + f"[a]()", + f'a', + f'a', + f'a', + f"[a](https://github.com/%50olicyEngine/x/../%6Dicrocosm/blob/{sha}/LICENSE?a/../../../../..#L3)", + f'{code_span}a', + ): + assert line_citations(written) == [], written + assert line_links_of_this_repository({"note": written}) == [written], written + # These resolve to another path: ``/microcosm/blob/...``; ``%2f`` in the + # owner or the name, which a browser does not resolve and GitHub serves + # no file for; another repository; and ``...``, which is no dot segment. + for written in ( + f"[a](https://github.com/PolicyEngine/x/../../microcosm/blob/{sha}/LICENSE#L3)", + f'a', + f'a', + f"[a](https://github.com/PolicyEngine/microcosm-x/../microcosm-dynamics/blob/{sha}/LICENSE#L3)", + f"[a](https://github.com/PolicyEngine/x/.../microcosm/blob/{sha}/LICENSE#L3)", + ): + assert line_links_of_this_repository({"note": written}) == [], written + + #: A ref segment holding punctuation at which a reading starts a name, though #: a URL's path holds it: ``feature+foo``, ``release!1``, ``v1,2;3=4``. _PUNCTUATED_REF_SEGMENT = st.from_regex( @@ -2797,20 +2837,33 @@ def test_a_line_link_of_this_repository_is_refused_by_its_path_alone() -> None: "a\tb c@ｇithub.com", "x y@github。com:443", ) -#: This repository's owner and name as GitHub routes them, and other -#: repositories' of the same shape. +#: This repository's owner and name as a browser and then GitHub route them, +#: through dot segments too, and other repositories' of the same shape, two +#: of them through a dot segment that leaves another path or none at all. _THIS_REPOSITORY_PATHS = ( "PolicyEngine/microcosm", "policyengine/MICROCOSM", "%50olicyEngine/%6Dicrocosm", "PolicyEngine//microcosm", "PolicyEngine/./microcosm", + "PolicyEngine/x/../microcosm", + "PolicyEngine//x/%2e%2E/../microcosm", + "PolicyEngine/a%2fb/../microcosm", ) _OTHER_REPOSITORY_PATHS = ( "PolicyEngine/microcosm-dynamics", "PolicyEngine/microcosms", "NotPolicyEngine/microcosm", "PolicyEngine/micro", + "PolicyEngine/x/../../microcosm", + "PolicyEngine/x%2f..%2fmicrocosm", +) +#: Files here whose names no reading takes for a source, so no rule but the +#: link's own refuses a line link of one. +_FILES_NO_READING_CITES = ( + "LICENSE", + "docs/evidence/us-release-reimputation-1191/scripts/platform/Dockerfile", + "packages/microcosm-build/tests/fixtures/staging/v1/SHA256SUMS", ) @@ -2822,13 +2875,43 @@ def test_a_line_link_of_this_repository_is_refused_by_its_path_alone() -> None: other=st.sampled_from(_OTHER_REPOSITORY_PATHS), route=st.sampled_from(("blob", "tree", "blame")), ref=st.lists(_PUNCTUATED_REF_SEGMENT, min_size=1, max_size=3).map("/".join), - path=st.sampled_from(_ROOTED_FILES), - query=st.sampled_from(("", "?plain=1", "?" + "a" * 30, "?a b=c")), + path=st.sampled_from(_ROOTED_FILES + _FILES_NO_READING_CITES), + query=st.sampled_from( + ("", "?plain=1", "?" + "a" * 30, "?a b=c", "?a/../../../../..") + ), fragment=st.sampled_from(("#L{a}", "#L{a}-L{b}")), sha=_SHA, a=st.integers(1, 99_999), b=st.integers(1, 99_999), ) +@example( + enclosure="[a](<{url}>)", + authority="github.com", + repository="PolicyEngine//x/%2e%2E/../microcosm", + other="PolicyEngine/x/../../microcosm", + route="blob", + ref="main", + path="LICENSE", + query="?a/../../../../..", + fragment="#L{a}", + sha="a1b2c3d", + a=3, + b=3, +) +@example( + enclosure='a', + authority="user@github.com", + repository="PolicyEngine/a%2fb/../microcosm", + other="PolicyEngine/x%2f..%2fmicrocosm", + route="blame", + ref="a1b2c3d", + path="packages/microcosm-build/tests/fixtures/staging/v1/SHA256SUMS", + query="", + fragment="#L{a}-L{b}", + sha="a1b2c3d", + a=1, + b=4, +) @example( enclosure="[a](<{url}>)", authority="github.com", @@ -3223,7 +3306,9 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: assert time.process_time() - started < 2, text[:24] # Since round 12, in 2 s of their own: runs of what this repository's path, # a line fragment, a run of slashes or ``.`` segments, a dot segment beside - # a ``%2f``, and the run of path characters before a name start with. + # a ``%2f``, and the run of path characters before a name start with; and + # dot segments a browser resolves, after a ``?`` too, escaped or beside a + # backslash, which pop 10,000 segments back to the owner. started = time.process_time() for text in ( "policyengine/microcosm" * 2_500 + "#L1", @@ -3236,6 +3321,11 @@ def test_a_long_string_is_scanned_in_linear_time() -> None: "a=" * 30_000 + "x.py:1", "=x.py:1" * 8_000, "lines 1 of " + "a=" * 30_000, + "PolicyEngine/" + "x/" * 10_000 + "../" * 10_000 + "microcosm/#L1", + "/.." * 20_000 + "#L1", + "?/.." * 15_000 + "#L1", + "%2e%2E/" * 9_000 + "#L1", + "\\.." * 20_000 + "#L1", ): line_citations(text) in_repository_line_citations(text) diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index ae2814d28c..a2ee33e64b 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -110,16 +110,24 @@ (``PolicyEngine/microcosm/``) with a line fragment (``#L12``) at the first ``#`` after it, whatever file the link names and whether or not a reading finds a citation in it (``line_links_of_this_repository``). The source is -read as GitHub reads a path: its escapes of unreserved characters and of -``/`` decoded, a backslash read as ``/``, a run of slashes as one, ``.`` -segments removed, and the owner and the name in any case. A renderer takes a -source to the URL it follows by the decodings ``_sources`` makes, the steps -this rule reads through, percent-encoding characters that neither the path -nor a ``#`` holds, and dot segments, which the rule before refuses; and the -first ``#`` after the path in a URL starts its fragment. So a link that -names this repository's path on GitHub and a line is refused however a -reading takes it. A relative link names no repository, and a redirect (a -short link, a renamed repository) names another path: neither is read. +read as a browser and then GitHub take a URL's path: a backslash read as +``/`` and the dot segments before each ``?`` or ``#`` resolved, either dot +perhaps ``%2e`` and an empty segment counted; only then the escapes of +unreserved characters and of ``/`` decoded and a run of slashes read as one; +and the owner and the name matched in any case. A dot segment can form the +path where no reading finds a citation, as in a link to ``LICENSE``, whose +name no reading takes for a source: +``https://github.com/PolicyEngine/x/../microcosm/blob//LICENSE#L3``. A +renderer takes a source to the URL it follows by the decodings ``_sources`` +makes, a browser's steps (a backslash as ``/``, dot segments resolved, and +characters that neither the path nor a ``#`` holds percent-encoded) and +GitHub's (unreserved escapes decoded, a run of slashes as one, the owner and +the name in any case; it decodes no ``%2f`` in them, so decoding one here +can only add a refusal); and the first ``#`` after the path in a URL starts +its fragment. So a link that names this +repository's path on GitHub and a line is refused however a reading takes +it. A relative link names no repository, and a redirect (a short link, a +renamed repository) names another path: neither is read. The recognised spellings are the ones ``_LINE_CITATIONS`` documents. A source name followed by a line locator in no recognised spelling, with no ``.``, @@ -444,8 +452,10 @@ _LINE_FRAGMENT = re.compile(r"L\d") #: A run of slashes, which GitHub reads as one. _SLASH_RUN = re.compile(r"/{2,}") -#: Single-dot segments after a slash, which the URL Standard removes. -_SINGLE_DOT_SEGMENTS = re.compile(r"(?<=/)(?:\./)+") +#: An escaped dot, which the URL Standard reads as a dot in a dot segment. +_ESCAPED_DOT = re.compile(r"%2[eE]") +#: What ends a URL's path: a browser resolves its dot segments before it. +_PATH_END = re.compile(r"([?#])") #: A dotted Microcosm symbol named in prose, such as #: ``microcosm.graph.executor._structural_columns``. DOTTED_MICROCOSM_SYMBOL = re.compile( @@ -1208,20 +1218,36 @@ def dot_segment_line_citations(value: object) -> list[LineCitation]: ] +def _browsed(source: str) -> str: + """A source with its dot segments resolved as a browser resolves a URL's. + + A backslash is read as ``/``, either dot may be ``%2e``, and an empty + segment counts: ``PolicyEngine//x/../../microcosm`` is + ``PolicyEngine/microcosm``. A ``?`` or a ``#`` ends a path, so no dot + segment after one removes a segment before it. + """ + + pieces = _PATH_END.split(_ESCAPED_DOT.sub(".", source.replace("\\", "/"))) + return "".join(map(_remove_dot_segments, pieces)) + + def _links_a_line_here(source: str) -> bool: """Whether a source holds this repository's path on GitHub with a line fragment at the first ``#`` after it. - The source is read as GitHub reads a URL's path, with no URL needed to - start before it: its ``_DECODED`` escapes decoded, a backslash read as - ``/``, a run of slashes as one and its ``.`` segments removed. + The source is read as a browser and then GitHub take a URL, with no URL + needed to start before the path: its dot segments resolved first + (``_browsed``), which can form the owner and the name + (``PolicyEngine/x/../microcosm/``), and only then its ``_DECODED`` + escapes decoded and its runs of slashes read as one. A browser reads + ``%2f`` as no separator, and GitHub decodes none in the owner or the + name, so decoding one here can only add a refusal. """ - path = _SLASH_RUN.sub("/", _decode(source).replace("\\", "/")) - pieces = _SINGLE_DOT_SEGMENTS.sub("", path).split("#") + path = _SLASH_RUN.sub("/", _decode(_browsed(source))) return any( _LINE_FRAGMENT.match(fragment) and _REPOSITORY_ON_GITHUB.search(before) - for before, fragment in pairwise(pieces) + for before, fragment in pairwise(path.split("#")) ) From 5578c82d48388348dd179bb2a5808bf97a387981 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 07:12:33 -0400 Subject: [PATCH 19/21] Test the round-12 review's dot-segment link with a source file The round-12 review (GPT-6.1 Sol, at 8a62c938) found the gap that 4dc0734e closes, with its own shape: a code span's `="` takes the attribute's opening quote, so every reading's URL ends at the space in `https://a b@%67ithub.com/PolicyEngine/x/../microcosm/blob//tools/ci_test_plan.py?<30 letters>=1#L12`. markdown-it-py 4.2, parse5 7.3 and bun 1.3 render it as a link to line 12 of `tools/ci_test_plan.py`. No rule at 8a62c938 refused it, and rule 7 refuses it at 4dc0734e. The dot-segment test now holds it, as the review asked. Co-Authored-By: Claude Opus 5.5 --- .../engine_free/shared/test_spec_source_citations.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 708c4b18db..0873a6fd2b 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -2793,10 +2793,12 @@ def test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its sums = "packages/microcosm-build/tests/fixtures/staging/v1/SHA256SUMS" code_span = 'Use `prefix="` before the value.\n\n' # markdown-it-py 4.2 and parse5 7.3 render each of these, and bun 1.3's - # ``URL`` resolves each, to a link to line 3 of a file here, and github.com + # ``URL`` resolves each, to a link to a line of a file here, and github.com # serves the file; at ``8a62c938`` no rule refused them. A browser - # resolves the dot segment before GitHub reads the path, and no reading - # takes these files' names for a source, so no citation is found. + # resolves the dot segment before GitHub reads the path, and no citation + # is found: no reading takes these files' names for a source, and in the + # last, the round-12 review's, every reading's URL ends at the space in + # its user. for written in ( f"[a](https://github.com/PolicyEngine/x/../microcosm/blob/{sha}/LICENSE#L3)", f'a', @@ -2806,6 +2808,7 @@ def test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its f'a', f"[a](https://github.com/%50olicyEngine/x/../%6Dicrocosm/blob/{sha}/LICENSE?a/../../../../..#L3)", f'{code_span}a', + f'{code_span}a', ): assert line_citations(written) == [], written assert line_links_of_this_repository({"note": written}) == [written], written From a738da7da49379271e18d875aba7504769c24c86 Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 07:38:12 -0400 Subject: [PATCH 20/21] Read a backslash in a line link here both as a browser and as Markdown does A fuzz of rule 7 against the renderers found links it missed at 4dc0734e, all Markdown destinations holding a backslash. markdown-it-py 4.2 percent-encodes a backslash (`%5C`), so in `[a](https://github.com/PolicyEngine/x\y/../microcosm/blob//LICENSE#L3)` the `..` removes the segment `x%5Cy`, and the link is to line 3 of LICENSE here; GitHub serves it. In an HTML `href` a browser reads the same backslash as `/`, and the URL links `PolicyEngine/x/microcosm`. Rule 7 read every backslash as `/`, so it missed the Markdown link. Of 40,000 generated inputs (detours through many spellings of `.`, `..`, `/`, `\`, `?` and `#`, nine enclosures, six authorities), 5,590 render to a line link here, and rule 7 at 4dc0734e missed 58, each a Markdown link with a backslash. Rule 7 now resolves each source's dot segments twice: with a backslash read as `/`, as a browser reads it in HTML, and with it kept, as a Markdown renderer escapes it. GitHub reads `%5C` as no separator (`PolicyEngine/x%5c..%5cmicrocosm` returns 404). All 5,590 are refused now, and the earlier fuzz's 385 too. The round-13 review (GPT-6.1 Sol, at 5578c82d) found the same class independently, with the inline shape. The dot-segment test adds four Markdown links: an inline link, a pointy destination, a reference definition and an autolink, each refused by no rule at 4dc0734e. Co-Authored-By: Claude Opus 5.5 --- .../spec-source-citations-us-uk.fixed.md | 2 +- .../shared/test_spec_source_citations.py | 9 +++ .../microcosm_build/spec_source_citations.py | 59 ++++++++++++------- 3 files changed, 47 insertions(+), 23 deletions(-) diff --git a/changelog.d/spec-source-citations-us-uk.fixed.md b/changelog.d/spec-source-citations-us-uk.fixed.md index 21ebfb8781..b73dd495e7 100644 --- a/changelog.d/spec-source-citations-us-uk.fixed.md +++ b/changelog.d/spec-source-citations-us-uk.fixed.md @@ -1 +1 @@ -Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A destination in angle brackets is found after the spaces, tabs and one line break CommonMark allows before its `<`. Where a string holds a `?` and a `#`, its URLs are also read through whitespace in their query, which GitHub ignores (`href=".../x.py?a b#L12"`). A citation any reading finds is checked. Every line citation in a string that holds a dot segment (`..` as a whole path segment, either dot perhaps `%2e`) is refused, pinned or not: whitespace, a backslash or markup in the segment it removes can make a renderer remove another one than any reading does, as in `[a]( )` or `[a](https://github.com/.../tools/a\b/../x.py#L12)`, where Markdown writes the backslash `%5C`. No resource cites a line in such a string. GitHub reads `%2f` as `/` before it resolves a path, so a `..` beside one is a dot segment too (`tools/a b%2f..%2fx.py`). A string is also refused where any of its readings' sources holds this repository's GitHub path, `PolicyEngine/microcosm/`, with a line fragment (`#L12`) at the first `#` after it, whatever file the link names and whether or not a reading finds the citation. The path is read as a browser and then GitHub read one: a backslash as `/` and dot segments resolved first (either dot perhaps `%2e`, an empty segment counted), then escapes decoded, a run of slashes read as one and the owner and name matched in any case. So a dot segment that forms the path (`PolicyEngine/x/../microcosm/blob//LICENSE#L3`) is refused even where the file's name is not one a reading cites. So a ref holding punctuation (`blob/feature+foo/tools/x.py#L12`), or whitespace before the host where markup has taken an attribute's opening quote (`href="https://a b@%67ithub.com/..."`), no longer hides a link of this repository. No resource holds such a link. A path here can hold an `=` (`docs/.../period=2025-01.json`), so the run of path characters before a cited name is tried as an in-repository path too, and a reverse locator reads a name after an `=` (`lines 3 of a/b=c.py`). An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. +Cite source code by symbol, or by line at a named commit, in the US and UK spec packages. A new engine-free guard (`test_spec_source_citations.py`, validators in `test_support/microcosm_build/spec_source_citations.py`) scans every resource in every country package. It refuses a line citation of a file in this repository, pinned or not, including one written as a `PolicyEngine/microcosm` GitHub URL at any ref (`main`, `refs/heads/main`, `feature/x`) or a `microcosm/tools/...` path. It refuses a line citation of any other source unless the citation binds to an inline commit (`commit `, `at `, `@`, or a GitHub `/blob/`, `/tree/`, `/raw/`, `/blame/` or `/commit/` URL): the nearest one in the citation's own clause, else the nearest one before it in its string, unless the citation's clause names a moving ref (`on main`, ``on `main` ``, or a `/blob/main/` or `raw.githubusercontent.com///main/` URL). A resource's `path`/`commit` mapping pins it too, but only when the cited path ends the pinned path. A commit-like token in no recognised spelling beside a citation is refused as a misspelled pin. So is a line locator after a source name in no recognised spelling (`x.py some_function, lines 12-19`). A truncated hash (`e2de2b41…`) or a hex directory (`at a1b2c3d/foo.py`) pins nothing, and a quoted locator (`x.py:"12-19"`) is a line citation like any other. Each URL is read the way a browser and then GitHub read it first. Its authority is parsed as the WHATWG URL Standard parses it: `\` read as `/`, any number of slashes after `https:` or in a `//host/...` reference, a user and a port dropped, and the host percent-decoded, mapped to ASCII as IDNA maps it and lower-cased, so `%67ithub.com`, `ｇithub.com` and `user@github.com:443` are `github.com`. Then `%` escapes of unreserved characters and `/` in its path are decoded, `.` and `..` segments resolved, and its query dropped, so `%74ools/x.py` and `x.py?plain=1#L12` are line citations of `tools/x.py` and `x.py`. A URL runs to whitespace, so a quote or a bracket in its path or query (`x.py?q='a'#L12`) is part of it. Markdown and prose end a URL at a quote, a parenthesis or an angle bracket, so a string is read that way too where it differs: in `[https://a/?x](https://github.com/...#L12)` the label's query does not hide the link. A string is also read with its Markdown and HTML escapes decoded (`x\.py`, `#L12`): as Markdown decodes them and as HTML does, which leaves a backslash as written (`https:\/github.com` names github.com), each as text and as a URL holds what it decodes to, with references decoded as the HTML standard decodes them. Each reading is also read with its ASCII tabs and line breaks deleted, as the URL Standard reads a URL in an HTML attribute. A quoted attribute value or a Markdown destination in angle brackets that starts a URL is read as the browser follows it, its escapes decoded before its ends are stripped and its spaces percent-encoded, and a URL is read through a Unicode space such as U+00A0, which the URL Standard percent-encodes. A destination in angle brackets is found after the spaces, tabs and one line break CommonMark allows before its `<`. Where a string holds a `?` and a `#`, its URLs are also read through whitespace in their query, which GitHub ignores (`href=".../x.py?a b#L12"`). A citation any reading finds is checked. Every line citation in a string that holds a dot segment (`..` as a whole path segment, either dot perhaps `%2e`) is refused, pinned or not: whitespace, a backslash or markup in the segment it removes can make a renderer remove another one than any reading does, as in `[a]( )` or `[a](https://github.com/.../tools/a\b/../x.py#L12)`, where Markdown writes the backslash `%5C`. No resource cites a line in such a string. GitHub reads `%2f` as `/` before it resolves a path, so a `..` beside one is a dot segment too (`tools/a b%2f..%2fx.py`). A string is also refused where any of its readings' sources holds this repository's GitHub path, `PolicyEngine/microcosm/`, with a line fragment (`#L12`) at the first `#` after it, whatever file the link names and whether or not a reading finds the citation. The path is read as a browser and then GitHub read one: a backslash as `/`, as in an HTML attribute, and again as no separator, as a Markdown renderer escapes it (`%5C`); dot segments resolved first (either dot perhaps `%2e`, an empty segment counted); then escapes decoded, a run of slashes read as one and the owner and name matched in any case. So a dot segment that forms the path (`PolicyEngine/x/../microcosm/blob//LICENSE#L3`) is refused even where the file's name is not one a reading cites. So a ref holding punctuation (`blob/feature+foo/tools/x.py#L12`), or whitespace before the host where markup has taken an attribute's opening quote (`href="https://a b@%67ithub.com/..."`), no longer hides a link of this repository. No resource holds such a link. A path here can hold an `=` (`docs/.../period=2025-01.json`), so the run of path characters before a cited name is tried as an in-repository path too, and a reverse locator reads a name after an `=` (`lines 3 of a/b=c.py`). An in-repository path's inner `..` segments are resolved (`tools/x/../y.py` is `tools/y.py`). In `us/ecps_parity_known_gaps.json`, the `asec_pool.py`, `tools/build_us_puf_support_base.py` and `experiments/build_j_recert/*` line citations now name `microcosm.build.us_runtime.asec_pool` symbols, the tool's functions, the `--asec-h5` arguments and `base_j.summary.json` keys. Where #720's Census person-column restore had made a sentence false, the sentence now says which columns the restore leaves out, and tests check those columns against `ASEC_CENSUS_PERSON_COLUMN_NAMES`. The us-data citations in `us/source_stages.json` and `us/target_parity_manifest.json` bind to archived commit `42ed5d45`. The PUF note now names that commit before its disclosure-record and disaggregator citations rather than after them. The manifest's ESI origin spells it `archived commit 42ed5d45 datasets/cps/cps.py` instead of leaving another commit in the string to pin it. The uk-data citations in `uk/uk_data_target_parity.json` name the register's incumbent commit `a43e8c5`, and those in `uk/uk_population_targets.json` name its registry ref `12a1e028`. Reading each cited range at its commit also corrected seven citations that pointed at the wrong code: SSI take-up (`cps.py` 584 / 1497-1499 labels swapped), SCF net worth (`source_impute.py` fit at 1227-1240), `hmrc_cgt.py` 195 to 196, `hmrc_spi.py` 230 to 229, `frs_only.py` to `services.py` 137-148, `_land.py` 52-73 (the file has 65 lines) to per-sector ranges, and the salary-sacrifice drop (`build_loss_matrix.py` 376-394 to the `except Exception` in `hmrc_salary_sacrifice.py`). Re-pinned: the `source_stages.json` raw hash (`sources.yaml` `stage_asset`, the generator's frozen hash, `us_spec_bundle.py`, `test_us_bundle_core_contracts.py`), the regenerated `us/spec/sources.yaml`, the US `spec_sha256` and `documentation_sha256`, three `inventory_coverage.EXPECTED_HASHES` (`source_manifest`, `late_resource_semantics`, `full_checkpoint`) and `docs/evidence/spec-engine/us-f0-coverage.json`. Existing stacked US pool stage checkpoints will be treated as stale and rebuilt. No behaviour or data changes. diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index 0873a6fd2b..f42cc19376 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -2809,6 +2809,15 @@ def test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its f"[a](https://github.com/%50olicyEngine/x/../%6Dicrocosm/blob/{sha}/LICENSE?a/../../../../..#L3)", f'{code_span}a', f'{code_span}a', + # A Markdown renderer escapes a backslash (``%5C``), so a ``..`` after + # ``x\y`` removes the whole segment. In an HTML ``href`` a browser + # reads the backslash as ``/``, and the same URL links + # ``PolicyEngine/x/microcosm``. At ``4dc0734e`` no rule refused these; + # the round-13 review found the first shape too. + f"[a](https://github.com/PolicyEngine/x\\y/../microcosm/blob/{sha}/LICENSE#L3)", + f"[a]\n\n[a]: https://github.com/PolicyEngine/x\\y/../microcosm/blob/{sha}/LICENSE#L3", + f"[a]()", + f"", ): assert line_citations(written) == [], written assert line_links_of_this_repository({"note": written}) == [written], written diff --git a/test_support/microcosm_build/spec_source_citations.py b/test_support/microcosm_build/spec_source_citations.py index a2ee33e64b..f4ae81e01e 100644 --- a/test_support/microcosm_build/spec_source_citations.py +++ b/test_support/microcosm_build/spec_source_citations.py @@ -111,23 +111,27 @@ ``#`` after it, whatever file the link names and whether or not a reading finds a citation in it (``line_links_of_this_repository``). The source is read as a browser and then GitHub take a URL's path: a backslash read as -``/`` and the dot segments before each ``?`` or ``#`` resolved, either dot -perhaps ``%2e`` and an empty segment counted; only then the escapes of -unreserved characters and of ``/`` decoded and a run of slashes read as one; -and the owner and the name matched in any case. A dot segment can form the -path where no reading finds a citation, as in a link to ``LICENSE``, whose -name no reading takes for a source: +``/``, as a browser reads it in an HTML attribute, and again as no +separator, as a Markdown renderer leaves it (escaped as ``%5C``); the dot +segments before each ``?`` or ``#`` resolved, either dot perhaps ``%2e`` and +an empty segment counted; only then the escapes of unreserved characters and +of ``/`` decoded and a run of slashes read as one; and the owner and the +name matched in any case. A dot segment can form the path where no reading +finds a citation, as in a link to ``LICENSE``, whose name no reading takes +for a source: ``https://github.com/PolicyEngine/x/../microcosm/blob//LICENSE#L3``. A renderer takes a source to the URL it follows by the decodings ``_sources`` -makes, a browser's steps (a backslash as ``/``, dot segments resolved, and -characters that neither the path nor a ``#`` holds percent-encoded) and -GitHub's (unreserved escapes decoded, a run of slashes as one, the owner and -the name in any case; it decodes no ``%2f`` in them, so decoding one here -can only add a refusal); and the first ``#`` after the path in a URL starts -its fragment. So a link that names this -repository's path on GitHub and a line is refused however a reading takes -it. A relative link names no repository, and a redirect (a short link, a -renamed repository) names another path: neither is read. +makes, a Markdown renderer's percent-encoding (of a backslash or a tab, +among others, which a browser then reads as part of a segment), a browser's +steps (a backslash as ``/``, dot segments resolved, and characters that +neither the path nor a ``#`` holds percent-encoded) and GitHub's (unreserved +escapes decoded, a run of slashes as one, the owner and the name in any +case; it decodes no ``%2f`` in them, so decoding one here can only add a +refusal); and the first ``#`` after the path in a URL starts its fragment. +So a link that names this repository's path on GitHub and a line is refused +however a reading takes it. A relative link names no repository, and a +redirect (a short link, a renamed repository) names another path: neither is +read. The recognised spellings are the ones ``_LINE_CITATIONS`` documents. A source name followed by a line locator in no recognised spelling, with no ``.``, @@ -456,6 +460,11 @@ _ESCAPED_DOT = re.compile(r"%2[eE]") #: What ends a URL's path: a browser resolves its dot segments before it. _PATH_END = re.compile(r"([?#])") +#: How a backslash in a URL's path is read: as ``/`` by a browser in an HTML +#: attribute, and as no separator in a Markdown destination, which a renderer +#: percent-encodes (``%5C``) before a browser reads it; GitHub reads ``%5C`` +#: as no separator either. +_BACKSLASH_READINGS = ("/", "\\") #: A dotted Microcosm symbol named in prose, such as #: ``microcosm.graph.executor._structural_columns``. DOTTED_MICROCOSM_SYMBOL = re.compile( @@ -1218,16 +1227,16 @@ def dot_segment_line_citations(value: object) -> list[LineCitation]: ] -def _browsed(source: str) -> str: +def _browsed(source: str, backslash: str) -> str: """A source with its dot segments resolved as a browser resolves a URL's. - A backslash is read as ``/``, either dot may be ``%2e``, and an empty - segment counts: ``PolicyEngine//x/../../microcosm`` is + A backslash is read as ``backslash``, either dot may be ``%2e``, and an + empty segment counts: ``PolicyEngine//x/../../microcosm`` is ``PolicyEngine/microcosm``. A ``?`` or a ``#`` ends a path, so no dot segment after one removes a segment before it. """ - pieces = _PATH_END.split(_ESCAPED_DOT.sub(".", source.replace("\\", "/"))) + pieces = _PATH_END.split(_ESCAPED_DOT.sub(".", source.replace("\\", backslash))) return "".join(map(_remove_dot_segments, pieces)) @@ -1241,13 +1250,19 @@ def _links_a_line_here(source: str) -> bool: (``PolicyEngine/x/../microcosm/``), and only then its ``_DECODED`` escapes decoded and its runs of slashes read as one. A browser reads ``%2f`` as no separator, and GitHub decodes none in the owner or the - name, so decoding one here can only add a refusal. + name, so decoding one here can only add a refusal. Each backslash + reading in ``_BACKSLASH_READINGS`` is tried: in + ``[a](https://github.com/PolicyEngine/x\\y/../microcosm/...)`` the + ``..`` removes ``x%5Cy``, and the same ``href`` in HTML links + ``PolicyEngine/x/microcosm``. """ - path = _SLASH_RUN.sub("/", _decode(_browsed(source))) return any( _LINE_FRAGMENT.match(fragment) and _REPOSITORY_ON_GITHUB.search(before) - for before, fragment in pairwise(path.split("#")) + for backslash in _BACKSLASH_READINGS + for before, fragment in pairwise( + _SLASH_RUN.sub("/", _decode(_browsed(source, backslash))).split("#") + ) ) From e5e14e96d6c1a2fa8d2cca08cfc91f304d408fdf Mon Sep 17 00:00:00 2001 From: Max Ghenis Date: Wed, 28 Oct 2026 07:54:42 -0400 Subject: [PATCH 21/21] Draw a backslash path in the property for line links here The round-13 review noted that rule 7's property drew no path through a segment holding a backslash, the shape its finding used. The property now draws `PolicyEngine/x\y/../microcosm`, which a Markdown renderer escapes (`x%5Cy`, removed by the `..`) and a browser in an `href` reads as `PolicyEngine/x/microcosm`; either way it is refused. Its twin, `PolicyEngine/x\..\..\microcosm`, links no line here in Markdown or in HTML and is not refused. An explicit example pairs the two in a reference definition with a `Dockerfile`. Co-Authored-By: Claude Opus 5.5 --- .../shared/test_spec_source_citations.py | 27 ++++++++++++++++--- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py index f42cc19376..240ad69690 100644 --- a/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py +++ b/packages/microcosm-build/tests/engine_free/shared/test_spec_source_citations.py @@ -2850,8 +2850,10 @@ def test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its "x y@github。com:443", ) #: This repository's owner and name as a browser and then GitHub route them, -#: through dot segments too, and other repositories' of the same shape, two -#: of them through a dot segment that leaves another path or none at all. +#: through dot segments too, one past a segment holding a backslash, which a +#: Markdown renderer escapes and a browser reads as ``/`` in an ``href``; and +#: other repositories' of the same shape, three of them through a dot segment +#: that leaves another path or none at all. _THIS_REPOSITORY_PATHS = ( "PolicyEngine/microcosm", "policyengine/MICROCOSM", @@ -2861,6 +2863,7 @@ def test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its "PolicyEngine/x/../microcosm", "PolicyEngine//x/%2e%2E/../microcosm", "PolicyEngine/a%2fb/../microcosm", + "PolicyEngine/x\\y/../microcosm", ) _OTHER_REPOSITORY_PATHS = ( "PolicyEngine/microcosm-dynamics", @@ -2869,6 +2872,7 @@ def test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its "PolicyEngine/micro", "PolicyEngine/x/../../microcosm", "PolicyEngine/x%2f..%2fmicrocosm", + "PolicyEngine/x\\..\\..\\microcosm", ) #: Files here whose names no reading takes for a source, so no rule but the #: link's own refuses a line link of one. @@ -2952,6 +2956,20 @@ def test_a_line_link_of_this_repository_is_refused_where_a_dot_segment_forms_its a=3, b=9, ) +@example( + enclosure="[a]\n\n[a]: <{url}>", + authority="github.com", + repository="PolicyEngine/x\\y/../microcosm", + other="PolicyEngine/x\\..\\..\\microcosm", + route="blob", + ref="main", + path="docs/evidence/us-release-reimputation-1191/scripts/platform/Dockerfile", + query="", + fragment="#L{a}", + sha="a1b2c3d", + a=3, + b=3, +) def test_a_line_link_of_this_repository_is_refused_whatever_its_ref_or_authority_holds( enclosure: str, authority: str, @@ -2969,8 +2987,9 @@ def test_a_line_link_of_this_repository_is_refused_whatever_its_ref_or_authority # markdown-it-py 4.2, parse5 7.3 and bun 1.3 render each of these as a # link to the file's lines on github.com, which routes the path, except a # full-width host in a Markdown destination, which markdown-it writes as - # no URL bun reads; each is refused, pinned or not, and another - # repository's link of the same shape is not. + # no URL bun reads, and ``x\y/..`` in an ``href``, which a browser reads + # as ``x/``; each is refused, pinned or not, and another repository's link + # of the same shape is not. def written(repository: str) -> str: line = fragment.format(a=a, b=b) url = f"https://{authority}/{repository}/{route}/{ref}/{path}{query}{line}"