From f171219dec318cc69bc15aa1f84cfe187116eb2b Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Mon, 7 Sep 2026 18:55:23 +0000 Subject: [PATCH 01/14] cdc_mortality_newimp --- .../us_cdc/county_mortality/README.md | 40 + .../county_mortality_metadata.csv | 11 + .../county_mortality_pvmap.csv | 143 ++ .../us_cdc/county_mortality/download.py | 599 +++++ .../us_cdc/county_mortality/download_test.py | 220 ++ .../golden_data/golden_observations.csv | 2095 +++++++++++++++++ .../golden_data/golden_summary_report.csv | 3 + .../us_cdc/county_mortality/manifest.json | 30 + .../underlyingcauseofdeath_county_input.csv | 71 + .../underlyingcauseofdeath_county_output.csv | 71 + .../underlyingcauseofdeath_county_output.tmcf | 8 + .../county_mortality/validation_config.json | 39 + 12 files changed, 3330 insertions(+) create mode 100644 statvar_imports/us_cdc/county_mortality/README.md create mode 100644 statvar_imports/us_cdc/county_mortality/county_mortality_metadata.csv create mode 100644 statvar_imports/us_cdc/county_mortality/county_mortality_pvmap.csv create mode 100644 statvar_imports/us_cdc/county_mortality/download.py create mode 100644 statvar_imports/us_cdc/county_mortality/download_test.py create mode 100644 statvar_imports/us_cdc/county_mortality/golden_data/golden_observations.csv create mode 100644 statvar_imports/us_cdc/county_mortality/golden_data/golden_summary_report.csv create mode 100644 statvar_imports/us_cdc/county_mortality/manifest.json create mode 100644 statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_input.csv create mode 100644 statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv create mode 100644 statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.tmcf create mode 100644 statvar_imports/us_cdc/county_mortality/validation_config.json diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md new file mode 100644 index 0000000000..d5a7339948 --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -0,0 +1,40 @@ +### CDC WONDER County-Level Mortality Across All Causes (`CDC_Mortality_Count`) + +This import acquires and processes county-level mortality statistics across all causes of death (ICD-10 113 Cause List) for all 50 US States and Washington D.C. from the CDC WONDER database. + +- **Import Name**: `CDC_Mortality_Count` +- **Source Database**: CDC WONDER Underlying Cause of Death (Database D158) +- **Source URL**: `https://wonder.cdc.gov/ucd-icd10-expanded.html` +- **Geographic Granularity**: County level (all ~3,143 US counties across all states) +- **Temporal Coverage**: 2018 to 2024 (P1Y frequency) +- **Cause Coverage**: All diseases in the ICD-10 113 Cause List (e.g. Septicemia, Diabetes Mellitus, Major Cardiovascular Diseases, Alzheimer's, Malignant Neoplasms, Respiratory Diseases, etc.) + +--- + +### Workflow + +The import consists of two fully automated steps: + +#### Step 1: Download Source Data +```bash +python3 download.py +``` +or with specific flags: +```bash +python3 download.py --states=all --years=2018-2024 --output_dir=input_files +``` +The download script automatically handles CDC WONDER sessions, agreements, rate-limiting backoffs, and query partitioning. + +#### Step 2: Process Data into Cleaned SVObs and TMCF +```bash +python3 ../../../tools/statvar_importer/stat_var_processor.py \ + --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ + --input_data=input_files/*.csv \ + --pv_map=county_mortality_pvmap.csv \ + --config_file=county_mortality_metadata.csv \ + --output_path=output/underlyingcauseofdeath_county +``` + +#### Output Artifacts +* `output/underlyingcauseofdeath_county.csv`: Cleaned observations mapping FIPS (`geoId/{fips}`), year, StatVar (`Count_MortalityEvent_`), and death count. +* `output/underlyingcauseofdeath_county.tmcf`: Template MCF mapping CSV columns to Data Commons Knowledge Graph entities. diff --git a/statvar_imports/us_cdc/county_mortality/county_mortality_metadata.csv b/statvar_imports/us_cdc/county_mortality/county_mortality_metadata.csv new file mode 100644 index 0000000000..5ebf2e3a4c --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/county_mortality_metadata.csv @@ -0,0 +1,11 @@ +parameter,value +url,https://wonder.cdc.gov/controller/datarequest/D158 +description,"County-level mortality statistics across all causes of death (ICD-10 113 Cause List) for all US states" +#place_type,county level +start_date,2018 +end_date,2024 +release_frequency,P1Y +output_columns,"observationAbout,observationDate,variableMeasured,value,unit" +mapped_rows,1 +mapped_columns,11 +header_rows,1 diff --git a/statvar_imports/us_cdc/county_mortality/county_mortality_pvmap.csv b/statvar_imports/us_cdc/county_mortality/county_mortality_pvmap.csv new file mode 100644 index 0000000000..e8c58f3d8a --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/county_mortality_pvmap.csv @@ -0,0 +1,143 @@ +key,p1,v1,p2,v2 +County Code,#Format,CountyCode1={Number:0>5},observationAbout,geoId/{CountyCode1} +Year Code,observationDate,{Number},observationPeriod,P1Y +Deaths,populationType,MortalityEvent,value,{Number} +Population,#ignore,ignore,value,{Number} +Crude Rate,#ignore,ignore,value,{Number} +GR113-001,causeOfDeath,SalmonellaInfections,, +GR113-003,causeOfDeath,CertainOtherIntestinalInfections,, +GR113-004,causeOfDeath,Tuberculosis,, +GR113-005,causeOfDeath,RespiratoryTuberculosis,, +GR113-006,causeOfDeath,OtherTuberculosis,, +GR113-009,causeOfDeath,MeningococcalInfection,, +GR113-010,causeOfDeath,Septicemia,, +GR113-015,causeOfDeath,ViralHepatitis,, +GR113-016,causeOfDeath,HumanImmunodeficiencyVirus(Hiv)Disease,, +GR113-018,causeOfDeath,OtherAndUnspecifiedInfectiousAndParasiticDiseasesAndTheirSequelae,, +GR113-019,causeOfDeath,MalignantNeoplasms,, +GR113-020,causeOfDeath,MalignantNeoplasmsOfLip_OralCavityAndPharynx,, +GR113-021,causeOfDeath,MalignantNeoplasmOfEsophagus,, +GR113-022,causeOfDeath,MalignantNeoplasmOfStomach,, +GR113-023,causeOfDeath,MalignantNeoplasmsOfColon_RectumAndAnus,, +GR113-024,causeOfDeath,MalignantNeoplasmsOfLiverAndIntrahepaticBileDucts,, +GR113-025,causeOfDeath,MalignantNeoplasmOfPancreas,, +GR113-026,causeOfDeath,MalignantNeoplasmOfLarynx,, +GR113-027,causeOfDeath,MalignantNeoplasmsOfTrachea_BronchusAndLung,, +GR113-028,causeOfDeath,MalignantMelanomaOfSkin,, +GR113-029,causeOfDeath,MalignantNeoplasmOfBreast,, +GR113-030,causeOfDeath,MalignantNeoplasmOfCervixUteri,, +GR113-031,causeOfDeath,MalignantNeoplasmsOfCorpusUteriAndUterus_PartUnspecified,, +GR113-032,causeOfDeath,MalignantNeoplasmOfOvary,, +GR113-033,causeOfDeath,MalignantNeoplasmOfProstate,, +GR113-034,causeOfDeath,MalignantNeoplasmsOfKidneyAndRenalPelvis,, +GR113-035,causeOfDeath,MalignantNeoplasmOfBladder,, +GR113-036,causeOfDeath,MalignantNeoplasmsOfMeninges_BrainAndOtherPartsOfCentralNervousSystem,, +GR113-037,causeOfDeath,MalignantNeoplasmsOfLymphoid_HematopoieticAndRelatedTissue,, +GR113-038,causeOfDeath,HodgkinsDisease,, +GR113-039,causeOfDeath,Non-HodgkinsLymphoma,, +GR113-040,causeOfDeath,Leukemia,, +GR113-041,causeOfDeath,MultipleMyelomaAndImmunoproliferativeNeoplasms,, +GR113-042,causeOfDeath,OtherAndUnspecifiedMalignantNeoplasmsOfLymphoid_HematopoieticAndRelatedTissue,, +GR113-043,causeOfDeath,AllOtherAndUnspecifiedMalignantNeoplasms,, +GR113-044,causeOfDeath,InSituNeoplasms_BenignNeoplasmsAndNeoplasmsOfUncertainOrUnknownBehavior,, +GR113-045,causeOfDeath,Anemias,, +GR113-046,causeOfDeath,DiabetesMellitus,, +GR113-047,causeOfDeath,NutritionalDeficiencies,, +GR113-048,causeOfDeath,Malnutrition,, +GR113-049,causeOfDeath,OtherNutritionalDeficiencies,, +GR113-050,causeOfDeath,Meningitis,, +GR113-051,causeOfDeath,ParkinsonsDisease,, +GR113-052,causeOfDeath,AlzheimersDisease,, +GR113-053,causeOfDeath,MajorCardiovascularDiseases,, +GR113-054,causeOfDeath,DiseasesOfHeart,, +GR113-055,causeOfDeath,AcuteRheumaticFeverAndChronicRheumaticHeartDiseases,, +GR113-056,causeOfDeath,HypertensiveHeartDisease,, +GR113-057,causeOfDeath,HypertensiveHeartAndRenalDisease,, +GR113-058,causeOfDeath,IschemicHeartDiseases,, +GR113-059,causeOfDeath,AcuteMyocardialInfarction,, +GR113-060,causeOfDeath,OtherAcuteIschemicHeartDiseases,, +GR113-061,causeOfDeath,OtherFormsOfChronicIschemicHeartDisease,, +GR113-062,causeOfDeath,AtheroscleroticCardiovascularDisease_SoDescribed,, +GR113-063,causeOfDeath,AllOtherFormsOfChronicIschemicHeartDisease,, +GR113-064,causeOfDeath,OtherHeartDiseases,, +GR113-065,causeOfDeath,AcuteAndSubacuteEndocarditis,, +GR113-066,causeOfDeath,DiseasesOfPericardiumAndAcuteMyocarditis,, +GR113-067,causeOfDeath,HeartFailure,, +GR113-068,causeOfDeath,AllOtherFormsOfHeartDisease,, +GR113-069,causeOfDeath,Essential(Primary)HypertensionAndHypertensiveRenalDisease,, +GR113-070,causeOfDeath,CerebrovascularDiseases,, +GR113-071,causeOfDeath,Atherosclerosis,, +GR113-072,causeOfDeath,OtherDiseasesOfCirculatorySystem,, +GR113-073,causeOfDeath,AorticAneurysmAndDissection,, +GR113-074,causeOfDeath,OtherDiseasesOfArteries_ArteriolesAndCapillaries,, +GR113-075,causeOfDeath,OtherDisordersOfCirculatorySystem,, +GR113-076,causeOfDeath,InfluenzaAndPneumonia,, +GR113-077,causeOfDeath,Influenza,, +GR113-078,causeOfDeath,Pneumonia,, +GR113-079,causeOfDeath,OtherAcuteLowerRespiratoryInfections,, +GR113-080,causeOfDeath,AcuteBronchitisAndBronchiolitis,, +GR113-081,causeOfDeath,UnspecifiedAcuteLowerRespiratoryInfection,, +GR113-082,causeOfDeath,ChronicLowerRespiratoryDiseases,, +GR113-083,causeOfDeath,Bronchitis_ChronicAndUnspecified,, +GR113-084,causeOfDeath,Emphysema,, +GR113-085,causeOfDeath,Asthma,, +GR113-086,causeOfDeath,OtherChronicLowerRespiratoryDiseases,, +GR113-087,causeOfDeath,PneumoconiosesAndChemicalEffects,, +GR113-088,causeOfDeath,PneumonitisDueToSolidsAndLiquids,, +GR113-089,causeOfDeath,OtherDiseasesOfRespiratorySystem,, +GR113-090,causeOfDeath,PepticUlcer,, +GR113-091,causeOfDeath,DiseasesOfAppendix,, +GR113-092,causeOfDeath,Hernia,, +GR113-093,causeOfDeath,ChronicLiverDiseaseAndCirrhosis,, +GR113-094,causeOfDeath,AlcoholicLiverDisease,, +GR113-095,causeOfDeath,OtherChronicLiverDiseaseAndCirrhosis,, +GR113-096,causeOfDeath,CholelithiasisAndOtherDisordersOfGallbladder,, +GR113-097,causeOfDeath,Nephritis_NephroticSyndromeAndNephrosis,, +GR113-098,causeOfDeath,AcuteAndRapidlyProgressiveNephriticAndNephroticSyndrome,, +GR113-099,causeOfDeath,ChronicGlomerulonephritis_NephritisAndNephritisNotSpecifiedAsAcuteOrChronic_AndRenalSclerosisUnspecified,, +GR113-100,causeOfDeath,RenalFailure,, +GR113-102,causeOfDeath,InfectionsOfKidney,, +GR113-103,causeOfDeath,HyperplasiaOfProstate,, +GR113-104,causeOfDeath,InflammatoryDiseasesOfFemalePelvicOrgans,, +GR113-105,causeOfDeath,Pregnancy_ChildbirthAndThePuerperium,, +GR113-106,causeOfDeath,PregnancyWithAbortiveOutcome,, +GR113-107,causeOfDeath,OtherComplicationsOfPregnancy_ChildbirthAndThePuerperium,, +GR113-108,causeOfDeath,CertainConditionsOriginatingInThePerinatalPeriod,, +GR113-109,causeOfDeath,CongenitalMalformations_DeformationsAndChromosomalAbnormalities,, +GR113-110,causeOfDeath,Symptoms_SignsAndAbnormalClinicalAndLaboratoryFindings_NotElsewhereClassified,, +GR113-112,causeOfDeath,Accidents(UnintentionalInjuries),, +GR113-113,causeOfDeath,TransportAccidents,, +GR113-114,causeOfDeath,MotorVehicleAccidents,, +GR113-115,causeOfDeath,OtherLandTransportAccidents,, +GR113-116,causeOfDeath,Water_AirAndSpace_AndOtherAndUnspecifiedTransportAccidentsAndTheirSequelae,, +GR113-117,causeOfDeath,NontransportAccidents,, +GR113-118,causeOfDeath,Falls,, +GR113-119,causeOfDeath,AccidentalDischargeOfFirearms,, +GR113-120,causeOfDeath,AccidentalDrowningAndSubmersion,, +GR113-121,causeOfDeath,AccidentalExposureToSmoke_FireAndFlames,, +GR113-122,causeOfDeath,AccidentalPoisoningAndExposureToNoxiousSubstances,, +GR113-123,causeOfDeath,OtherAndUnspecifiedNontransportAccidentsAndTheirSequelae,, +GR113-124,causeOfDeath,Suicide,, +GR113-125,causeOfDeath,IntentionalSelf-Harm(Suicide)ByDischargeOfFirearms,, +GR113-126,causeOfDeath,IntentionalSelf-Harm(Suicide)ByOtherAndUnspecifiedMeansAndTheirSequelae,, +GR113-127,causeOfDeath,Assault(Homicide),, +GR113-128,causeOfDeath,Assault(Homicide)ByDischargeOfFirearms,, +GR113-129,causeOfDeath,Assault(Homicide)ByOtherAndUnspecifiedMeansAndTheirSequelae,, +GR113-130,causeOfDeath,LegalIntervention,, +GR113-131,causeOfDeath,EventsOfUndeterminedIntent,, +GR113-132,causeOfDeath,DischargeOfFirearms_UndeterminedIntent,, +GR113-133,causeOfDeath,OtherAndUnspecifiedEventsOfUndeterminedIntentAndTheirSequelae,, +GR113-135,causeOfDeath,ComplicationsOfMedicalAndSurgicalCare,, +GR113-136,causeOfDeath,EnterocolitisDueToClostridiumDifficile,, +GR113-137,causeOfDeath,COVID_19,, +GR113-111,causeOfDeath,AllOtherDiseases(Residual),, +GR113-013,causeOfDeath,Arthropod-BorneViralEncephalitis,, +GR113-017,causeOfDeath,Malaria,, +GR113-134,causeOfDeath,OperationsOfWarAndTheirSequelae,, +GR113-101,causeOfDeath,OtherDisordersOfKidney,, +GR113-002,causeOfDeath,ShigellosisAndAmebiasis,, +GR113-011,causeOfDeath,Syphilis,, +GR113-007,causeOfDeath,WhoopingCough,, +GR113-008,causeOfDeath,ScarletFeverAndErysipelas,, +GR113-012,causeOfDeath,AcutePoliomyelitis,, +GR113-014,causeOfDeath,Measles,, diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py new file mode 100644 index 0000000000..eb2ea91bd1 --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -0,0 +1,599 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Automated live downloader for CDC WONDER County-Level Mortality Data (D158). + +This script automates downloading county-level mortality statistics across all +causes of death (ICD-10 113 Cause List) from CDC WONDER (Database D158: +Underlying Cause of Death). + +Data is broken down by: +- Year (2018 to 2024, or specified range) +- County (all US counties across 50 states + DC) +- ICD-10 113 Cause List (all diseases/causes) + +CDC WONDER imposes a hard limit of 75,000 rows per export query. This script +queries state by state, automatically detects if a state query exceeds the +75,000 row cap, and splits into smaller year chunks to download complete data. +""" + +import csv +import io +import os +from pathlib import Path +import time +from typing import List, Optional, Tuple +from urllib.parse import urljoin + +from absl import app, flags, logging +from bs4 import BeautifulSoup +import requests +from retry import retry + +script_dir = os.path.dirname(os.path.abspath(__file__)) +DEFAULT_INPUT_DIR = os.path.join(script_dir, "input_files") + +# Mapping of 2-digit FIPS codes to US State / Territory names +US_STATES = { + "01": "Alabama", + "02": "Alaska", + "04": "Arizona", + "05": "Arkansas", + "06": "California", + "08": "Colorado", + "09": "Connecticut", + "10": "Delaware", + "11": "District of Columbia", + "12": "Florida", + "13": "Georgia", + "15": "Hawaii", + "16": "Idaho", + "17": "Illinois", + "18": "Indiana", + "19": "Iowa", + "20": "Kansas", + "21": "Kentucky", + "22": "Louisiana", + "23": "Maine", + "24": "Maryland", + "25": "Massachusetts", + "26": "Michigan", + "27": "Minnesota", + "28": "Mississippi", + "29": "Missouri", + "30": "Montana", + "31": "Nebraska", + "32": "Nevada", + "33": "New Hampshire", + "34": "New Jersey", + "35": "New Mexico", + "36": "New York", + "37": "North Carolina", + "38": "North Dakota", + "39": "Ohio", + "40": "Oklahoma", + "41": "Oregon", + "42": "Pennsylvania", + "44": "Rhode Island", + "45": "South Carolina", + "46": "South Dakota", + "47": "Tennessee", + "48": "Texas", + "49": "Utah", + "50": "Vermont", + "51": "Virginia", + "53": "Washington", + "54": "West Virginia", + "55": "Wisconsin", + "56": "Wyoming", +} + +# High-population states that exceed CDC WONDER's row and memory limits when queried across 7 years. +# Querying directly in 2-year chunks prevents query buffer overruns and HTTP 400 errors. +LARGE_STATES: set[str] = { + "01", "05", "06", "12", "13", "17", "18", "21", "22", "26", "27", "28", "29", + "34", "36", "37", "39", "40", "42", "45", "47", "48", "51", "53", "55", +} + +# States with exceptionally large county counts (e.g. Texas with 254 counties) that frequently +# trigger HTTP 504 Gateway Timeouts when queried across multiple years during peak traffic hours. +SINGLE_YEAR_STATES: set[str] = {"48"} + +FLAGS = flags.FLAGS + +flags.DEFINE_string( + "states", + "all", + "Comma-separated 2-digit FIPS codes of states to download (e.g. '02,48'), or 'all'.", +) +flags.DEFINE_string( + "years", + "2018-2024", + "Year range ('2018-2024') or comma-separated years ('2018,2019,2020').", +) +flags.DEFINE_string( + "output_dir", + DEFAULT_INPUT_DIR, + "Directory where downloaded CSV files will be saved.", +) +flags.DEFINE_float( + "delay", + 3.0, + "Politeness delay in seconds between successive HTTP queries.", +) +flags.DEFINE_integer( + "timeout", + 120, + "HTTP request timeout in seconds.", +) +flags.DEFINE_boolean( + "skip_existing", + True, + "Skip downloading states that already have existing non-empty CSV files in output_dir.", +) +flags.DEFINE_integer( + "batch_size", + 10, + "Number of states to process per session before automatically refreshing session.", +) +flags.DEFINE_float( + "batch_cooldown", + 20.0, + "Cooldown delay in seconds between session batches to prevent rate limits.", +) + + +def parse_year_list(year_str: str) -> List[str]: + """Parses a year string like '2018-2024' or '2018,2019' into a list of year strings.""" + year_str = year_str.strip() + if "-" in year_str and not year_str.startswith("-"): + parts = year_str.split("-") + start, end = int(parts[0]), int(parts[1]) + return [str(y) for y in range(start, end + 1)] + return [y.strip() for y in year_str.split(",") if y.strip()] + + +class CdcWonderCountyMortalityDownloader: + """Automates CDC WONDER sessions and queries for County-Level Mortality data across all causes.""" + + def __init__( + self, + landing_url: str = "https://wonder.cdc.gov/ucd-icd10-expanded.html", + timeout: int = 120, + delay: float = 3.0, + ): + self.landing_url = landing_url + self.timeout = timeout + self.delay = delay + self.session = requests.Session() + self.session.headers.update( + {"User-Agent": "Mozilla/5.0 (DataCommons CDC Importer; contact: support@datacommons.org)"} + ) + self.action_url: Optional[str] = None + self.base_post_data: List[Tuple[str, str]] = [] + + @retry( + tries=3, + delay=5, + backoff=2, + exceptions=(requests.RequestException, ValueError), + ) + def init_session(self): + """Connects to landing page, agrees to data use terms, and stores pre-populated form state.""" + if hasattr(self, "session") and self.session: + try: + self.session.close() + except Exception: + pass + self.session = requests.Session() + self.session.headers.update( + {"User-Agent": "Mozilla/5.0 (DataCommons CDC Importer; contact: support@datacommons.org)"} + ) + self.action_url = None + self.base_post_data = [] + + logging.info("Connecting to CDC WONDER landing page: %s", self.landing_url) + res = self.session.get(self.landing_url, timeout=self.timeout) + res.raise_for_status() + + soup = BeautifulSoup(res.text, "lxml") + form = soup.find("form", id="wonderform") + if not form: + raise ValueError("Could not find initial wonderform on CDC WONDER page.") + + action = urljoin(self.landing_url, form.get("action")) + agree_inputs = [ + (inp.get("name"), inp.get("value", "")) + for inp in form.find_all("input") + if inp.get("name") + ] + agree_inputs.append(("action-I Agree", "I Agree")) + + logging.info("Submitting Data Use Agreement (I Agree)...") + res_agree = self.session.post(action, data=agree_inputs, timeout=self.timeout) + res_agree.raise_for_status() + + soup_req = BeautifulSoup(res_agree.text, "lxml") + form_req = soup_req.find("form", id="wonderform") + if not form_req: + raise ValueError("Could not find request form after agreeing to terms.") + + self.action_url = urljoin(self.landing_url, form_req.get("action")) + + # Extract pre-populated query parameters + self.base_post_data = [] + for el in form_req.find_all(["input", "select", "textarea"]): + name = el.get("name") + if not name: + continue + if el.name == "input": + itype = el.get("type", "text").lower() + if itype in ["submit", "button", "reset", "image"]: + continue + if itype in ["checkbox", "radio"]: + if el.has_attr("checked"): + self.base_post_data.append((name, el.get("value", "on"))) + else: + self.base_post_data.append((name, el.get("value", ""))) + elif el.name == "select": + selected_opts = [opt for opt in el.find_all("option") if opt.has_attr("selected")] + if selected_opts: + for opt in selected_opts: + self.base_post_data.append((name, opt.get("value", ""))) + else: + if not el.has_attr("multiple"): + first_opt = el.find("option") + if first_opt: + self.base_post_data.append((name, first_opt.get("value", ""))) + elif el.name == "textarea": + self.base_post_data.append((name, el.text or "")) + + logging.info("Successfully established CDC WONDER session with action: %s", self.action_url) + + def _build_post_data( + self, state_fips: str, years: Optional[List[str]] = None + ) -> List[Tuple[str, str]]: + """Constructs query payload for county-level mortality across all causes.""" + query_data: List[Tuple[str, str]] = [] + for k, v in self.base_post_data: + # Grouping fields: + # B_1: Year + # B_2: County + # B_3: ICD-10 113 Cause List + # B_4: *None* + # B_5: *None* + if k == "B_1": + query_data.append((k, "D158.V1-level1")) + elif k == "B_2": + query_data.append((k, "D158.V9-level2")) + elif k == "B_3": + query_data.append((k, "D158.V4")) + elif k == "B_4": + query_data.append((k, "*None*")) + elif k == "B_5": + query_data.append((k, "*None*")) + elif k == "F_D158.V9": + # Filter by state FIPS code + query_data.append((k, state_fips)) + elif k == "F_D158.V1": + # Year filter - handled below + if not years: + query_data.append((k, v)) + else: + query_data.append((k, v)) + + if years: + for y in years: + query_data.append(("F_D158.V1", y)) + + query_data.append(("action-Export Results", "Export Results")) + return query_data + + def execute_query( + self, + state_fips: str, + years: Optional[List[str]] = None, + max_retries: int = 5, + ) -> str: + """Executes query with automatic 429 rate-limit backoff and session renewal.""" + if not self.action_url or not self.base_post_data: + self.init_session() + + payload = self._build_post_data(state_fips, years) + + for attempt in range(1, max_retries + 1): + try: + res = self.session.post(self.action_url, data=payload, timeout=self.timeout) + + if res.status_code == 429: + retry_after = res.headers.get("Retry-After") + wait_seconds = int(retry_after) if retry_after and retry_after.isdigit() else 1860 + logging.warning( + "Encountered HTTP 429 (Too Many Requests). CDC WONDER enforces a 30-minute " + "IP block. Waiting %d seconds (%d min) in complete silence for block to " + "clear (attempt %d)...", + wait_seconds, + wait_seconds // 60, + attempt, + ) + time.sleep(wait_seconds) + logging.info("Block elapsed. Re-initializing new CDC WONDER session...") + self.init_session() + payload = self._build_post_data(state_fips, years) + continue + + if res.status_code == 400 and ("too much data" in res.text or "simplify your filters" in res.text): + logging.warning( + "FIPS %s query returned 'too much data' (HTTP 400). Returning response for partitioning.", + state_fips, + ) + return res.text + + res.raise_for_status() + return res.text + + except requests.RequestException as e: + logging.warning("Query failed for FIPS %s (attempt %d/%d): %s", state_fips, attempt, max_retries, e) + if attempt == max_retries: + raise + time.sleep(self.delay * attempt) + self.init_session() + payload = self._build_post_data(state_fips, years) + + raise RuntimeError(f"Failed to fetch data for state FIPS {state_fips} after {max_retries} retries.") + + def download_state( + self, state_fips: str, years: List[str] + ) -> List[Tuple[str, str]]: + """Downloads county mortality data for a state, automatically partitioning if needed.""" + state_name = US_STATES.get(state_fips, f"FIPS-{state_fips}") + results: List[Tuple[str, str]] = [] + need_partitioning = state_fips in LARGE_STATES + + if not need_partitioning: + logging.info("Querying full year range (%s) for state FIPS %s (%s)...", years, state_fips, state_name) + try: + tsv_text = self.execute_query(state_fips, years) + first_line = tsv_text.splitlines()[0] if tsv_text else "" + if "County Code" in first_line: + results.append(("all", tsv_text)) + time.sleep(self.delay) + return results + logging.warning( + "%s response not TSV (likely exceeded 75k rows / too much data). Partitioning into chunks...", + state_name, + ) + need_partitioning = True + except Exception as e: + logging.warning( + "Querying all %d years for %s encountered %s. Partitioning into year chunks...", + len(years), + state_name, + e, + ) + need_partitioning = True + + if need_partitioning: + logging.info("Partitioning %s (%s) into chunks...", state_name, state_fips) + use_single_years = (len(years) <= 2) or (state_fips in SINGLE_YEAR_STATES) + i = 0 + while i < len(years): + if not use_single_years and (i + 1 < len(years)): + year_chunk = years[i : i + 2] + chunk_label = f"{year_chunk[0]}_{year_chunk[-1]}" + logging.info("Querying chunk %s for %s...", chunk_label, state_name) + time.sleep(self.delay) + + chunk_tsv = "" + success = False + try: + chunk_tsv = self.execute_query(state_fips, year_chunk, max_retries=2) + chunk_first_line = chunk_tsv.splitlines()[0] if chunk_tsv else "" + if "County Code" in chunk_first_line: + success = True + else: + logging.warning( + "Chunk %s response for %s did not contain valid TSV data. Splitting into single years.", + chunk_label, + state_name, + ) + except Exception as e: + logging.warning( + "Chunk %s for %s failed with %s. Falling back to single-year queries for this state.", + chunk_label, + state_name, + e, + ) + + if success: + results.append((chunk_label, chunk_tsv)) + i += 2 + continue + else: + use_single_years = True + + single_year = years[i] + logging.info("Querying single year %s for %s...", single_year, state_name) + time.sleep(self.delay) + sy_text = self.execute_query(state_fips, [single_year]) + first_line = sy_text.splitlines()[0] if sy_text else "" + if "County Code" not in first_line: + raise ValueError(f"Failed to query {state_name} even for single year {single_year}.") + results.append((single_year, sy_text)) + i += 1 + + return results + + +def save_tsv_as_csv(raw_tsv: str, output_csv_path: str) -> int: + """Parses raw CDC TSV export into clean CSV format, stripping footer caveats.""" + Path(os.path.dirname(output_csv_path)).mkdir(parents=True, exist_ok=True) + tsv_reader = csv.reader(io.StringIO(raw_tsv), delimiter="\t") + + temp_csv_path = f"{output_csv_path}.tmp" + row_count = 0 + with open(temp_csv_path, "w", newline="", encoding="utf-8") as f: + csv_writer = csv.writer(f) + for row in tsv_reader: + if not row: + continue + if row[0].startswith("---") or (len(row) > 1 and row[1].startswith("---")): + break + csv_writer.writerow(row) + row_count += 1 + + os.replace(temp_csv_path, output_csv_path) + return row_count + + +def is_state_downloaded( + output_dir: str, state_fips: str, years: Optional[List[str]] = None +) -> bool: + """Checks if valid non-empty CSV files for this state already exist covering requested years.""" + pattern = f"UnderlyingCauseofDeath_County_{state_fips}*.csv" + matches = list(Path(output_dir).glob(pattern)) + if not matches: + return False + if not all(f.stat().st_size > 100 for f in matches): + return False + if years: + initial_year = years[0] + latest_year = years[-1] + has_initial_chunk = any( + f.name.startswith(f"UnderlyingCauseofDeath_County_{state_fips}_{initial_year}") + for f in matches + ) + has_latest_chunk = any( + f.name.endswith(f"_{latest_year}.csv") or f"_{latest_year}_" in f.name + for f in matches + ) + if has_initial_chunk and has_latest_chunk: + return True + single_file = Path(output_dir) / f"UnderlyingCauseofDeath_County_{state_fips}.csv" + if single_file.exists(): + found_initial = False + found_latest = False + with open(single_file, "r", encoding="utf-8", errors="replace") as f: + for line in f: + if not found_initial and f",{initial_year}," in line: + found_initial = True + if not found_latest and f",{latest_year}," in line: + found_latest = True + if found_initial and found_latest: + return True + return False + return False + return True + + +def download_county_mortality_data( + states: List[str], + years: List[str], + output_dir: str, + delay: float = 3.0, + timeout: int = 120, + skip_existing: bool = True, + batch_size: int = 10, + batch_cooldown: float = 20.0, +): + """Downloads CDC County Mortality data for specified states and years.""" + os.makedirs(output_dir, exist_ok=True) + downloader = CdcWonderCountyMortalityDownloader(timeout=timeout, delay=delay) + downloader.init_session() + + total_files = 0 + total_rows = 0 + states_in_batch = 0 + + for idx, state_fips in enumerate(states, start=1): + state_name = US_STATES.get(state_fips, f"FIPS-{state_fips}") + + if skip_existing and is_state_downloaded(output_dir, state_fips, years=years): + existing_files = list(Path(output_dir).glob(f"UnderlyingCauseofDeath_County_{state_fips}*.csv")) + logging.info( + "[%d/%d] Skipping %s (FIPS %s): %d existing file(s) found.", + idx, + len(states), + state_name, + state_fips, + len(existing_files), + ) + continue + + logging.info( + "[%d/%d] Processing %s (FIPS %s) (Session batch item %d/%d)...", + idx, + len(states), + state_name, + state_fips, + states_in_batch + 1, + batch_size, + ) + + try: + results = downloader.download_state(state_fips, years) + for chunk_label, tsv_data in results: + if chunk_label == "all": + filename = f"UnderlyingCauseofDeath_County_{state_fips}.csv" + else: + filename = f"UnderlyingCauseofDeath_County_{state_fips}_{chunk_label}.csv" + + filepath = os.path.join(output_dir, filename) + rows = save_tsv_as_csv(tsv_data, filepath) + total_files += 1 + total_rows += rows + logging.info("Saved %s with %d rows.", filename, rows) + + states_in_batch += 1 + + # Proactive session rotation after batch_size states + if states_in_batch >= batch_size and idx < len(states): + logging.info( + "Processed batch of %d states. Taking a %.1fs cooldown and refreshing session...", + states_in_batch, + batch_cooldown, + ) + time.sleep(batch_cooldown) + downloader.init_session() + states_in_batch = 0 + + except Exception as e: + logging.fatal("Failed downloading state %s (FIPS %s): %s", state_name, state_fips, e) + + logging.info("Download complete: Saved %d files with %d total rows in %s.", total_files, total_rows, output_dir) + + +def main(_): + years = parse_year_list(FLAGS.years) + + if FLAGS.states.lower() == "all": + states = sorted(list(US_STATES.keys())) + else: + states = [s.strip().zfill(2) for s in FLAGS.states.split(",") if s.strip()] + + logging.info("Starting CDC County Mortality live download for %d states, years: %s", len(states), years) + download_county_mortality_data( + states=states, + years=years, + output_dir=FLAGS.output_dir, + delay=FLAGS.delay, + timeout=FLAGS.timeout, + skip_existing=FLAGS.skip_existing, + batch_size=FLAGS.batch_size, + batch_cooldown=FLAGS.batch_cooldown, + ) + + +if __name__ == "__main__": + app.run(main) diff --git a/statvar_imports/us_cdc/county_mortality/download_test.py b/statvar_imports/us_cdc/county_mortality/download_test.py new file mode 100644 index 0000000000..fd908eb663 --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/download_test.py @@ -0,0 +1,220 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Unit tests for CDC WONDER County Mortality Downloader.""" + +import os +from pathlib import Path +import tempfile +import unittest +from unittest import mock + +import requests + +import download + + +class DownloadTest(unittest.TestCase): + + def test_parse_year_list_range(self): + years = download.parse_year_list("2018-2024") + self.assertEqual(years, ["2018", "2019", "2020", "2021", "2022", "2023", "2024"]) + + def test_parse_year_list_comma(self): + years = download.parse_year_list("2018, 2020, 2022") + self.assertEqual(years, ["2018", "2020", "2022"]) + + def test_parse_year_list_single(self): + years = download.parse_year_list("2024") + self.assertEqual(years, ["2024"]) + + @mock.patch.object(download.requests, "Session") + def test_init_session_success(self, mock_session_cls): + mock_session = mock.MagicMock() + mock_session_cls.return_value = mock_session + + mock_res1 = mock.MagicMock() + mock_res1.text = """ + + +
+ +
+ + + """ + mock_res1.raise_for_status.return_value = None + + mock_res2 = mock.MagicMock() + mock_res2.text = """ + + +
+ + + +
+ + + """ + mock_res2.raise_for_status.return_value = None + + mock_session.get.return_value = mock_res1 + mock_session.post.return_value = mock_res2 + + downloader = download.CdcWonderCountyMortalityDownloader() + downloader.init_session() + + self.assertIn("controller/datarequest/D158;jsessionid=TEST1234", downloader.action_url) + self.assertTrue(len(downloader.base_post_data) > 0) + self.assertEqual(mock_session.post.call_count, 1) + + def test_build_post_data(self): + downloader = download.CdcWonderCountyMortalityDownloader() + downloader.base_post_data = [ + ("B_1", "old_val"), + ("B_2", "old_val"), + ("B_3", "old_val"), + ("B_4", "old_val"), + ("B_5", "old_val"), + ("F_D158.V9", "*All*"), + ("F_D158.V1", "*All*"), + ("other_key", "other_val"), + ] + + payload = downloader._build_post_data("10", ["2018", "2019"]) + payload_dict = dict(payload) + + # Check that County and ICD-10 113 Cause List are selected, while B_4/B_5 are *None* + self.assertEqual(payload_dict["B_1"], "D158.V1-level1") + self.assertEqual(payload_dict["B_2"], "D158.V9-level2") + self.assertEqual(payload_dict["B_3"], "D158.V4") + self.assertEqual(payload_dict["B_4"], "*None*") + self.assertEqual(payload_dict["B_5"], "*None*") + self.assertEqual(payload_dict["F_D158.V9"], "10") + self.assertEqual(payload_dict["action-Export Results"], "Export Results") + + year_params = [v for k, v in payload if k == "F_D158.V1"] + self.assertEqual(year_params, ["2018", "2019"]) + + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "execute_query") + def test_download_state_single_query(self, mock_query): + tsv_output = ( + "Notes\tYear\tCounty\tCounty Code\tICD-10 113 Cause List\tDeaths\n" + "\t2018\tKent County, DE\t10001\tSepticemia\t20\n" + ) + mock_query.return_value = tsv_output + + downloader = download.CdcWonderCountyMortalityDownloader() + results = downloader.download_state("10", ["2018", "2019"]) + + self.assertEqual(len(results), 1) + self.assertEqual(results[0][0], "all") + self.assertEqual(results[0][1], tsv_output) + mock_query.assert_called_once_with("10", ["2018", "2019"]) + + def test_save_tsv_as_csv(self): + raw_tsv = ( + "Notes\tYear\tCounty\tCounty Code\tDeaths\n" + "\t2018\tKent County, DE\t10001\t20\n" + "---\n" + "Query Parameters:\n" + "Caveats:\n" + ) + + with tempfile.TemporaryDirectory() as temp_dir: + output_csv = os.path.join(temp_dir, "test_output.csv") + rows = download.save_tsv_as_csv(raw_tsv, output_csv) + + self.assertEqual(rows, 2) + self.assertTrue(os.path.exists(output_csv)) + lines = Path(output_csv).read_text(encoding="utf-8").splitlines() + + self.assertEqual(len(lines), 2) + self.assertEqual(lines[0], "Notes,Year,County,County Code,Deaths") + self.assertEqual(lines[1], ',2018,"Kent County, DE",10001,20') + + def test_is_state_downloaded(self): + with tempfile.TemporaryDirectory() as temp_dir: + self.assertFalse(download.is_state_downloaded(temp_dir, "10")) + + # Create empty file + f = Path(temp_dir) / "UnderlyingCauseofDeath_County_10.csv" + f.write_text("") + self.assertFalse(download.is_state_downloaded(temp_dir, "10")) + + # Create file with only 2024 (missing 2018 initial year) -> should be False + f.write_text("Header,col1,col2,col3\n" + ",2024,val2,val3\n" * 10) + self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) + + # Create file with both initial (2018) and latest (2024) -> should be True + f.write_text("Header,col1,col2,col3\n" + ",2018,val2,val3\n" * 5 + ",2024,val2,val3\n" * 5) + self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) + self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2025"])) + + # Test partitioned chunk files + f.unlink() + chunk_2024 = Path(temp_dir) / "UnderlyingCauseofDeath_County_10_2024.csv" + chunk_2024.write_text("Header,col1\n" + "val1,val2\n" * 10) + # Only latest chunk present -> should be False + self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) + + # Both initial chunk and latest chunk present -> should be True + chunk_2018 = Path(temp_dir) / "UnderlyingCauseofDeath_County_10_2018_2019.csv" + chunk_2018.write_text("Header,col1\n" + "val1,val2\n" * 10) + self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) + + @mock.patch.object(download.time, "sleep") + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "init_session") + def test_execute_query_429_backoff(self, mock_init, mock_sleep): + downloader = download.CdcWonderCountyMortalityDownloader() + downloader.action_url = "https://wonder.cdc.gov/test" + downloader.base_post_data = [("B_1", "test")] + + mock_res_429 = mock.MagicMock() + mock_res_429.status_code = 429 + mock_res_429.headers = {"Retry-After": "1"} + + mock_res_200 = mock.MagicMock() + mock_res_200.status_code = 200 + mock_res_200.text = "Notes\tCounty Code\n" + mock_res_200.raise_for_status.return_value = None + + downloader.session.post = mock.MagicMock(side_effect=[mock_res_429, mock_res_200]) + + result = downloader.execute_query("10", ["2018"], max_retries=2) + self.assertEqual(result, "Notes\tCounty Code\n") + self.assertEqual(downloader.session.post.call_count, 2) + mock_sleep.assert_called_with(1) + mock_init.assert_called_once() + + + @mock.patch.object(download.time, "sleep") + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "execute_query") + def test_download_state_timeout_fallback_to_single_years(self, mock_query, mock_sleep): + def query_side_effect(state_fips, years, **kwargs): + if len(years) > 1: + raise requests.exceptions.HTTPError("504 Server Error: Gateway Time-out") + return f"Notes\tYear\tCounty Code\n\t{years[0]}\t01001\n" + + mock_query.side_effect = query_side_effect + downloader = download.CdcWonderCountyMortalityDownloader() + results = downloader.download_state("01", ["2018", "2019", "2020"]) + + self.assertEqual(len(results), 3) + self.assertEqual([r[0] for r in results], ["2018", "2019", "2020"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/statvar_imports/us_cdc/county_mortality/golden_data/golden_observations.csv b/statvar_imports/us_cdc/county_mortality/golden_data/golden_observations.csv new file mode 100644 index 0000000000..d6b519d995 --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/golden_data/golden_observations.csv @@ -0,0 +1,2095 @@ +"observationAbout" +"geoId/01001" +"geoId/01003" +"geoId/01005" +"geoId/01007" +"geoId/01009" +"geoId/01013" +"geoId/01015" +"geoId/01017" +"geoId/01019" +"geoId/01021" +"geoId/01023" +"geoId/01025" +"geoId/01031" +"geoId/01033" +"geoId/01039" +"geoId/01043" +"geoId/01045" +"geoId/01047" +"geoId/01049" +"geoId/01051" +"geoId/01053" +"geoId/01055" +"geoId/01057" +"geoId/01059" +"geoId/01061" +"geoId/01065" +"geoId/01067" +"geoId/01069" +"geoId/01071" +"geoId/01073" +"geoId/01077" +"geoId/01079" +"geoId/01081" +"geoId/01083" +"geoId/01087" +"geoId/01089" +"geoId/01091" +"geoId/01093" +"geoId/01095" +"geoId/01097" +"geoId/01099" +"geoId/01101" +"geoId/01103" +"geoId/01107" +"geoId/01109" +"geoId/01111" +"geoId/01113" +"geoId/01115" +"geoId/01117" +"geoId/01119" +"geoId/01121" +"geoId/01123" +"geoId/01125" +"geoId/01127" +"geoId/01129" +"geoId/01133" +"geoId/02020" +"geoId/02050" +"geoId/02090" +"geoId/02110" +"geoId/02122" +"geoId/02170" +"geoId/04001" +"geoId/04003" +"geoId/04005" +"geoId/04007" +"geoId/04009" +"geoId/04012" +"geoId/04013" +"geoId/04015" +"geoId/04017" +"geoId/04019" +"geoId/04021" +"geoId/04023" +"geoId/04025" +"geoId/04027" +"geoId/05001" +"geoId/05003" +"geoId/05005" +"geoId/05007" +"geoId/05009" +"geoId/05015" +"geoId/05017" +"geoId/05019" +"geoId/05021" +"geoId/05023" +"geoId/05027" +"geoId/05029" +"geoId/05031" +"geoId/05033" +"geoId/05035" +"geoId/05037" +"geoId/05041" +"geoId/05043" +"geoId/05045" +"geoId/05047" +"geoId/05051" +"geoId/05053" +"geoId/05055" +"geoId/05057" +"geoId/05059" +"geoId/05063" +"geoId/05067" +"geoId/05069" +"geoId/05071" +"geoId/05075" +"geoId/05077" +"geoId/05083" +"geoId/05085" +"geoId/05087" +"geoId/05089" +"geoId/05091" +"geoId/05093" +"geoId/05103" +"geoId/05107" +"geoId/05111" +"geoId/05113" +"geoId/05115" +"geoId/05119" +"geoId/05121" +"geoId/05123" +"geoId/05125" +"geoId/05131" +"geoId/05133" +"geoId/05135" +"geoId/05139" +"geoId/05141" +"geoId/05143" +"geoId/05145" +"geoId/05149" +"geoId/06001" +"geoId/06005" +"geoId/06007" +"geoId/06009" +"geoId/06011" +"geoId/06013" +"geoId/06015" +"geoId/06017" +"geoId/06019" +"geoId/06021" +"geoId/06023" +"geoId/06025" +"geoId/06027" +"geoId/06029" +"geoId/06031" +"geoId/06033" +"geoId/06035" +"geoId/06037" +"geoId/06039" +"geoId/06041" +"geoId/06043" +"geoId/06045" +"geoId/06047" +"geoId/06053" +"geoId/06055" +"geoId/06057" +"geoId/06059" +"geoId/06061" +"geoId/06063" +"geoId/06065" +"geoId/06067" +"geoId/06069" +"geoId/06071" +"geoId/06073" +"geoId/06075" +"geoId/06077" +"geoId/06079" +"geoId/06081" +"geoId/06083" +"geoId/06085" +"geoId/06087" +"geoId/06089" +"geoId/06093" +"geoId/06095" +"geoId/06097" +"geoId/06099" +"geoId/06101" +"geoId/06103" +"geoId/06107" +"geoId/06109" +"geoId/06111" +"geoId/06113" +"geoId/06115" +"geoId/08001" +"geoId/08005" +"geoId/08013" +"geoId/08014" +"geoId/08015" +"geoId/08029" +"geoId/08031" +"geoId/08035" +"geoId/08037" +"geoId/08039" +"geoId/08041" +"geoId/08043" +"geoId/08045" +"geoId/08051" +"geoId/08059" +"geoId/08067" +"geoId/08069" +"geoId/08075" +"geoId/08077" +"geoId/08083" +"geoId/08085" +"geoId/08087" +"geoId/08089" +"geoId/08093" +"geoId/08097" +"geoId/08101" +"geoId/08107" +"geoId/08117" +"geoId/08119" +"geoId/08123" +"geoId/09001" +"geoId/09003" +"geoId/09005" +"geoId/09007" +"geoId/09009" +"geoId/09011" +"geoId/09013" +"geoId/09015" +"geoId/10001" +"geoId/10003" +"geoId/10005" +"geoId/11001" +"geoId/12001" +"geoId/12003" +"geoId/12005" +"geoId/12007" +"geoId/12009" +"geoId/12011" +"geoId/12015" +"geoId/12017" +"geoId/12019" +"geoId/12021" +"geoId/12023" +"geoId/12027" +"geoId/12029" +"geoId/12031" +"geoId/12033" +"geoId/12035" +"geoId/12039" +"geoId/12041" +"geoId/12049" +"geoId/12051" +"geoId/12053" +"geoId/12055" +"geoId/12057" +"geoId/12059" +"geoId/12061" +"geoId/12063" +"geoId/12069" +"geoId/12071" +"geoId/12073" +"geoId/12075" +"geoId/12079" +"geoId/12081" +"geoId/12083" +"geoId/12085" +"geoId/12086" +"geoId/12087" +"geoId/12089" +"geoId/12091" +"geoId/12093" +"geoId/12095" +"geoId/12097" +"geoId/12099" +"geoId/12101" +"geoId/12103" +"geoId/12105" +"geoId/12107" +"geoId/12109" +"geoId/12111" +"geoId/12113" +"geoId/12115" +"geoId/12117" +"geoId/12119" +"geoId/12121" +"geoId/12123" +"geoId/12127" +"geoId/12129" +"geoId/12131" +"geoId/12133" +"geoId/13001" +"geoId/13009" +"geoId/13011" +"geoId/13013" +"geoId/13015" +"geoId/13017" +"geoId/13019" +"geoId/13021" +"geoId/13025" +"geoId/13029" +"geoId/13031" +"geoId/13033" +"geoId/13035" +"geoId/13039" +"geoId/13045" +"geoId/13047" +"geoId/13051" +"geoId/13053" +"geoId/13055" +"geoId/13057" +"geoId/13059" +"geoId/13063" +"geoId/13067" +"geoId/13069" +"geoId/13071" +"geoId/13073" +"geoId/13075" +"geoId/13077" +"geoId/13081" +"geoId/13085" +"geoId/13087" +"geoId/13089" +"geoId/13091" +"geoId/13095" +"geoId/13097" +"geoId/13103" +"geoId/13105" +"geoId/13107" +"geoId/13111" +"geoId/13113" +"geoId/13115" +"geoId/13117" +"geoId/13119" +"geoId/13121" +"geoId/13123" +"geoId/13127" +"geoId/13129" +"geoId/13131" +"geoId/13133" +"geoId/13135" +"geoId/13137" +"geoId/13139" +"geoId/13143" +"geoId/13145" +"geoId/13147" +"geoId/13151" +"geoId/13153" +"geoId/13157" +"geoId/13159" +"geoId/13163" +"geoId/13169" +"geoId/13171" +"geoId/13175" +"geoId/13177" +"geoId/13179" +"geoId/13183" +"geoId/13185" +"geoId/13187" +"geoId/13189" +"geoId/13195" +"geoId/13199" +"geoId/13205" +"geoId/13207" +"geoId/13211" +"geoId/13213" +"geoId/13215" +"geoId/13217" +"geoId/13219" +"geoId/13223" +"geoId/13225" +"geoId/13227" +"geoId/13229" +"geoId/13231" +"geoId/13233" +"geoId/13237" +"geoId/13241" +"geoId/13245" +"geoId/13247" +"geoId/13255" +"geoId/13257" +"geoId/13261" +"geoId/13267" +"geoId/13275" +"geoId/13277" +"geoId/13279" +"geoId/13285" +"geoId/13291" +"geoId/13293" +"geoId/13295" +"geoId/13297" +"geoId/13299" +"geoId/13303" +"geoId/13305" +"geoId/13311" +"geoId/13313" +"geoId/13321" +"geoId/15001" +"geoId/15003" +"geoId/15007" +"geoId/15009" +"geoId/16001" +"geoId/16005" +"geoId/16011" +"geoId/16013" +"geoId/16017" +"geoId/16019" +"geoId/16027" +"geoId/16031" +"geoId/16039" +"geoId/16045" +"geoId/16049" +"geoId/16051" +"geoId/16053" +"geoId/16055" +"geoId/16057" +"geoId/16065" +"geoId/16067" +"geoId/16069" +"geoId/16075" +"geoId/16079" +"geoId/16083" +"geoId/17001" +"geoId/17005" +"geoId/17007" +"geoId/17011" +"geoId/17015" +"geoId/17019" +"geoId/17021" +"geoId/17023" +"geoId/17027" +"geoId/17029" +"geoId/17031" +"geoId/17033" +"geoId/17037" +"geoId/17039" +"geoId/17041" +"geoId/17043" +"geoId/17045" +"geoId/17049" +"geoId/17051" +"geoId/17055" +"geoId/17057" +"geoId/17061" +"geoId/17063" +"geoId/17067" +"geoId/17073" +"geoId/17075" +"geoId/17077" +"geoId/17081" +"geoId/17083" +"geoId/17085" +"geoId/17089" +"geoId/17091" +"geoId/17093" +"geoId/17095" +"geoId/17097" +"geoId/17099" +"geoId/17101" +"geoId/17103" +"geoId/17105" +"geoId/17107" +"geoId/17109" +"geoId/17111" +"geoId/17113" +"geoId/17115" +"geoId/17117" +"geoId/17119" +"geoId/17121" +"geoId/17125" +"geoId/17131" +"geoId/17133" +"geoId/17135" +"geoId/17137" +"geoId/17141" +"geoId/17143" +"geoId/17145" +"geoId/17149" +"geoId/17157" +"geoId/17159" +"geoId/17161" +"geoId/17163" +"geoId/17165" +"geoId/17167" +"geoId/17173" +"geoId/17177" +"geoId/17179" +"geoId/17181" +"geoId/17183" +"geoId/17187" +"geoId/17191" +"geoId/17193" +"geoId/17195" +"geoId/17197" +"geoId/17199" +"geoId/17201" +"geoId/17203" +"geoId/18001" +"geoId/18003" +"geoId/18005" +"geoId/18011" +"geoId/18015" +"geoId/18017" +"geoId/18019" +"geoId/18021" +"geoId/18023" +"geoId/18027" +"geoId/18029" +"geoId/18031" +"geoId/18033" +"geoId/18035" +"geoId/18037" +"geoId/18039" +"geoId/18041" +"geoId/18043" +"geoId/18045" +"geoId/18047" +"geoId/18049" +"geoId/18051" +"geoId/18053" +"geoId/18055" +"geoId/18057" +"geoId/18059" +"geoId/18061" +"geoId/18063" +"geoId/18065" +"geoId/18067" +"geoId/18069" +"geoId/18071" +"geoId/18073" +"geoId/18075" +"geoId/18077" +"geoId/18079" +"geoId/18081" +"geoId/18083" +"geoId/18085" +"geoId/18087" +"geoId/18089" +"geoId/18091" +"geoId/18093" +"geoId/18095" +"geoId/18097" +"geoId/18099" +"geoId/18103" +"geoId/18105" +"geoId/18107" +"geoId/18109" +"geoId/18113" +"geoId/18117" +"geoId/18119" +"geoId/18121" +"geoId/18123" +"geoId/18127" +"geoId/18129" +"geoId/18133" +"geoId/18135" +"geoId/18137" +"geoId/18139" +"geoId/18141" +"geoId/18143" +"geoId/18145" +"geoId/18147" +"geoId/18149" +"geoId/18151" +"geoId/18153" +"geoId/18157" +"geoId/18163" +"geoId/18165" +"geoId/18167" +"geoId/18169" +"geoId/18173" +"geoId/18175" +"geoId/18177" +"geoId/18179" +"geoId/18181" +"geoId/18183" +"geoId/19011" +"geoId/19013" +"geoId/19015" +"geoId/19017" +"geoId/19019" +"geoId/19021" +"geoId/19023" +"geoId/19027" +"geoId/19029" +"geoId/19031" +"geoId/19033" +"geoId/19035" +"geoId/19041" +"geoId/19043" +"geoId/19045" +"geoId/19047" +"geoId/19049" +"geoId/19055" +"geoId/19057" +"geoId/19059" +"geoId/19061" +"geoId/19065" +"geoId/19067" +"geoId/19079" +"geoId/19083" +"geoId/19085" +"geoId/19087" +"geoId/19097" +"geoId/19099" +"geoId/19101" +"geoId/19103" +"geoId/19105" +"geoId/19109" +"geoId/19111" +"geoId/19113" +"geoId/19121" +"geoId/19123" +"geoId/19125" +"geoId/19127" +"geoId/19133" +"geoId/19139" +"geoId/19141" +"geoId/19145" +"geoId/19149" +"geoId/19153" +"geoId/19155" +"geoId/19157" +"geoId/19163" +"geoId/19167" +"geoId/19169" +"geoId/19171" +"geoId/19179" +"geoId/19181" +"geoId/19183" +"geoId/19187" +"geoId/19191" +"geoId/19193" +"geoId/19197" +"geoId/20005" +"geoId/20009" +"geoId/20015" +"geoId/20021" +"geoId/20035" +"geoId/20037" +"geoId/20041" +"geoId/20045" +"geoId/20051" +"geoId/20055" +"geoId/20057" +"geoId/20059" +"geoId/20061" +"geoId/20079" +"geoId/20087" +"geoId/20091" +"geoId/20099" +"geoId/20103" +"geoId/20111" +"geoId/20113" +"geoId/20121" +"geoId/20125" +"geoId/20133" +"geoId/20139" +"geoId/20149" +"geoId/20155" +"geoId/20161" +"geoId/20169" +"geoId/20173" +"geoId/20175" +"geoId/20177" +"geoId/20191" +"geoId/20209" +"geoId/21001" +"geoId/21003" +"geoId/21005" +"geoId/21009" +"geoId/21013" +"geoId/21015" +"geoId/21017" +"geoId/21019" +"geoId/21021" +"geoId/21025" +"geoId/21027" +"geoId/21029" +"geoId/21035" +"geoId/21037" +"geoId/21043" +"geoId/21047" +"geoId/21049" +"geoId/21051" +"geoId/21059" +"geoId/21067" +"geoId/21071" +"geoId/21073" +"geoId/21079" +"geoId/21081" +"geoId/21083" +"geoId/21085" +"geoId/21089" +"geoId/21093" +"geoId/21095" +"geoId/21097" +"geoId/21099" +"geoId/21101" +"geoId/21107" +"geoId/21111" +"geoId/21113" +"geoId/21115" +"geoId/21117" +"geoId/21119" +"geoId/21121" +"geoId/21125" +"geoId/21133" +"geoId/21137" +"geoId/21141" +"geoId/21145" +"geoId/21147" +"geoId/21151" +"geoId/21155" +"geoId/21157" +"geoId/21161" +"geoId/21163" +"geoId/21167" +"geoId/21173" +"geoId/21177" +"geoId/21179" +"geoId/21183" +"geoId/21185" +"geoId/21193" +"geoId/21195" +"geoId/21199" +"geoId/21203" +"geoId/21205" +"geoId/21207" +"geoId/21209" +"geoId/21211" +"geoId/21213" +"geoId/21215" +"geoId/21217" +"geoId/21225" +"geoId/21227" +"geoId/21231" +"geoId/21235" +"geoId/21239" +"geoId/22001" +"geoId/22003" +"geoId/22005" +"geoId/22007" +"geoId/22009" +"geoId/22011" +"geoId/22015" +"geoId/22017" +"geoId/22019" +"geoId/22027" +"geoId/22029" +"geoId/22031" +"geoId/22033" +"geoId/22037" +"geoId/22039" +"geoId/22041" +"geoId/22043" +"geoId/22045" +"geoId/22047" +"geoId/22049" +"geoId/22051" +"geoId/22053" +"geoId/22055" +"geoId/22057" +"geoId/22061" +"geoId/22063" +"geoId/22067" +"geoId/22069" +"geoId/22071" +"geoId/22073" +"geoId/22075" +"geoId/22077" +"geoId/22079" +"geoId/22083" +"geoId/22085" +"geoId/22087" +"geoId/22089" +"geoId/22093" +"geoId/22095" +"geoId/22097" +"geoId/22099" +"geoId/22101" +"geoId/22103" +"geoId/22105" +"geoId/22109" +"geoId/22111" +"geoId/22113" +"geoId/22115" +"geoId/22117" +"geoId/22119" +"geoId/22121" +"geoId/22127" +"geoId/23001" +"geoId/23003" +"geoId/23005" +"geoId/23007" +"geoId/23009" +"geoId/23011" +"geoId/23013" +"geoId/23015" +"geoId/23017" +"geoId/23019" +"geoId/23021" +"geoId/23023" +"geoId/23025" +"geoId/23027" +"geoId/23029" +"geoId/23031" +"geoId/24001" +"geoId/24003" +"geoId/24005" +"geoId/24009" +"geoId/24011" +"geoId/24013" +"geoId/24015" +"geoId/24017" +"geoId/24019" +"geoId/24021" +"geoId/24023" +"geoId/24025" +"geoId/24027" +"geoId/24029" +"geoId/24031" +"geoId/24033" +"geoId/24035" +"geoId/24037" +"geoId/24039" +"geoId/24041" +"geoId/24043" +"geoId/24045" +"geoId/24047" +"geoId/24510" +"geoId/25001" +"geoId/25003" +"geoId/25005" +"geoId/25007" +"geoId/25009" +"geoId/25011" +"geoId/25013" +"geoId/25015" +"geoId/25017" +"geoId/25021" +"geoId/25023" +"geoId/25025" +"geoId/25027" +"geoId/26005" +"geoId/26007" +"geoId/26009" +"geoId/26011" +"geoId/26015" +"geoId/26017" +"geoId/26019" +"geoId/26021" +"geoId/26023" +"geoId/26025" +"geoId/26027" +"geoId/26029" +"geoId/26031" +"geoId/26033" +"geoId/26035" +"geoId/26037" +"geoId/26041" +"geoId/26043" +"geoId/26045" +"geoId/26047" +"geoId/26049" +"geoId/26051" +"geoId/26053" +"geoId/26055" +"geoId/26057" +"geoId/26059" +"geoId/26061" +"geoId/26063" +"geoId/26065" +"geoId/26067" +"geoId/26069" +"geoId/26073" +"geoId/26075" +"geoId/26077" +"geoId/26079" +"geoId/26081" +"geoId/26087" +"geoId/26089" +"geoId/26091" +"geoId/26093" +"geoId/26099" +"geoId/26101" +"geoId/26103" +"geoId/26105" +"geoId/26107" +"geoId/26109" +"geoId/26111" +"geoId/26115" +"geoId/26117" +"geoId/26121" +"geoId/26123" +"geoId/26125" +"geoId/26127" +"geoId/26129" +"geoId/26133" +"geoId/26137" +"geoId/26139" +"geoId/26143" +"geoId/26145" +"geoId/26147" +"geoId/26149" +"geoId/26151" +"geoId/26155" +"geoId/26157" +"geoId/26159" +"geoId/26161" +"geoId/26163" +"geoId/26165" +"geoId/27003" +"geoId/27005" +"geoId/27007" +"geoId/27009" +"geoId/27013" +"geoId/27015" +"geoId/27017" +"geoId/27019" +"geoId/27021" +"geoId/27025" +"geoId/27027" +"geoId/27035" +"geoId/27037" +"geoId/27039" +"geoId/27041" +"geoId/27043" +"geoId/27045" +"geoId/27047" +"geoId/27049" +"geoId/27053" +"geoId/27055" +"geoId/27057" +"geoId/27059" +"geoId/27061" +"geoId/27067" +"geoId/27071" +"geoId/27079" +"geoId/27083" +"geoId/27085" +"geoId/27091" +"geoId/27093" +"geoId/27095" +"geoId/27097" +"geoId/27099" +"geoId/27103" +"geoId/27105" +"geoId/27109" +"geoId/27111" +"geoId/27115" +"geoId/27119" +"geoId/27123" +"geoId/27127" +"geoId/27129" +"geoId/27131" +"geoId/27137" +"geoId/27139" +"geoId/27141" +"geoId/27145" +"geoId/27147" +"geoId/27153" +"geoId/27157" +"geoId/27161" +"geoId/27163" +"geoId/27169" +"geoId/27171" +"geoId/28001" +"geoId/28003" +"geoId/28007" +"geoId/28011" +"geoId/28017" +"geoId/28023" +"geoId/28025" +"geoId/28027" +"geoId/28029" +"geoId/28031" +"geoId/28033" +"geoId/28035" +"geoId/28039" +"geoId/28043" +"geoId/28045" +"geoId/28047" +"geoId/28049" +"geoId/28051" +"geoId/28057" +"geoId/28059" +"geoId/28061" +"geoId/28067" +"geoId/28071" +"geoId/28073" +"geoId/28075" +"geoId/28079" +"geoId/28081" +"geoId/28083" +"geoId/28085" +"geoId/28087" +"geoId/28089" +"geoId/28091" +"geoId/28093" +"geoId/28095" +"geoId/28099" +"geoId/28101" +"geoId/28105" +"geoId/28107" +"geoId/28109" +"geoId/28113" +"geoId/28115" +"geoId/28117" +"geoId/28121" +"geoId/28123" +"geoId/28127" +"geoId/28131" +"geoId/28133" +"geoId/28135" +"geoId/28137" +"geoId/28139" +"geoId/28141" +"geoId/28145" +"geoId/28149" +"geoId/28151" +"geoId/28153" +"geoId/28159" +"geoId/28163" +"geoId/29001" +"geoId/29003" +"geoId/29007" +"geoId/29009" +"geoId/29013" +"geoId/29015" +"geoId/29019" +"geoId/29021" +"geoId/29023" +"geoId/29027" +"geoId/29029" +"geoId/29031" +"geoId/29037" +"geoId/29043" +"geoId/29047" +"geoId/29049" +"geoId/29051" +"geoId/29053" +"geoId/29055" +"geoId/29059" +"geoId/29069" +"geoId/29071" +"geoId/29077" +"geoId/29083" +"geoId/29091" +"geoId/29095" +"geoId/29097" +"geoId/29099" +"geoId/29101" +"geoId/29105" +"geoId/29107" +"geoId/29109" +"geoId/29113" +"geoId/29119" +"geoId/29127" +"geoId/29131" +"geoId/29141" +"geoId/29143" +"geoId/29145" +"geoId/29147" +"geoId/29155" +"geoId/29157" +"geoId/29159" +"geoId/29161" +"geoId/29163" +"geoId/29165" +"geoId/29167" +"geoId/29169" +"geoId/29175" +"geoId/29177" +"geoId/29183" +"geoId/29186" +"geoId/29187" +"geoId/29189" +"geoId/29195" +"geoId/29201" +"geoId/29207" +"geoId/29209" +"geoId/29213" +"geoId/29215" +"geoId/29217" +"geoId/29219" +"geoId/29221" +"geoId/29225" +"geoId/29229" +"geoId/29510" +"geoId/30013" +"geoId/30029" +"geoId/30031" +"geoId/30041" +"geoId/30047" +"geoId/30049" +"geoId/30053" +"geoId/30063" +"geoId/30067" +"geoId/30081" +"geoId/30093" +"geoId/30111" +"geoId/31001" +"geoId/31019" +"geoId/31025" +"geoId/31043" +"geoId/31047" +"geoId/31053" +"geoId/31055" +"geoId/31067" +"geoId/31079" +"geoId/31109" +"geoId/31111" +"geoId/31119" +"geoId/31141" +"geoId/31153" +"geoId/31155" +"geoId/31157" +"geoId/31159" +"geoId/31177" +"geoId/32001" +"geoId/32003" +"geoId/32005" +"geoId/32007" +"geoId/32013" +"geoId/32019" +"geoId/32023" +"geoId/32031" +"geoId/32510" +"geoId/33001" +"geoId/33003" +"geoId/33005" +"geoId/33007" +"geoId/33009" +"geoId/33011" +"geoId/33013" +"geoId/33015" +"geoId/33017" +"geoId/33019" +"geoId/34001" +"geoId/34003" +"geoId/34005" +"geoId/34007" +"geoId/34009" +"geoId/34011" +"geoId/34013" +"geoId/34015" +"geoId/34017" +"geoId/34019" +"geoId/34021" +"geoId/34023" +"geoId/34025" +"geoId/34027" +"geoId/34029" +"geoId/34031" +"geoId/34033" +"geoId/34035" +"geoId/34037" +"geoId/34039" +"geoId/34041" +"geoId/35001" +"geoId/35005" +"geoId/35006" +"geoId/35009" +"geoId/35013" +"geoId/35015" +"geoId/35017" +"geoId/35025" +"geoId/35027" +"geoId/35028" +"geoId/35029" +"geoId/35031" +"geoId/35035" +"geoId/35039" +"geoId/35041" +"geoId/35043" +"geoId/35045" +"geoId/35047" +"geoId/35049" +"geoId/35053" +"geoId/35055" +"geoId/35057" +"geoId/35061" +"geoId/36001" +"geoId/36003" +"geoId/36005" +"geoId/36007" +"geoId/36009" +"geoId/36011" +"geoId/36013" +"geoId/36015" +"geoId/36017" +"geoId/36019" +"geoId/36021" +"geoId/36023" +"geoId/36025" +"geoId/36027" +"geoId/36029" +"geoId/36031" +"geoId/36033" +"geoId/36035" +"geoId/36037" +"geoId/36039" +"geoId/36043" +"geoId/36045" +"geoId/36047" +"geoId/36049" +"geoId/36051" +"geoId/36053" +"geoId/36055" +"geoId/36057" +"geoId/36059" +"geoId/36061" +"geoId/36063" +"geoId/36065" +"geoId/36067" +"geoId/36069" +"geoId/36071" +"geoId/36073" +"geoId/36075" +"geoId/36077" +"geoId/36079" +"geoId/36081" +"geoId/36083" +"geoId/36085" +"geoId/36087" +"geoId/36089" +"geoId/36091" +"geoId/36093" +"geoId/36095" +"geoId/36097" +"geoId/36099" +"geoId/36101" +"geoId/36103" +"geoId/36105" +"geoId/36107" +"geoId/36109" +"geoId/36111" +"geoId/36113" +"geoId/36115" +"geoId/36117" +"geoId/36119" +"geoId/36121" +"geoId/36123" +"geoId/37001" +"geoId/37003" +"geoId/37007" +"geoId/37009" +"geoId/37011" +"geoId/37013" +"geoId/37015" +"geoId/37017" +"geoId/37019" +"geoId/37021" +"geoId/37023" +"geoId/37025" +"geoId/37027" +"geoId/37031" +"geoId/37033" +"geoId/37035" +"geoId/37037" +"geoId/37039" +"geoId/37045" +"geoId/37047" +"geoId/37049" +"geoId/37051" +"geoId/37053" +"geoId/37055" +"geoId/37057" +"geoId/37059" +"geoId/37061" +"geoId/37063" +"geoId/37065" +"geoId/37067" +"geoId/37069" +"geoId/37071" +"geoId/37077" +"geoId/37079" +"geoId/37081" +"geoId/37083" +"geoId/37085" +"geoId/37087" +"geoId/37089" +"geoId/37091" +"geoId/37093" +"geoId/37097" +"geoId/37099" +"geoId/37101" +"geoId/37105" +"geoId/37107" +"geoId/37109" +"geoId/37111" +"geoId/37113" +"geoId/37115" +"geoId/37117" +"geoId/37119" +"geoId/37123" +"geoId/37125" +"geoId/37127" +"geoId/37129" +"geoId/37131" +"geoId/37133" +"geoId/37135" +"geoId/37139" +"geoId/37141" +"geoId/37145" +"geoId/37147" +"geoId/37149" +"geoId/37151" +"geoId/37153" +"geoId/37155" +"geoId/37157" +"geoId/37159" +"geoId/37161" +"geoId/37163" +"geoId/37165" +"geoId/37167" +"geoId/37169" +"geoId/37171" +"geoId/37175" +"geoId/37179" +"geoId/37181" +"geoId/37183" +"geoId/37185" +"geoId/37189" +"geoId/37191" +"geoId/37193" +"geoId/37195" +"geoId/37197" +"geoId/37199" +"geoId/38015" +"geoId/38017" +"geoId/38035" +"geoId/38059" +"geoId/38077" +"geoId/38089" +"geoId/38093" +"geoId/38101" +"geoId/38105" +"geoId/39001" +"geoId/39003" +"geoId/39005" +"geoId/39007" +"geoId/39009" +"geoId/39011" +"geoId/39013" +"geoId/39015" +"geoId/39017" +"geoId/39019" +"geoId/39021" +"geoId/39023" +"geoId/39025" +"geoId/39027" +"geoId/39029" +"geoId/39031" +"geoId/39033" +"geoId/39035" +"geoId/39037" +"geoId/39039" +"geoId/39041" +"geoId/39043" +"geoId/39045" +"geoId/39047" +"geoId/39049" +"geoId/39051" +"geoId/39053" +"geoId/39055" +"geoId/39057" +"geoId/39059" +"geoId/39061" +"geoId/39063" +"geoId/39065" +"geoId/39067" +"geoId/39069" +"geoId/39071" +"geoId/39073" +"geoId/39075" +"geoId/39077" +"geoId/39079" +"geoId/39081" +"geoId/39083" +"geoId/39085" +"geoId/39087" +"geoId/39089" +"geoId/39091" +"geoId/39093" +"geoId/39095" +"geoId/39097" +"geoId/39099" +"geoId/39101" +"geoId/39103" +"geoId/39105" +"geoId/39107" +"geoId/39109" +"geoId/39111" +"geoId/39113" +"geoId/39117" +"geoId/39119" +"geoId/39123" +"geoId/39125" +"geoId/39127" +"geoId/39129" +"geoId/39131" +"geoId/39133" +"geoId/39135" +"geoId/39137" +"geoId/39139" +"geoId/39141" +"geoId/39143" +"geoId/39145" +"geoId/39147" +"geoId/39149" +"geoId/39151" +"geoId/39153" +"geoId/39155" +"geoId/39157" +"geoId/39159" +"geoId/39161" +"geoId/39165" +"geoId/39167" +"geoId/39169" +"geoId/39171" +"geoId/39173" +"geoId/39175" +"geoId/40001" +"geoId/40009" +"geoId/40013" +"geoId/40015" +"geoId/40017" +"geoId/40019" +"geoId/40021" +"geoId/40023" +"geoId/40027" +"geoId/40031" +"geoId/40037" +"geoId/40039" +"geoId/40041" +"geoId/40047" +"geoId/40049" +"geoId/40051" +"geoId/40065" +"geoId/40071" +"geoId/40079" +"geoId/40081" +"geoId/40083" +"geoId/40087" +"geoId/40089" +"geoId/40091" +"geoId/40095" +"geoId/40097" +"geoId/40101" +"geoId/40109" +"geoId/40111" +"geoId/40113" +"geoId/40115" +"geoId/40119" +"geoId/40121" +"geoId/40123" +"geoId/40125" +"geoId/40131" +"geoId/40133" +"geoId/40135" +"geoId/40137" +"geoId/40139" +"geoId/40143" +"geoId/40145" +"geoId/40147" +"geoId/40149" +"geoId/40153" +"geoId/41003" +"geoId/41005" +"geoId/41007" +"geoId/41009" +"geoId/41011" +"geoId/41013" +"geoId/41015" +"geoId/41017" +"geoId/41019" +"geoId/41027" +"geoId/41029" +"geoId/41031" +"geoId/41033" +"geoId/41035" +"geoId/41039" +"geoId/41041" +"geoId/41043" +"geoId/41045" +"geoId/41047" +"geoId/41051" +"geoId/41053" +"geoId/41057" +"geoId/41059" +"geoId/41061" +"geoId/41065" +"geoId/41067" +"geoId/41071" +"geoId/42001" +"geoId/42003" +"geoId/42005" +"geoId/42007" +"geoId/42009" +"geoId/42011" +"geoId/42013" +"geoId/42015" +"geoId/42017" +"geoId/42019" +"geoId/42021" +"geoId/42025" +"geoId/42027" +"geoId/42029" +"geoId/42031" +"geoId/42033" +"geoId/42035" +"geoId/42037" +"geoId/42039" +"geoId/42041" +"geoId/42043" +"geoId/42045" +"geoId/42047" +"geoId/42049" +"geoId/42051" +"geoId/42055" +"geoId/42059" +"geoId/42061" +"geoId/42063" +"geoId/42065" +"geoId/42067" +"geoId/42069" +"geoId/42071" +"geoId/42073" +"geoId/42075" +"geoId/42077" +"geoId/42079" +"geoId/42081" +"geoId/42083" +"geoId/42085" +"geoId/42087" +"geoId/42089" +"geoId/42091" +"geoId/42093" +"geoId/42095" +"geoId/42097" +"geoId/42099" +"geoId/42101" +"geoId/42103" +"geoId/42105" +"geoId/42107" +"geoId/42109" +"geoId/42111" +"geoId/42115" +"geoId/42117" +"geoId/42119" +"geoId/42121" +"geoId/42123" +"geoId/42125" +"geoId/42127" +"geoId/42129" +"geoId/42131" +"geoId/42133" +"geoId/44001" +"geoId/44003" +"geoId/44005" +"geoId/44007" +"geoId/44009" +"geoId/45001" +"geoId/45003" +"geoId/45007" +"geoId/45009" +"geoId/45011" +"geoId/45013" +"geoId/45015" +"geoId/45019" +"geoId/45021" +"geoId/45023" +"geoId/45025" +"geoId/45027" +"geoId/45029" +"geoId/45031" +"geoId/45033" +"geoId/45035" +"geoId/45037" +"geoId/45039" +"geoId/45041" +"geoId/45043" +"geoId/45045" +"geoId/45047" +"geoId/45049" +"geoId/45051" +"geoId/45053" +"geoId/45055" +"geoId/45057" +"geoId/45059" +"geoId/45061" +"geoId/45063" +"geoId/45067" +"geoId/45069" +"geoId/45071" +"geoId/45073" +"geoId/45075" +"geoId/45077" +"geoId/45079" +"geoId/45081" +"geoId/45083" +"geoId/45085" +"geoId/45087" +"geoId/45089" +"geoId/45091" +"geoId/46005" +"geoId/46011" +"geoId/46013" +"geoId/46029" +"geoId/46035" +"geoId/46065" +"geoId/46081" +"geoId/46083" +"geoId/46093" +"geoId/46099" +"geoId/46103" +"geoId/46127" +"geoId/46135" +"geoId/47001" +"geoId/47003" +"geoId/47009" +"geoId/47011" +"geoId/47013" +"geoId/47017" +"geoId/47019" +"geoId/47021" +"geoId/47023" +"geoId/47025" +"geoId/47029" +"geoId/47031" +"geoId/47035" +"geoId/47037" +"geoId/47041" +"geoId/47043" +"geoId/47045" +"geoId/47047" +"geoId/47049" +"geoId/47051" +"geoId/47053" +"geoId/47055" +"geoId/47057" +"geoId/47059" +"geoId/47063" +"geoId/47065" +"geoId/47069" +"geoId/47071" +"geoId/47073" +"geoId/47075" +"geoId/47077" +"geoId/47079" +"geoId/47081" +"geoId/47085" +"geoId/47089" +"geoId/47091" +"geoId/47093" +"geoId/47097" +"geoId/47099" +"geoId/47103" +"geoId/47105" +"geoId/47107" +"geoId/47109" +"geoId/47111" +"geoId/47113" +"geoId/47115" +"geoId/47117" +"geoId/47119" +"geoId/47123" +"geoId/47125" +"geoId/47129" +"geoId/47131" +"geoId/47133" +"geoId/47139" +"geoId/47141" +"geoId/47143" +"geoId/47145" +"geoId/47147" +"geoId/47149" +"geoId/47151" +"geoId/47153" +"geoId/47155" +"geoId/47157" +"geoId/47159" +"geoId/47163" +"geoId/47165" +"geoId/47167" +"geoId/47171" +"geoId/47173" +"geoId/47177" +"geoId/47179" +"geoId/47181" +"geoId/47183" +"geoId/47185" +"geoId/47187" +"geoId/47189" +"geoId/48001" +"geoId/48003" +"geoId/48005" +"geoId/48007" +"geoId/48013" +"geoId/48015" +"geoId/48019" +"geoId/48021" +"geoId/48025" +"geoId/48027" +"geoId/48029" +"geoId/48035" +"geoId/48037" +"geoId/48039" +"geoId/48041" +"geoId/48049" +"geoId/48051" +"geoId/48053" +"geoId/48055" +"geoId/48057" +"geoId/48061" +"geoId/48067" +"geoId/48071" +"geoId/48073" +"geoId/48085" +"geoId/48089" +"geoId/48091" +"geoId/48097" +"geoId/48099" +"geoId/48113" +"geoId/48117" +"geoId/48121" +"geoId/48123" +"geoId/48133" +"geoId/48135" +"geoId/48139" +"geoId/48141" +"geoId/48143" +"geoId/48145" +"geoId/48147" +"geoId/48149" +"geoId/48157" +"geoId/48161" +"geoId/48163" +"geoId/48165" +"geoId/48167" +"geoId/48171" +"geoId/48177" +"geoId/48179" +"geoId/48181" +"geoId/48183" +"geoId/48185" +"geoId/48187" +"geoId/48189" +"geoId/48199" +"geoId/48201" +"geoId/48203" +"geoId/48209" +"geoId/48213" +"geoId/48215" +"geoId/48217" +"geoId/48219" +"geoId/48221" +"geoId/48223" +"geoId/48225" +"geoId/48227" +"geoId/48231" +"geoId/48233" +"geoId/48241" +"geoId/48245" +"geoId/48249" +"geoId/48251" +"geoId/48253" +"geoId/48257" +"geoId/48259" +"geoId/48265" +"geoId/48273" +"geoId/48277" +"geoId/48279" +"geoId/48281" +"geoId/48285" +"geoId/48287" +"geoId/48289" +"geoId/48291" +"geoId/48293" +"geoId/48299" +"geoId/48303" +"geoId/48309" +"geoId/48321" +"geoId/48323" +"geoId/48325" +"geoId/48329" +"geoId/48331" +"geoId/48337" +"geoId/48339" +"geoId/48341" +"geoId/48347" +"geoId/48349" +"geoId/48353" +"geoId/48355" +"geoId/48361" +"geoId/48363" +"geoId/48365" +"geoId/48367" +"geoId/48371" +"geoId/48373" +"geoId/48375" +"geoId/48381" +"geoId/48389" +"geoId/48395" +"geoId/48397" +"geoId/48401" +"geoId/48407" +"geoId/48409" +"geoId/48415" +"geoId/48419" +"geoId/48423" +"geoId/48427" +"geoId/48439" +"geoId/48441" +"geoId/48449" +"geoId/48451" +"geoId/48453" +"geoId/48457" +"geoId/48459" +"geoId/48463" +"geoId/48465" +"geoId/48467" +"geoId/48469" +"geoId/48471" +"geoId/48473" +"geoId/48477" +"geoId/48479" +"geoId/48481" +"geoId/48485" +"geoId/48489" +"geoId/48491" +"geoId/48493" +"geoId/48497" +"geoId/48499" +"geoId/48503" +"geoId/49003" +"geoId/49005" +"geoId/49007" +"geoId/49011" +"geoId/49013" +"geoId/49021" +"geoId/49035" +"geoId/49039" +"geoId/49041" +"geoId/49043" +"geoId/49045" +"geoId/49047" +"geoId/49049" +"geoId/49051" +"geoId/49053" +"geoId/49057" +"geoId/50001" +"geoId/50003" +"geoId/50005" +"geoId/50007" +"geoId/50011" +"geoId/50015" +"geoId/50017" +"geoId/50019" +"geoId/50021" +"geoId/50023" +"geoId/50025" +"geoId/50027" +"geoId/51001" +"geoId/51003" +"geoId/51005" +"geoId/51009" +"geoId/51013" +"geoId/51015" +"geoId/51019" +"geoId/51023" +"geoId/51025" +"geoId/51027" +"geoId/51029" +"geoId/51031" +"geoId/51033" +"geoId/51035" +"geoId/51041" +"geoId/51047" +"geoId/51051" +"geoId/51053" +"geoId/51059" +"geoId/51061" +"geoId/51065" +"geoId/51067" +"geoId/51069" +"geoId/51071" +"geoId/51073" +"geoId/51075" +"geoId/51077" +"geoId/51079" +"geoId/51083" +"geoId/51085" +"geoId/51087" +"geoId/51089" +"geoId/51093" +"geoId/51095" +"geoId/51099" +"geoId/51101" +"geoId/51105" +"geoId/51107" +"geoId/51109" +"geoId/51117" +"geoId/51121" +"geoId/51127" +"geoId/51137" +"geoId/51139" +"geoId/51141" +"geoId/51143" +"geoId/51145" +"geoId/51147" +"geoId/51149" +"geoId/51153" +"geoId/51155" +"geoId/51161" +"geoId/51163" +"geoId/51165" +"geoId/51167" +"geoId/51169" +"geoId/51171" +"geoId/51173" +"geoId/51175" +"geoId/51177" +"geoId/51179" +"geoId/51185" +"geoId/51187" +"geoId/51191" +"geoId/51193" +"geoId/51195" +"geoId/51197" +"geoId/51199" +"geoId/51510" +"geoId/51520" +"geoId/51540" +"geoId/51550" +"geoId/51570" +"geoId/51590" +"geoId/51600" +"geoId/51630" +"geoId/51650" +"geoId/51660" +"geoId/51670" +"geoId/51680" +"geoId/51683" +"geoId/51685" +"geoId/51690" +"geoId/51700" +"geoId/51710" +"geoId/51730" +"geoId/51740" +"geoId/51750" +"geoId/51760" +"geoId/51770" +"geoId/51775" +"geoId/51790" +"geoId/51800" +"geoId/51810" +"geoId/51820" +"geoId/51840" +"geoId/53001" +"geoId/53003" +"geoId/53005" +"geoId/53007" +"geoId/53009" +"geoId/53011" +"geoId/53015" +"geoId/53017" +"geoId/53021" +"geoId/53025" +"geoId/53027" +"geoId/53029" +"geoId/53031" +"geoId/53033" +"geoId/53035" +"geoId/53037" +"geoId/53039" +"geoId/53041" +"geoId/53045" +"geoId/53047" +"geoId/53049" +"geoId/53053" +"geoId/53055" +"geoId/53057" +"geoId/53061" +"geoId/53063" +"geoId/53065" +"geoId/53067" +"geoId/53071" +"geoId/53073" +"geoId/53075" +"geoId/53077" +"geoId/54003" +"geoId/54005" +"geoId/54009" +"geoId/54011" +"geoId/54019" +"geoId/54025" +"geoId/54027" +"geoId/54029" +"geoId/54033" +"geoId/54035" +"geoId/54037" +"geoId/54039" +"geoId/54041" +"geoId/54043" +"geoId/54045" +"geoId/54047" +"geoId/54049" +"geoId/54051" +"geoId/54053" +"geoId/54055" +"geoId/54057" +"geoId/54059" +"geoId/54061" +"geoId/54065" +"geoId/54067" +"geoId/54069" +"geoId/54077" +"geoId/54079" +"geoId/54081" +"geoId/54083" +"geoId/54091" +"geoId/54097" +"geoId/54099" +"geoId/54103" +"geoId/54107" +"geoId/54109" +"geoId/55001" +"geoId/55003" +"geoId/55005" +"geoId/55009" +"geoId/55013" +"geoId/55015" +"geoId/55017" +"geoId/55019" +"geoId/55021" +"geoId/55023" +"geoId/55025" +"geoId/55027" +"geoId/55029" +"geoId/55031" +"geoId/55033" +"geoId/55035" +"geoId/55039" +"geoId/55043" +"geoId/55045" +"geoId/55047" +"geoId/55049" +"geoId/55053" +"geoId/55055" +"geoId/55057" +"geoId/55059" +"geoId/55061" +"geoId/55063" +"geoId/55065" +"geoId/55067" +"geoId/55069" +"geoId/55071" +"geoId/55073" +"geoId/55075" +"geoId/55079" +"geoId/55081" +"geoId/55083" +"geoId/55085" +"geoId/55087" +"geoId/55089" +"geoId/55093" +"geoId/55095" +"geoId/55097" +"geoId/55101" +"geoId/55103" +"geoId/55105" +"geoId/55109" +"geoId/55111" +"geoId/55113" +"geoId/55115" +"geoId/55117" +"geoId/55119" +"geoId/55121" +"geoId/55123" +"geoId/55125" +"geoId/55127" +"geoId/55131" +"geoId/55133" +"geoId/55135" +"geoId/55137" +"geoId/55139" +"geoId/55141" +"geoId/56001" +"geoId/56005" +"geoId/56007" +"geoId/56013" +"geoId/56021" +"geoId/56023" +"geoId/56025" +"geoId/56029" +"geoId/56033" +"geoId/56037" +"geoId/56039" +"geoId/56041" diff --git a/statvar_imports/us_cdc/county_mortality/golden_data/golden_summary_report.csv b/statvar_imports/us_cdc/county_mortality/golden_data/golden_summary_report.csv new file mode 100644 index 0000000000..f0fcaf4892 --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/golden_data/golden_summary_report.csv @@ -0,0 +1,3 @@ +"StatVar","MeasurementMethods","MinDate","observationPeriods","NumPlaces","Units","ScalingFactors" +"Count_MortalityEvent_Assault(Homicide)","[]","2018","[P1Y]","576","[]","[]" +"Count_MortalityEvent_Suicide","[]","2018","[P1Y]","1421","[]","[]" diff --git a/statvar_imports/us_cdc/county_mortality/manifest.json b/statvar_imports/us_cdc/county_mortality/manifest.json new file mode 100644 index 0000000000..6cbc5793a0 --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/manifest.json @@ -0,0 +1,30 @@ +{ + "import_specifications": [ + { + "import_name": "CDC_Mortality_Count", + "curator_emails": [ + "support@datacommons.org" + ], + "provenance_url": "https://wonder.cdc.gov/ucd-icd10-expanded.html", + "provenance_description": "County-level mortality statistics for all causes of death (ICD-10 113 Cause List) across all US states from CDC WONDER.", + "scripts": [ + "download.py", + "../../../tools/statvar_importer/stat_var_processor.py --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --input_data=input_files/*.csv --pv_map=county_mortality_pvmap.csv --config_file=county_mortality_metadata.csv --output_path=output/underlyingcauseofdeath_county --output_counters=counters/underlyingcauseofdeath_county_counters.csv" + ], + "import_inputs": [ + { + "template_mcf": "output/underlyingcauseofdeath_county.tmcf", + "cleaned_csv": "output/underlyingcauseofdeath_county.csv", + "node_mcf": "output/*.mcf" + } + ], + "source_files": [ + "input_files/*.csv", + "golden_data/*.csv", + "counters/*.csv" + ], + "cron_schedule": "0 5 * * 5", + "validation_config_file": "validation_config.json" + } + ] +} \ No newline at end of file diff --git a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_input.csv b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_input.csv new file mode 100644 index 0000000000..504fe6f140 --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_input.csv @@ -0,0 +1,71 @@ +Notes,Year,Year Code,County,County Code,ICD-10 113 Cause List,ICD-10 113 Cause List Code,Deaths,Population,Crude Rate,Crude Rate Lower 95% Confidence Interval,Crude Rate Upper 95% Confidence Interval +,2023,2023,"Kent County, DE",10001,#Septicemia (A40-A41),GR113-010,26,189789,13.7,8.9,20.1 +,2023,2023,"Kent County, DE",10001,"Other and unspecified infectious and parasitic diseases and their sequelae (A00,A05,A20-A36,A42-A44,A48-A49,A54-A79,A81-A82,A85.0-A85.1,A85.8,A86-B04,B06-B09,B25-B49,B55-B99,U07.1)",GR113-018,42,189789,22.1,15.9,29.9 +,2023,2023,"Kent County, DE",10001,#Malignant neoplasms (C00-C97),GR113-019,407,189789,214.4,193.6,235.3 +,2023,2023,"Kent County, DE",10001,"Malignant neoplasms of colon, rectum and anus (C18-C21)",GR113-023,28,189789,14.8,9.8,21.3 +,2023,2023,"Kent County, DE",10001,Malignant neoplasms of liver and intrahepatic bile ducts (C22),GR113-024,18,189789,9.5,5.6,15.0 +,2023,2023,"Kent County, DE",10001,Malignant neoplasm of pancreas (C25),GR113-025,31,189789,16.3,11.1,23.2 +,2023,2023,"Kent County, DE",10001,"Malignant neoplasms of trachea, bronchus and lung (C33-C34)",GR113-027,100,189789,52.7,42.4,63.0 +,2023,2023,"Kent County, DE",10001,Malignant neoplasm of breast (C50),GR113-029,36,189789,19.0,13.3,26.3 +,2023,2023,"Kent County, DE",10001,Malignant neoplasm of prostate (C61),GR113-033,14,189789,7.4,4.0,12.4 +,2023,2023,"Kent County, DE",10001,Malignant neoplasm of bladder (C67),GR113-035,20,189789,10.5,6.4,16.3 +,2023,2023,"Kent County, DE",10001,"Malignant neoplasms of meninges, brain and other parts of central nervous system (C70-C72)",GR113-036,10,189789,5.3,2.5,9.7 +,2023,2023,"Kent County, DE",10001,"Malignant neoplasms of lymphoid, hematopoietic and related tissue (C81-C96)",GR113-037,38,189789,20.0,14.2,27.5 +,2023,2023,"Kent County, DE",10001,Non-Hodgkin lymphoma (C82-C85),GR113-039,15,189789,7.9,4.4,13.0 +,2023,2023,"Kent County, DE",10001,Leukemia (C91-C95),GR113-040,18,189789,9.5,5.6,15.0 +,2023,2023,"Kent County, DE",10001,"All other and unspecified malignant neoplasms (C17,C23-C24,C26-C31,C37-C41,C44-C49,C51-C52,C57-C60,C62-C63,C66,C68-C69,C73-C80,C97)",GR113-043,57,189789,30.0,22.7,38.9 +,2023,2023,"Kent County, DE",10001,"#In situ neoplasms, benign neoplasms and neoplasms of uncertain or unknown behavior (D00-D48)",GR113-044,12,189789,6.3,3.3,11.0 +,2023,2023,"Kent County, DE",10001,#Diabetes mellitus (E10-E14),GR113-046,78,189789,41.1,32.5,51.3 +,2023,2023,"Kent County, DE",10001,#Nutritional deficiencies (E40-E64),GR113-047,11,189789,5.8,2.9,10.4 +,2023,2023,"Kent County, DE",10001,Malnutrition (E40-E46),GR113-048,11,189789,5.8,2.9,10.4 +,2023,2023,"Kent County, DE",10001,#Parkinson disease (G20-G21),GR113-051,16,189789,8.4,4.8,13.7 +,2023,2023,"Kent County, DE",10001,#Alzheimer disease (G30),GR113-052,72,189789,37.9,29.7,47.8 +,2023,2023,"Kent County, DE",10001,Major cardiovascular diseases (I00-I78),GR113-053,612,189789,322.5,296.9,348.0 +,2023,2023,"Kent County, DE",10001,"#Diseases of heart (I00-I09,I11,I13,I20-I51)",GR113-054,416,189789,219.2,198.1,240.3 +,2023,2023,"Kent County, DE",10001,Hypertensive heart disease (I11),GR113-056,51,189789,26.9,20.0,35.3 +,2023,2023,"Kent County, DE",10001,Ischemic heart diseases (I20-I25),GR113-058,223,189789,117.5,102.1,132.9 +,2023,2023,"Kent County, DE",10001,Acute myocardial infarction (I21-I22),GR113-059,33,189789,17.4,12.0,24.4 +,2023,2023,"Kent County, DE",10001,"Other forms of chronic ischemic heart disease (I20,I25)",GR113-061,187,189789,98.5,84.4,112.7 +,2023,2023,"Kent County, DE",10001,"Atherosclerotic cardiovascular disease, so described (I25.0)",GR113-062,67,189789,35.3,27.4,44.8 +,2023,2023,"Kent County, DE",10001,"All other forms of chronic ischemic heart disease (I20,I25.1-I25.9)",GR113-063,120,189789,63.2,51.9,74.5 +,2023,2023,"Kent County, DE",10001,Other heart diseases (I26-I51),GR113-064,128,189789,67.4,55.8,79.1 +,2023,2023,"Kent County, DE",10001,Heart failure (I50),GR113-067,37,189789,19.5,13.7,26.9 +,2023,2023,"Kent County, DE",10001,"All other forms of heart disease (I26-I28,I34-I38,I42-I49,I51)",GR113-068,91,189789,47.9,38.6,58.9 +,2023,2023,"Kent County, DE",10001,"#Essential hypertension and hypertensive renal disease (I10,I12,I15)",GR113-069,17,189789,9.0,5.2,14.3 +,2023,2023,"Kent County, DE",10001,#Cerebrovascular diseases (I60-I69),GR113-070,167,189789,88.0,74.6,101.3 +,2023,2023,"Kent County, DE",10001,Other diseases of circulatory system (I71-I78),GR113-072,11,189789,5.8,2.9,10.4 +,2024 ,2024,"Kent County, DE",10001,#Septicemia (A40-A41),GR113-010,37,192690,19.2,13.5,26.5 +,2024 ,2024,"Kent County, DE",10001,"Other and unspecified infectious and parasitic diseases and their sequelae (A00,A05,A20-A36,A42-A44,A48-A49,A54-A79,A81-A82,A85.0-A85.1,A85.8,A86-B04,B06-B09,B25-B49,B55-B99,U07.1)",GR113-018,38,192690,19.7,14.0,27.1 +,2024 ,2024,"Kent County, DE",10001,#Malignant neoplasms (C00-C97),GR113-019,420,192690,218.0,197.1,238.8 +,2024 ,2024,"Kent County, DE",10001,"Malignant neoplasms of lip, oral cavity and pharynx (C00-C14)",GR113-020,12,192690,6.2,3.2,10.9 +,2024 ,2024,"Kent County, DE",10001,Malignant neoplasm of esophagus (C15),GR113-021,11,192690,5.7,2.8,10.2 +,2024 ,2024,"Kent County, DE",10001,"Malignant neoplasms of colon, rectum and anus (C18-C21)",GR113-023,32,192690,16.6,11.4,23.4 +,2024 ,2024,"Kent County, DE",10001,Malignant neoplasms of liver and intrahepatic bile ducts (C22),GR113-024,20,192690,10.4,6.3,16.0 +,2024 ,2024,"Kent County, DE",10001,Malignant neoplasm of pancreas (C25),GR113-025,38,192690,19.7,14.0,27.1 +,2024 ,2024,"Kent County, DE",10001,"Malignant neoplasms of trachea, bronchus and lung (C33-C34)",GR113-027,96,192690,49.8,40.4,60.8 +,2024 ,2024,"Kent County, DE",10001,Malignant neoplasm of breast (C50),GR113-029,29,192690,15.1,10.1,21.6 +,2024 ,2024,"Kent County, DE",10001,"Malignant neoplasms of corpus uteri and uterus, part unspecified (C54-C55)",GR113-031,13,192690,6.7,3.6,11.5 +,2024 ,2024,"Kent County, DE",10001,Malignant neoplasm of prostate (C61),GR113-033,12,192690,6.2,3.2,10.9 +,2024 ,2024,"Kent County, DE",10001,Malignant neoplasms of kidney and renal pelvis (C64-C65),GR113-034,11,192690,5.7,2.8,10.2 +,2024 ,2024,"Kent County, DE",10001,Malignant neoplasm of bladder (C67),GR113-035,12,192690,6.2,3.2,10.9 +,2024 ,2024,"Kent County, DE",10001,"Malignant neoplasms of meninges, brain and other parts of central nervous system (C70-C72)",GR113-036,11,192690,5.7,2.8,10.2 +,2024 ,2024,"Kent County, DE",10001,"Malignant neoplasms of lymphoid, hematopoietic and related tissue (C81-C96)",GR113-037,46,192690,23.9,17.5,31.8 +,2024 ,2024,"Kent County, DE",10001,Non-Hodgkin lymphoma (C82-C85),GR113-039,16,192690,8.3,4.7,13.5 +,2024 ,2024,"Kent County, DE",10001,Leukemia (C91-C95),GR113-040,19,192690,9.9,5.9,15.4 +,2024 ,2024,"Kent County, DE",10001,"Multiple myeloma and immunoproliferative neoplasms (C88,C90)",GR113-041,11,192690,5.7,2.8,10.2 +,2024 ,2024,"Kent County, DE",10001,"All other and unspecified malignant neoplasms (C17,C23-C24,C26-C31,C37-C41,C44-C49,C51-C52,C57-C60,C62-C63,C66,C68-C69,C73-C80,C97)",GR113-043,50,192690,25.9,19.3,34.2 +,2024 ,2024,"Kent County, DE",10001,"#In situ neoplasms, benign neoplasms and neoplasms of uncertain or unknown behavior (D00-D48)",GR113-044,14,192690,7.3,4.0,12.2 +,2024 ,2024,"Kent County, DE",10001,#Diabetes mellitus (E10-E14),GR113-046,75,192690,38.9,30.6,48.8 +,2024 ,2024,"Kent County, DE",10001,#Parkinson disease (G20-G21),GR113-051,30,192690,15.6,10.5,22.2 +,2024 ,2024,"Kent County, DE",10001,#Alzheimer disease (G30),GR113-052,65,192690,33.7,26.0,43.0 +,2024 ,2024,"Kent County, DE",10001,Major cardiovascular diseases (I00-I78),GR113-053,650,192690,337.3,311.4,363.3 +,2024 ,2024,"Kent County, DE",10001,"#Diseases of heart (I00-I09,I11,I13,I20-I51)",GR113-054,444,192690,230.4,209.0,251.9 +,2024 ,2024,"Kent County, DE",10001,Hypertensive heart disease (I11),GR113-056,44,192690,22.8,16.6,30.7 +,2024 ,2024,"Kent County, DE",10001,Ischemic heart diseases (I20-I25),GR113-058,245,192690,127.1,111.2,143.1 +,2024 ,2024,"Kent County, DE",10001,Acute myocardial infarction (I21-I22),GR113-059,34,192690,17.6,12.2,24.7 +,2024 ,2024,"Kent County, DE",10001,"Other forms of chronic ischemic heart disease (I20,I25)",GR113-061,207,192690,107.4,92.8,122.1 +,2024 ,2024,"Kent County, DE",10001,"Atherosclerotic cardiovascular disease, so described (I25.0)",GR113-062,64,192690,33.2,25.6,42.4 +,2024 ,2024,"Kent County, DE",10001,"All other forms of chronic ischemic heart disease (I20,I25.1-I25.9)",GR113-063,143,192690,74.2,62.0,86.4 +,2024 ,2024,"Kent County, DE",10001,Other heart diseases (I26-I51),GR113-064,144,192690,74.7,62.5,86.9 +,2024 ,2024,"Kent County, DE",10001,Heart failure (I50),GR113-067,45,192690,23.4,17.0,31.2 +,2024 ,2024,"Kent County, DE",10001,"All other forms of heart disease (I26-I28,I34-I38,I42-I49,I51)",GR113-068,96,192690,49.8,40.4,60.8 diff --git a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv new file mode 100644 index 0000000000..47725a651f --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv @@ -0,0 +1,71 @@ +observationAbout,observationDate,variableMeasured,value,unit +geoId/10001,2023,dcid:Count_MortalityEvent_Septicemia,26, +geoId/10001,2023,dcid:Count_MortalityEvent_OtherAndUnspecifiedInfectiousAndParasiticDiseasesAndTheirSequelae,42, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasms,407, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfColonRectumAndAnus,28, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLiverAndIntrahepaticBileDucts,18, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfPancreas,31, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfTracheaBronchusAndLung,100, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfBreast,36, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfProstate,14, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfBladder,20, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfMeningesBrainAndOtherPartsOfCentralNervousSystem,10, +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLymphoidHematopoieticAndRelatedTissue,38, +geoId/10001,2023,dcid:Count_MortalityEvent_Non-HodgkinsLymphoma,15, +geoId/10001,2023,dcid:Count_MortalityEvent_Leukemia,18, +geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherAndUnspecifiedMalignantNeoplasms,57, +geoId/10001,2023,dcid:Count_MortalityEvent_InSituNeoplasmsBenignNeoplasmsAndNeoplasmsOfUncertainOrUnknownBehavior,12, +geoId/10001,2023,dcid:Count_MortalityEvent_DiabetesMellitus,78, +geoId/10001,2023,dcid:Count_MortalityEvent_NutritionalDeficiencies,11, +geoId/10001,2023,dcid:Count_MortalityEvent_Malnutrition,11, +geoId/10001,2023,dcid:Count_MortalityEvent_ParkinsonsDisease,16, +geoId/10001,2023,dcid:Count_MortalityEvent_AlzheimersDisease,72, +geoId/10001,2023,dcid:Count_MortalityEvent_MajorCardiovascularDiseases,612, +geoId/10001,2023,dcid:Count_MortalityEvent_DiseasesOfHeart,416, +geoId/10001,2023,dcid:Count_MortalityEvent_HypertensiveHeartDisease,51, +geoId/10001,2023,dcid:Count_MortalityEvent_IschemicHeartDiseases,223, +geoId/10001,2023,dcid:Count_MortalityEvent_AcuteMyocardialInfarction,33, +geoId/10001,2023,dcid:Count_MortalityEvent_OtherFormsOfChronicIschemicHeartDisease,187, +geoId/10001,2023,dcid:Count_MortalityEvent_AtheroscleroticCardiovascularDiseaseSoDescribed,67, +geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherFormsOfChronicIschemicHeartDisease,120, +geoId/10001,2023,dcid:Count_MortalityEvent_OtherHeartDiseases,128, +geoId/10001,2023,dcid:Count_MortalityEvent_HeartFailure,37, +geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherFormsOfHeartDisease,91, +geoId/10001,2023,dcid:Count_MortalityEvent_Essential_Primary_HypertensionAndHypertensiveRenalDisease,17, +geoId/10001,2023,dcid:Count_MortalityEvent_CerebrovascularDiseases,167, +geoId/10001,2023,dcid:Count_MortalityEvent_OtherDiseasesOfCirculatorySystem,11, +geoId/10001,2024,dcid:Count_MortalityEvent_Septicemia,37, +geoId/10001,2024,dcid:Count_MortalityEvent_OtherAndUnspecifiedInfectiousAndParasiticDiseasesAndTheirSequelae,38, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasms,420, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLipOralCavityAndPharynx,12, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfEsophagus,11, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfColonRectumAndAnus,32, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLiverAndIntrahepaticBileDucts,20, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfPancreas,38, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfTracheaBronchusAndLung,96, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfBreast,29, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfCorpusUteriAndUterusPartUnspecified,13, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfProstate,12, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfKidneyAndRenalPelvis,11, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfBladder,12, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfMeningesBrainAndOtherPartsOfCentralNervousSystem,11, +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLymphoidHematopoieticAndRelatedTissue,46, +geoId/10001,2024,dcid:Count_MortalityEvent_Non-HodgkinsLymphoma,16, +geoId/10001,2024,dcid:Count_MortalityEvent_Leukemia,19, +geoId/10001,2024,dcid:Count_MortalityEvent_MultipleMyelomaAndImmunoproliferativeNeoplasms,11, +geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherAndUnspecifiedMalignantNeoplasms,50, +geoId/10001,2024,dcid:Count_MortalityEvent_InSituNeoplasmsBenignNeoplasmsAndNeoplasmsOfUncertainOrUnknownBehavior,14, +geoId/10001,2024,dcid:Count_MortalityEvent_DiabetesMellitus,75, +geoId/10001,2024,dcid:Count_MortalityEvent_ParkinsonsDisease,30, +geoId/10001,2024,dcid:Count_MortalityEvent_AlzheimersDisease,65, +geoId/10001,2024,dcid:Count_MortalityEvent_MajorCardiovascularDiseases,650, +geoId/10001,2024,dcid:Count_MortalityEvent_DiseasesOfHeart,444, +geoId/10001,2024,dcid:Count_MortalityEvent_HypertensiveHeartDisease,44, +geoId/10001,2024,dcid:Count_MortalityEvent_IschemicHeartDiseases,245, +geoId/10001,2024,dcid:Count_MortalityEvent_AcuteMyocardialInfarction,34, +geoId/10001,2024,dcid:Count_MortalityEvent_OtherFormsOfChronicIschemicHeartDisease,207, +geoId/10001,2024,dcid:Count_MortalityEvent_AtheroscleroticCardiovascularDiseaseSoDescribed,64, +geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherFormsOfChronicIschemicHeartDisease,143, +geoId/10001,2024,dcid:Count_MortalityEvent_OtherHeartDiseases,144, +geoId/10001,2024,dcid:Count_MortalityEvent_HeartFailure,45, +geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherFormsOfHeartDisease,96, diff --git a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.tmcf b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.tmcf new file mode 100644 index 0000000000..a6afa2944d --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.tmcf @@ -0,0 +1,8 @@ +Node: E:underlyingcauseofdeath_county_output->E0 +observationAbout: C:underlyingcauseofdeath_county_output->observationAbout +observationDate: C:underlyingcauseofdeath_county_output->observationDate +variableMeasured: C:underlyingcauseofdeath_county_output->variableMeasured +value: C:underlyingcauseofdeath_county_output->value +unit: C:underlyingcauseofdeath_county_output->unit +typeOf: dcs:StatVarObservation +observationPeriod: P1Y diff --git a/statvar_imports/us_cdc/county_mortality/validation_config.json b/statvar_imports/us_cdc/county_mortality/validation_config.json new file mode 100644 index 0000000000..70454728a2 --- /dev/null +++ b/statvar_imports/us_cdc/county_mortality/validation_config.json @@ -0,0 +1,39 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Strictly enforce historical deletion average threshold of 0.1%", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_max_date_freshness", + "description": "Verify MaxDate is 2024 for all StatVars except rare historical exceptions with 2023", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar NOT IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", + "condition": "MaxDate = '2024'" + } + }, + { + "rule_id": "check_goldens_summary_report", + "description": "Validates summary_report.csv against the golden summary data", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv" + } + }, + { + "rule_id": "check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output/underlyingcauseofdeath_county.csv" + } + } + ] +} From 50b2adc783824f8e0f6d2b38f43991db18e08acb Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Tue, 8 Sep 2026 06:37:30 +0000 Subject: [PATCH 02/14] gemini code review suggestion --- .../us_cdc/county_mortality/download.py | 45 ++++++++++--------- .../us_cdc/county_mortality/download_test.py | 38 +++++++++++++++- 2 files changed, 60 insertions(+), 23 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py index eb2ea91bd1..b6fa97c1ef 100644 --- a/statvar_imports/us_cdc/county_mortality/download.py +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -468,32 +468,31 @@ def is_state_downloaded( if not all(f.stat().st_size > 100 for f in matches): return False if years: - initial_year = years[0] - latest_year = years[-1] - has_initial_chunk = any( - f.name.startswith(f"UnderlyingCauseofDeath_County_{state_fips}_{initial_year}") - for f in matches - ) - has_latest_chunk = any( - f.name.endswith(f"_{latest_year}.csv") or f"_{latest_year}_" in f.name - for f in matches - ) - if has_initial_chunk and has_latest_chunk: - return True + # Check if the single combined file exists and contains all years single_file = Path(output_dir) / f"UnderlyingCauseofDeath_County_{state_fips}.csv" if single_file.exists(): - found_initial = False - found_latest = False + found_years = set() with open(single_file, "r", encoding="utf-8", errors="replace") as f: for line in f: - if not found_initial and f",{initial_year}," in line: - found_initial = True - if not found_latest and f",{latest_year}," in line: - found_latest = True - if found_initial and found_latest: + for y in years: + if f",{y}," in line: + found_years.add(y) + if len(found_years) == len(years): return True return False - return False + + # Otherwise, check if every year in years is covered by at least one chunk file + for y in years: + year_covered = False + for f in matches: + if f.name == single_file.name: + continue + if f"_{y}.csv" in f.name or f"_{y}_" in f.name: + year_covered = True + break + if not year_covered: + return False + return True return True @@ -515,6 +514,7 @@ def download_county_mortality_data( total_files = 0 total_rows = 0 states_in_batch = 0 + failed_states = [] for idx, state_fips in enumerate(states, start=1): state_name = US_STATES.get(state_fips, f"FIPS-{state_fips}") @@ -569,9 +569,12 @@ def download_county_mortality_data( states_in_batch = 0 except Exception as e: - logging.fatal("Failed downloading state %s (FIPS %s): %s", state_name, state_fips, e) + logging.error("Failed downloading state %s (FIPS %s): %s", state_name, state_fips, e) + failed_states.append(state_name) logging.info("Download complete: Saved %d files with %d total rows in %s.", total_files, total_rows, output_dir) + if failed_states: + raise RuntimeError(f"Failed to download data for states: {', '.join(failed_states)}") def main(_): diff --git a/statvar_imports/us_cdc/county_mortality/download_test.py b/statvar_imports/us_cdc/county_mortality/download_test.py index fd908eb663..9e6aaa085b 100644 --- a/statvar_imports/us_cdc/county_mortality/download_test.py +++ b/statvar_imports/us_cdc/county_mortality/download_test.py @@ -170,11 +170,19 @@ def test_is_state_downloaded(self): # Only latest chunk present -> should be False self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) - # Both initial chunk and latest chunk present -> should be True + # Both initial chunk and latest chunk present -> should be True for 2018, 2024 chunk_2018 = Path(temp_dir) / "UnderlyingCauseofDeath_County_10_2018_2019.csv" chunk_2018.write_text("Header,col1\n" + "val1,val2\n" * 10) self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) + # If intermediate year (e.g. 2021) is requested but missing chunk -> should be False + self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + + # Add intermediate chunk covering 2021 -> should now be True + chunk_2020 = Path(temp_dir) / "UnderlyingCauseofDeath_County_10_2020_2021.csv" + chunk_2020.write_text("Header,col1\n" + "val1,val2\n" * 10) + self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + @mock.patch.object(download.time, "sleep") @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "init_session") def test_execute_query_429_backoff(self, mock_init, mock_sleep): @@ -199,7 +207,6 @@ def test_execute_query_429_backoff(self, mock_init, mock_sleep): mock_sleep.assert_called_with(1) mock_init.assert_called_once() - @mock.patch.object(download.time, "sleep") @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "execute_query") def test_download_state_timeout_fallback_to_single_years(self, mock_query, mock_sleep): @@ -215,6 +222,33 @@ def query_side_effect(state_fips, years, **kwargs): self.assertEqual(len(results), 3) self.assertEqual([r[0] for r in results], ["2018", "2019", "2020"]) + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "init_session") + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "download_state") + def test_download_county_mortality_data_continues_and_raises_on_failure( + self, mock_download, mock_init + ): + with tempfile.TemporaryDirectory() as temp_dir: + def side_effect(state_fips, years): + if state_fips == "10": + raise requests.exceptions.ConnectionError("Connection dropped") + return [("all", "Notes\tCounty Code\tDeaths\n\t11001\t50\n")] + + mock_download.side_effect = side_effect + + with self.assertRaises(RuntimeError) as ctx: + download.download_county_mortality_data( + states=["10", "11"], + years=["2024"], + output_dir=temp_dir, + skip_existing=False, + ) + + # Assert error message contains the failed state name + self.assertIn("Delaware", str(ctx.exception)) + # Assert state 11 was still attempted and saved despite state 10 failure + state_11_csv = Path(temp_dir) / "UnderlyingCauseofDeath_County_11.csv" + self.assertTrue(state_11_csv.exists()) + if __name__ == "__main__": unittest.main() From 7e5738cc3a1bfa78442b21c02056fd85e68b4778 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Tue, 8 Sep 2026 10:19:44 +0000 Subject: [PATCH 03/14] cra fixes --- .../us_cdc/county_mortality/README.md | 30 +++++++- .../us_cdc/county_mortality/download.py | 1 - .../us_cdc/county_mortality/download_test.py | 7 ++ .../county_mortality/validation_config.json | 68 ++++++++----------- 4 files changed, 65 insertions(+), 41 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index d5a7339948..c690634b27 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -32,9 +32,35 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ --input_data=input_files/*.csv \ --pv_map=county_mortality_pvmap.csv \ --config_file=county_mortality_metadata.csv \ - --output_path=output/underlyingcauseofdeath_county + --output_path=output/underlyingcauseofdeath_county \ + --output_counters=counters/underlyingcauseofdeath_county_counters.csv ``` #### Output Artifacts -* `output/underlyingcauseofdeath_county.csv`: Cleaned observations mapping FIPS (`geoId/{fips}`), year, StatVar (`Count_MortalityEvent_`), and death count. +* `output/underlyingcauseofdeath_county.csv`: Cleaned observations mapping FIPS (`geoId/{fips}`), year, StatVar (`Count_MortalityEvent_`), death count, and unit. * `output/underlyingcauseofdeath_county.tmcf`: Template MCF mapping CSV columns to Data Commons Knowledge Graph entities. + +--- + +### Important Files + +| File | Description | +|---|---| +| `download.py` | Automated live CDC WONDER downloader, session manager, and query partitioner. | +| `download_test.py` | Comprehensive unit test suite for download session management and error handling. | +| `county_mortality_metadata.csv` | Metadata specifying header row offsets, frequency, and output columns. | +| `county_mortality_pvmap.csv` | Property-Value mapping resolving county FIPS and ICD-10 113 causes of death. | +| `manifest.json` | Automation manifest declaring scripts, inputs/outputs, cron schedule, and validation config. | +| `validation_config.json` | Configuration file defining validation rules (deleted records, date freshness, goldens). | +| `golden_data/` | Established golden summary and observation records for automated regression validation. | +| `test_data/` | Trimmed sample Delaware dataset and expected outputs for offline verification. | +| `runbook.md` | Operational runbook, architecture documentation, and troubleshooting guide. | + +--- + +### Testing + +Run the unit test suite from this directory: +```bash +python3 -m unittest discover -v -s . -p "*_test.py" +``` diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py index b6fa97c1ef..6aec3eab3f 100644 --- a/statvar_imports/us_cdc/county_mortality/download.py +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -479,7 +479,6 @@ def is_state_downloaded( found_years.add(y) if len(found_years) == len(years): return True - return False # Otherwise, check if every year in years is covered by at least one chunk file for y in years: diff --git a/statvar_imports/us_cdc/county_mortality/download_test.py b/statvar_imports/us_cdc/county_mortality/download_test.py index 9e6aaa085b..be3da473b9 100644 --- a/statvar_imports/us_cdc/county_mortality/download_test.py +++ b/statvar_imports/us_cdc/county_mortality/download_test.py @@ -183,6 +183,13 @@ def test_is_state_downloaded(self): chunk_2020.write_text("Header,col1\n" + "val1,val2\n" * 10) self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + # If an incomplete single combined file also exists (e.g. from an aborted run), + # but chunk files cover all requested years, it should still return True + f = Path(temp_dir) / "UnderlyingCauseofDeath_County_10.csv" + f.write_text("Header,col1\n,2018,val1\n" * 5) # only contains 2018 + self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + f.unlink() + @mock.patch.object(download.time, "sleep") @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "init_session") def test_execute_query_429_backoff(self, mock_init, mock_sleep): diff --git a/statvar_imports/us_cdc/county_mortality/validation_config.json b/statvar_imports/us_cdc/county_mortality/validation_config.json index 70454728a2..1983748baf 100644 --- a/statvar_imports/us_cdc/county_mortality/validation_config.json +++ b/statvar_imports/us_cdc/county_mortality/validation_config.json @@ -1,39 +1,31 @@ { - "schema_version": "1.0", - "rules": [ - { - "rule_id": "check_deleted_records_percent", - "description": "Strictly enforce historical deletion average threshold of 0.1%", - "validator": "DELETED_RECORDS_PERCENT", - "params": { - "threshold": 0.1 - } - }, - { - "rule_id": "check_max_date_freshness", - "description": "Verify MaxDate is 2024 for all StatVars except rare historical exceptions with 2023", - "validator": "SQL_VALIDATOR", - "params": { - "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar NOT IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", - "condition": "MaxDate = '2024'" - } - }, - { - "rule_id": "check_goldens_summary_report", - "description": "Validates summary_report.csv against the golden summary data", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "../../../../golden_data/golden_summary_report.csv" - } - }, - { - "rule_id": "check_goldens_output_csv", - "description": "Verifies the generated output CSV data matches established critical golden records", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "../../../../golden_data/golden_observations.csv", - "input_files": "../../../../output/underlyingcauseofdeath_county.csv" - } - } - ] -} + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Strictly enforce historical deletion average threshold of 0.1%", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 0.1 + } + }, + { + "rule_id": "check_goldens_summary_report", + "description": "Validates summary_report.csv against the golden summary data", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_summary_report.csv", + "input_files": "../../input0/genmcf/summary_report.csv" + } + }, + { + "rule_id": "check_goldens_output_csv", + "description": "Verifies the generated output CSV data matches established critical golden records", + "validator": "GOLDENS_CHECK", + "params": { + "golden_files": "../../../../golden_data/golden_observations.csv", + "input_files": "../../../../output/underlyingcauseofdeath_county.csv" + } + } + ] +} \ No newline at end of file From cc5530324fc23a7fbbb94c58a41171c9de76be7f Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Tue, 8 Sep 2026 12:37:17 +0000 Subject: [PATCH 04/14] cra fixes2 --- .../us_cdc/county_mortality/README.md | 3 +-- .../us_cdc/county_mortality/manifest.json | 1 - .../underlyingcauseofdeath_county_output.csv | 2 +- .../county_mortality/validation_config.json | 18 ------------------ 4 files changed, 2 insertions(+), 22 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index c690634b27..9ba7fc9607 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -51,8 +51,7 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ | `county_mortality_metadata.csv` | Metadata specifying header row offsets, frequency, and output columns. | | `county_mortality_pvmap.csv` | Property-Value mapping resolving county FIPS and ICD-10 113 causes of death. | | `manifest.json` | Automation manifest declaring scripts, inputs/outputs, cron schedule, and validation config. | -| `validation_config.json` | Configuration file defining validation rules (deleted records, date freshness, goldens). | -| `golden_data/` | Established golden summary and observation records for automated regression validation. | +| `validation_config.json` | Configuration file defining import validation rules (historical deleted records threshold). | | `test_data/` | Trimmed sample Delaware dataset and expected outputs for offline verification. | | `runbook.md` | Operational runbook, architecture documentation, and troubleshooting guide. | diff --git a/statvar_imports/us_cdc/county_mortality/manifest.json b/statvar_imports/us_cdc/county_mortality/manifest.json index 6cbc5793a0..db2e5c594c 100644 --- a/statvar_imports/us_cdc/county_mortality/manifest.json +++ b/statvar_imports/us_cdc/county_mortality/manifest.json @@ -20,7 +20,6 @@ ], "source_files": [ "input_files/*.csv", - "golden_data/*.csv", "counters/*.csv" ], "cron_schedule": "0 5 * * 5", diff --git a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv index 47725a651f..9d7b6dd21b 100644 --- a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv +++ b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv @@ -31,7 +31,7 @@ geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherFormsOfChronicIschemicHeartDi geoId/10001,2023,dcid:Count_MortalityEvent_OtherHeartDiseases,128, geoId/10001,2023,dcid:Count_MortalityEvent_HeartFailure,37, geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherFormsOfHeartDisease,91, -geoId/10001,2023,dcid:Count_MortalityEvent_Essential_Primary_HypertensionAndHypertensiveRenalDisease,17, +geoId/10001,2023,dcid:Count_MortalityEvent_Essential(Primary)HypertensionAndHypertensiveRenalDisease,17, geoId/10001,2023,dcid:Count_MortalityEvent_CerebrovascularDiseases,167, geoId/10001,2023,dcid:Count_MortalityEvent_OtherDiseasesOfCirculatorySystem,11, geoId/10001,2024,dcid:Count_MortalityEvent_Septicemia,37, diff --git a/statvar_imports/us_cdc/county_mortality/validation_config.json b/statvar_imports/us_cdc/county_mortality/validation_config.json index 1983748baf..39a6784647 100644 --- a/statvar_imports/us_cdc/county_mortality/validation_config.json +++ b/statvar_imports/us_cdc/county_mortality/validation_config.json @@ -8,24 +8,6 @@ "params": { "threshold": 0.1 } - }, - { - "rule_id": "check_goldens_summary_report", - "description": "Validates summary_report.csv against the golden summary data", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "../../../../golden_data/golden_summary_report.csv", - "input_files": "../../input0/genmcf/summary_report.csv" - } - }, - { - "rule_id": "check_goldens_output_csv", - "description": "Verifies the generated output CSV data matches established critical golden records", - "validator": "GOLDENS_CHECK", - "params": { - "golden_files": "../../../../golden_data/golden_observations.csv", - "input_files": "../../../../output/underlyingcauseofdeath_county.csv" - } } ] } \ No newline at end of file From 143335116395b678ced9d6fcec143b9cb2ee46e8 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Tue, 8 Sep 2026 12:40:37 +0000 Subject: [PATCH 05/14] goldens removed --- .../golden_data/golden_observations.csv | 2095 ----------------- .../golden_data/golden_summary_report.csv | 3 - 2 files changed, 2098 deletions(-) delete mode 100644 statvar_imports/us_cdc/county_mortality/golden_data/golden_observations.csv delete mode 100644 statvar_imports/us_cdc/county_mortality/golden_data/golden_summary_report.csv diff --git a/statvar_imports/us_cdc/county_mortality/golden_data/golden_observations.csv b/statvar_imports/us_cdc/county_mortality/golden_data/golden_observations.csv deleted file mode 100644 index d6b519d995..0000000000 --- a/statvar_imports/us_cdc/county_mortality/golden_data/golden_observations.csv +++ /dev/null @@ -1,2095 +0,0 @@ -"observationAbout" -"geoId/01001" -"geoId/01003" -"geoId/01005" -"geoId/01007" -"geoId/01009" -"geoId/01013" -"geoId/01015" -"geoId/01017" -"geoId/01019" -"geoId/01021" -"geoId/01023" -"geoId/01025" -"geoId/01031" -"geoId/01033" -"geoId/01039" -"geoId/01043" -"geoId/01045" -"geoId/01047" -"geoId/01049" -"geoId/01051" -"geoId/01053" -"geoId/01055" -"geoId/01057" -"geoId/01059" -"geoId/01061" -"geoId/01065" -"geoId/01067" -"geoId/01069" -"geoId/01071" -"geoId/01073" -"geoId/01077" -"geoId/01079" -"geoId/01081" -"geoId/01083" -"geoId/01087" -"geoId/01089" -"geoId/01091" -"geoId/01093" -"geoId/01095" -"geoId/01097" -"geoId/01099" -"geoId/01101" -"geoId/01103" -"geoId/01107" -"geoId/01109" -"geoId/01111" -"geoId/01113" -"geoId/01115" -"geoId/01117" -"geoId/01119" -"geoId/01121" -"geoId/01123" -"geoId/01125" -"geoId/01127" -"geoId/01129" -"geoId/01133" -"geoId/02020" -"geoId/02050" -"geoId/02090" -"geoId/02110" -"geoId/02122" -"geoId/02170" -"geoId/04001" -"geoId/04003" -"geoId/04005" -"geoId/04007" -"geoId/04009" -"geoId/04012" -"geoId/04013" -"geoId/04015" -"geoId/04017" -"geoId/04019" -"geoId/04021" -"geoId/04023" -"geoId/04025" -"geoId/04027" -"geoId/05001" -"geoId/05003" -"geoId/05005" -"geoId/05007" -"geoId/05009" -"geoId/05015" -"geoId/05017" -"geoId/05019" -"geoId/05021" -"geoId/05023" -"geoId/05027" -"geoId/05029" -"geoId/05031" -"geoId/05033" -"geoId/05035" -"geoId/05037" -"geoId/05041" -"geoId/05043" -"geoId/05045" -"geoId/05047" -"geoId/05051" -"geoId/05053" -"geoId/05055" -"geoId/05057" -"geoId/05059" -"geoId/05063" -"geoId/05067" -"geoId/05069" -"geoId/05071" -"geoId/05075" -"geoId/05077" -"geoId/05083" -"geoId/05085" -"geoId/05087" -"geoId/05089" -"geoId/05091" -"geoId/05093" -"geoId/05103" -"geoId/05107" -"geoId/05111" -"geoId/05113" -"geoId/05115" -"geoId/05119" -"geoId/05121" -"geoId/05123" -"geoId/05125" -"geoId/05131" -"geoId/05133" -"geoId/05135" -"geoId/05139" -"geoId/05141" -"geoId/05143" -"geoId/05145" -"geoId/05149" -"geoId/06001" -"geoId/06005" -"geoId/06007" -"geoId/06009" -"geoId/06011" -"geoId/06013" -"geoId/06015" -"geoId/06017" -"geoId/06019" -"geoId/06021" -"geoId/06023" -"geoId/06025" -"geoId/06027" -"geoId/06029" -"geoId/06031" -"geoId/06033" -"geoId/06035" -"geoId/06037" -"geoId/06039" -"geoId/06041" -"geoId/06043" -"geoId/06045" -"geoId/06047" -"geoId/06053" -"geoId/06055" -"geoId/06057" -"geoId/06059" -"geoId/06061" -"geoId/06063" -"geoId/06065" -"geoId/06067" -"geoId/06069" -"geoId/06071" -"geoId/06073" -"geoId/06075" -"geoId/06077" -"geoId/06079" -"geoId/06081" -"geoId/06083" -"geoId/06085" -"geoId/06087" -"geoId/06089" -"geoId/06093" -"geoId/06095" -"geoId/06097" -"geoId/06099" -"geoId/06101" -"geoId/06103" -"geoId/06107" -"geoId/06109" -"geoId/06111" -"geoId/06113" -"geoId/06115" -"geoId/08001" -"geoId/08005" -"geoId/08013" -"geoId/08014" -"geoId/08015" -"geoId/08029" -"geoId/08031" -"geoId/08035" -"geoId/08037" -"geoId/08039" -"geoId/08041" -"geoId/08043" -"geoId/08045" -"geoId/08051" -"geoId/08059" -"geoId/08067" -"geoId/08069" -"geoId/08075" -"geoId/08077" -"geoId/08083" -"geoId/08085" -"geoId/08087" -"geoId/08089" -"geoId/08093" -"geoId/08097" -"geoId/08101" -"geoId/08107" -"geoId/08117" -"geoId/08119" -"geoId/08123" -"geoId/09001" -"geoId/09003" -"geoId/09005" -"geoId/09007" -"geoId/09009" -"geoId/09011" -"geoId/09013" -"geoId/09015" -"geoId/10001" -"geoId/10003" -"geoId/10005" -"geoId/11001" -"geoId/12001" -"geoId/12003" -"geoId/12005" -"geoId/12007" -"geoId/12009" -"geoId/12011" -"geoId/12015" -"geoId/12017" -"geoId/12019" -"geoId/12021" -"geoId/12023" -"geoId/12027" -"geoId/12029" -"geoId/12031" -"geoId/12033" -"geoId/12035" -"geoId/12039" -"geoId/12041" -"geoId/12049" -"geoId/12051" -"geoId/12053" -"geoId/12055" -"geoId/12057" -"geoId/12059" -"geoId/12061" -"geoId/12063" -"geoId/12069" -"geoId/12071" -"geoId/12073" -"geoId/12075" -"geoId/12079" -"geoId/12081" -"geoId/12083" -"geoId/12085" -"geoId/12086" -"geoId/12087" -"geoId/12089" -"geoId/12091" -"geoId/12093" -"geoId/12095" -"geoId/12097" -"geoId/12099" -"geoId/12101" -"geoId/12103" -"geoId/12105" -"geoId/12107" -"geoId/12109" -"geoId/12111" -"geoId/12113" -"geoId/12115" -"geoId/12117" -"geoId/12119" -"geoId/12121" -"geoId/12123" -"geoId/12127" -"geoId/12129" -"geoId/12131" -"geoId/12133" -"geoId/13001" -"geoId/13009" -"geoId/13011" -"geoId/13013" -"geoId/13015" -"geoId/13017" -"geoId/13019" -"geoId/13021" -"geoId/13025" -"geoId/13029" -"geoId/13031" -"geoId/13033" -"geoId/13035" -"geoId/13039" -"geoId/13045" -"geoId/13047" -"geoId/13051" -"geoId/13053" -"geoId/13055" -"geoId/13057" -"geoId/13059" -"geoId/13063" -"geoId/13067" -"geoId/13069" -"geoId/13071" -"geoId/13073" -"geoId/13075" -"geoId/13077" -"geoId/13081" -"geoId/13085" -"geoId/13087" -"geoId/13089" -"geoId/13091" -"geoId/13095" -"geoId/13097" -"geoId/13103" -"geoId/13105" -"geoId/13107" -"geoId/13111" -"geoId/13113" -"geoId/13115" -"geoId/13117" -"geoId/13119" -"geoId/13121" -"geoId/13123" -"geoId/13127" -"geoId/13129" -"geoId/13131" -"geoId/13133" -"geoId/13135" -"geoId/13137" -"geoId/13139" -"geoId/13143" -"geoId/13145" -"geoId/13147" -"geoId/13151" -"geoId/13153" -"geoId/13157" -"geoId/13159" -"geoId/13163" -"geoId/13169" -"geoId/13171" -"geoId/13175" -"geoId/13177" -"geoId/13179" -"geoId/13183" -"geoId/13185" -"geoId/13187" -"geoId/13189" -"geoId/13195" -"geoId/13199" -"geoId/13205" -"geoId/13207" -"geoId/13211" -"geoId/13213" -"geoId/13215" -"geoId/13217" -"geoId/13219" -"geoId/13223" -"geoId/13225" -"geoId/13227" -"geoId/13229" -"geoId/13231" -"geoId/13233" -"geoId/13237" -"geoId/13241" -"geoId/13245" -"geoId/13247" -"geoId/13255" -"geoId/13257" -"geoId/13261" -"geoId/13267" -"geoId/13275" -"geoId/13277" -"geoId/13279" -"geoId/13285" -"geoId/13291" -"geoId/13293" -"geoId/13295" -"geoId/13297" -"geoId/13299" -"geoId/13303" -"geoId/13305" -"geoId/13311" -"geoId/13313" -"geoId/13321" -"geoId/15001" -"geoId/15003" -"geoId/15007" -"geoId/15009" -"geoId/16001" -"geoId/16005" -"geoId/16011" -"geoId/16013" -"geoId/16017" -"geoId/16019" -"geoId/16027" -"geoId/16031" -"geoId/16039" -"geoId/16045" -"geoId/16049" -"geoId/16051" -"geoId/16053" -"geoId/16055" -"geoId/16057" -"geoId/16065" -"geoId/16067" -"geoId/16069" -"geoId/16075" -"geoId/16079" -"geoId/16083" -"geoId/17001" -"geoId/17005" -"geoId/17007" -"geoId/17011" -"geoId/17015" -"geoId/17019" -"geoId/17021" -"geoId/17023" -"geoId/17027" -"geoId/17029" -"geoId/17031" -"geoId/17033" -"geoId/17037" -"geoId/17039" -"geoId/17041" -"geoId/17043" -"geoId/17045" -"geoId/17049" -"geoId/17051" -"geoId/17055" -"geoId/17057" -"geoId/17061" -"geoId/17063" -"geoId/17067" -"geoId/17073" -"geoId/17075" -"geoId/17077" -"geoId/17081" -"geoId/17083" -"geoId/17085" -"geoId/17089" -"geoId/17091" -"geoId/17093" -"geoId/17095" -"geoId/17097" -"geoId/17099" -"geoId/17101" -"geoId/17103" -"geoId/17105" -"geoId/17107" -"geoId/17109" -"geoId/17111" -"geoId/17113" -"geoId/17115" -"geoId/17117" -"geoId/17119" -"geoId/17121" -"geoId/17125" -"geoId/17131" -"geoId/17133" -"geoId/17135" -"geoId/17137" -"geoId/17141" -"geoId/17143" -"geoId/17145" -"geoId/17149" -"geoId/17157" -"geoId/17159" -"geoId/17161" -"geoId/17163" -"geoId/17165" -"geoId/17167" -"geoId/17173" -"geoId/17177" -"geoId/17179" -"geoId/17181" -"geoId/17183" -"geoId/17187" -"geoId/17191" -"geoId/17193" -"geoId/17195" -"geoId/17197" -"geoId/17199" -"geoId/17201" -"geoId/17203" -"geoId/18001" -"geoId/18003" -"geoId/18005" -"geoId/18011" -"geoId/18015" -"geoId/18017" -"geoId/18019" -"geoId/18021" -"geoId/18023" -"geoId/18027" -"geoId/18029" -"geoId/18031" -"geoId/18033" -"geoId/18035" -"geoId/18037" -"geoId/18039" -"geoId/18041" -"geoId/18043" -"geoId/18045" -"geoId/18047" -"geoId/18049" -"geoId/18051" -"geoId/18053" -"geoId/18055" -"geoId/18057" -"geoId/18059" -"geoId/18061" -"geoId/18063" -"geoId/18065" -"geoId/18067" -"geoId/18069" -"geoId/18071" -"geoId/18073" -"geoId/18075" -"geoId/18077" -"geoId/18079" -"geoId/18081" -"geoId/18083" -"geoId/18085" -"geoId/18087" -"geoId/18089" -"geoId/18091" -"geoId/18093" -"geoId/18095" -"geoId/18097" -"geoId/18099" -"geoId/18103" -"geoId/18105" -"geoId/18107" -"geoId/18109" -"geoId/18113" -"geoId/18117" -"geoId/18119" -"geoId/18121" -"geoId/18123" -"geoId/18127" -"geoId/18129" -"geoId/18133" -"geoId/18135" -"geoId/18137" -"geoId/18139" -"geoId/18141" -"geoId/18143" -"geoId/18145" -"geoId/18147" -"geoId/18149" -"geoId/18151" -"geoId/18153" -"geoId/18157" -"geoId/18163" -"geoId/18165" -"geoId/18167" -"geoId/18169" -"geoId/18173" -"geoId/18175" -"geoId/18177" -"geoId/18179" -"geoId/18181" -"geoId/18183" -"geoId/19011" -"geoId/19013" -"geoId/19015" -"geoId/19017" -"geoId/19019" -"geoId/19021" -"geoId/19023" -"geoId/19027" -"geoId/19029" -"geoId/19031" -"geoId/19033" -"geoId/19035" -"geoId/19041" -"geoId/19043" -"geoId/19045" -"geoId/19047" -"geoId/19049" -"geoId/19055" -"geoId/19057" -"geoId/19059" -"geoId/19061" -"geoId/19065" -"geoId/19067" -"geoId/19079" -"geoId/19083" -"geoId/19085" -"geoId/19087" -"geoId/19097" -"geoId/19099" -"geoId/19101" -"geoId/19103" -"geoId/19105" -"geoId/19109" -"geoId/19111" -"geoId/19113" -"geoId/19121" -"geoId/19123" -"geoId/19125" -"geoId/19127" -"geoId/19133" -"geoId/19139" -"geoId/19141" -"geoId/19145" -"geoId/19149" -"geoId/19153" -"geoId/19155" -"geoId/19157" -"geoId/19163" -"geoId/19167" -"geoId/19169" -"geoId/19171" -"geoId/19179" -"geoId/19181" -"geoId/19183" -"geoId/19187" -"geoId/19191" -"geoId/19193" -"geoId/19197" -"geoId/20005" -"geoId/20009" -"geoId/20015" -"geoId/20021" -"geoId/20035" -"geoId/20037" -"geoId/20041" -"geoId/20045" -"geoId/20051" -"geoId/20055" -"geoId/20057" -"geoId/20059" -"geoId/20061" -"geoId/20079" -"geoId/20087" -"geoId/20091" -"geoId/20099" -"geoId/20103" -"geoId/20111" -"geoId/20113" -"geoId/20121" -"geoId/20125" -"geoId/20133" -"geoId/20139" -"geoId/20149" -"geoId/20155" -"geoId/20161" -"geoId/20169" -"geoId/20173" -"geoId/20175" -"geoId/20177" -"geoId/20191" -"geoId/20209" -"geoId/21001" -"geoId/21003" -"geoId/21005" -"geoId/21009" -"geoId/21013" -"geoId/21015" -"geoId/21017" -"geoId/21019" -"geoId/21021" -"geoId/21025" -"geoId/21027" -"geoId/21029" -"geoId/21035" -"geoId/21037" -"geoId/21043" -"geoId/21047" -"geoId/21049" -"geoId/21051" -"geoId/21059" -"geoId/21067" -"geoId/21071" -"geoId/21073" -"geoId/21079" -"geoId/21081" -"geoId/21083" -"geoId/21085" -"geoId/21089" -"geoId/21093" -"geoId/21095" -"geoId/21097" -"geoId/21099" -"geoId/21101" -"geoId/21107" -"geoId/21111" -"geoId/21113" -"geoId/21115" -"geoId/21117" -"geoId/21119" -"geoId/21121" -"geoId/21125" -"geoId/21133" -"geoId/21137" -"geoId/21141" -"geoId/21145" -"geoId/21147" -"geoId/21151" -"geoId/21155" -"geoId/21157" -"geoId/21161" -"geoId/21163" -"geoId/21167" -"geoId/21173" -"geoId/21177" -"geoId/21179" -"geoId/21183" -"geoId/21185" -"geoId/21193" -"geoId/21195" -"geoId/21199" -"geoId/21203" -"geoId/21205" -"geoId/21207" -"geoId/21209" -"geoId/21211" -"geoId/21213" -"geoId/21215" -"geoId/21217" -"geoId/21225" -"geoId/21227" -"geoId/21231" -"geoId/21235" -"geoId/21239" -"geoId/22001" -"geoId/22003" -"geoId/22005" -"geoId/22007" -"geoId/22009" -"geoId/22011" -"geoId/22015" -"geoId/22017" -"geoId/22019" -"geoId/22027" -"geoId/22029" -"geoId/22031" -"geoId/22033" -"geoId/22037" -"geoId/22039" -"geoId/22041" -"geoId/22043" -"geoId/22045" -"geoId/22047" -"geoId/22049" -"geoId/22051" -"geoId/22053" -"geoId/22055" -"geoId/22057" -"geoId/22061" -"geoId/22063" -"geoId/22067" -"geoId/22069" -"geoId/22071" -"geoId/22073" -"geoId/22075" -"geoId/22077" -"geoId/22079" -"geoId/22083" -"geoId/22085" -"geoId/22087" -"geoId/22089" -"geoId/22093" -"geoId/22095" -"geoId/22097" -"geoId/22099" -"geoId/22101" -"geoId/22103" -"geoId/22105" -"geoId/22109" -"geoId/22111" -"geoId/22113" -"geoId/22115" -"geoId/22117" -"geoId/22119" -"geoId/22121" -"geoId/22127" -"geoId/23001" -"geoId/23003" -"geoId/23005" -"geoId/23007" -"geoId/23009" -"geoId/23011" -"geoId/23013" -"geoId/23015" -"geoId/23017" -"geoId/23019" -"geoId/23021" -"geoId/23023" -"geoId/23025" -"geoId/23027" -"geoId/23029" -"geoId/23031" -"geoId/24001" -"geoId/24003" -"geoId/24005" -"geoId/24009" -"geoId/24011" -"geoId/24013" -"geoId/24015" -"geoId/24017" -"geoId/24019" -"geoId/24021" -"geoId/24023" -"geoId/24025" -"geoId/24027" -"geoId/24029" -"geoId/24031" -"geoId/24033" -"geoId/24035" -"geoId/24037" -"geoId/24039" -"geoId/24041" -"geoId/24043" -"geoId/24045" -"geoId/24047" -"geoId/24510" -"geoId/25001" -"geoId/25003" -"geoId/25005" -"geoId/25007" -"geoId/25009" -"geoId/25011" -"geoId/25013" -"geoId/25015" -"geoId/25017" -"geoId/25021" -"geoId/25023" -"geoId/25025" -"geoId/25027" -"geoId/26005" -"geoId/26007" -"geoId/26009" -"geoId/26011" -"geoId/26015" -"geoId/26017" -"geoId/26019" -"geoId/26021" -"geoId/26023" -"geoId/26025" -"geoId/26027" -"geoId/26029" -"geoId/26031" -"geoId/26033" -"geoId/26035" -"geoId/26037" -"geoId/26041" -"geoId/26043" -"geoId/26045" -"geoId/26047" -"geoId/26049" -"geoId/26051" -"geoId/26053" -"geoId/26055" -"geoId/26057" -"geoId/26059" -"geoId/26061" -"geoId/26063" -"geoId/26065" -"geoId/26067" -"geoId/26069" -"geoId/26073" -"geoId/26075" -"geoId/26077" -"geoId/26079" -"geoId/26081" -"geoId/26087" -"geoId/26089" -"geoId/26091" -"geoId/26093" -"geoId/26099" -"geoId/26101" -"geoId/26103" -"geoId/26105" -"geoId/26107" -"geoId/26109" -"geoId/26111" -"geoId/26115" -"geoId/26117" -"geoId/26121" -"geoId/26123" -"geoId/26125" -"geoId/26127" -"geoId/26129" -"geoId/26133" -"geoId/26137" -"geoId/26139" -"geoId/26143" -"geoId/26145" -"geoId/26147" -"geoId/26149" -"geoId/26151" -"geoId/26155" -"geoId/26157" -"geoId/26159" -"geoId/26161" -"geoId/26163" -"geoId/26165" -"geoId/27003" -"geoId/27005" -"geoId/27007" -"geoId/27009" -"geoId/27013" -"geoId/27015" -"geoId/27017" -"geoId/27019" -"geoId/27021" -"geoId/27025" -"geoId/27027" -"geoId/27035" -"geoId/27037" -"geoId/27039" -"geoId/27041" -"geoId/27043" -"geoId/27045" -"geoId/27047" -"geoId/27049" -"geoId/27053" -"geoId/27055" -"geoId/27057" -"geoId/27059" -"geoId/27061" -"geoId/27067" -"geoId/27071" -"geoId/27079" -"geoId/27083" -"geoId/27085" -"geoId/27091" -"geoId/27093" -"geoId/27095" -"geoId/27097" -"geoId/27099" -"geoId/27103" -"geoId/27105" -"geoId/27109" -"geoId/27111" -"geoId/27115" -"geoId/27119" -"geoId/27123" -"geoId/27127" -"geoId/27129" -"geoId/27131" -"geoId/27137" -"geoId/27139" -"geoId/27141" -"geoId/27145" -"geoId/27147" -"geoId/27153" -"geoId/27157" -"geoId/27161" -"geoId/27163" -"geoId/27169" -"geoId/27171" -"geoId/28001" -"geoId/28003" -"geoId/28007" -"geoId/28011" -"geoId/28017" -"geoId/28023" -"geoId/28025" -"geoId/28027" -"geoId/28029" -"geoId/28031" -"geoId/28033" -"geoId/28035" -"geoId/28039" -"geoId/28043" -"geoId/28045" -"geoId/28047" -"geoId/28049" -"geoId/28051" -"geoId/28057" -"geoId/28059" -"geoId/28061" -"geoId/28067" -"geoId/28071" -"geoId/28073" -"geoId/28075" -"geoId/28079" -"geoId/28081" -"geoId/28083" -"geoId/28085" -"geoId/28087" -"geoId/28089" -"geoId/28091" -"geoId/28093" -"geoId/28095" -"geoId/28099" -"geoId/28101" -"geoId/28105" -"geoId/28107" -"geoId/28109" -"geoId/28113" -"geoId/28115" -"geoId/28117" -"geoId/28121" -"geoId/28123" -"geoId/28127" -"geoId/28131" -"geoId/28133" -"geoId/28135" -"geoId/28137" -"geoId/28139" -"geoId/28141" -"geoId/28145" -"geoId/28149" -"geoId/28151" -"geoId/28153" -"geoId/28159" -"geoId/28163" -"geoId/29001" -"geoId/29003" -"geoId/29007" -"geoId/29009" -"geoId/29013" -"geoId/29015" -"geoId/29019" -"geoId/29021" -"geoId/29023" -"geoId/29027" -"geoId/29029" -"geoId/29031" -"geoId/29037" -"geoId/29043" -"geoId/29047" -"geoId/29049" -"geoId/29051" -"geoId/29053" -"geoId/29055" -"geoId/29059" -"geoId/29069" -"geoId/29071" -"geoId/29077" -"geoId/29083" -"geoId/29091" -"geoId/29095" -"geoId/29097" -"geoId/29099" -"geoId/29101" -"geoId/29105" -"geoId/29107" -"geoId/29109" -"geoId/29113" -"geoId/29119" -"geoId/29127" -"geoId/29131" -"geoId/29141" -"geoId/29143" -"geoId/29145" -"geoId/29147" -"geoId/29155" -"geoId/29157" -"geoId/29159" -"geoId/29161" -"geoId/29163" -"geoId/29165" -"geoId/29167" -"geoId/29169" -"geoId/29175" -"geoId/29177" -"geoId/29183" -"geoId/29186" -"geoId/29187" -"geoId/29189" -"geoId/29195" -"geoId/29201" -"geoId/29207" -"geoId/29209" -"geoId/29213" -"geoId/29215" -"geoId/29217" -"geoId/29219" -"geoId/29221" -"geoId/29225" -"geoId/29229" -"geoId/29510" -"geoId/30013" -"geoId/30029" -"geoId/30031" -"geoId/30041" -"geoId/30047" -"geoId/30049" -"geoId/30053" -"geoId/30063" -"geoId/30067" -"geoId/30081" -"geoId/30093" -"geoId/30111" -"geoId/31001" -"geoId/31019" -"geoId/31025" -"geoId/31043" -"geoId/31047" -"geoId/31053" -"geoId/31055" -"geoId/31067" -"geoId/31079" -"geoId/31109" -"geoId/31111" -"geoId/31119" -"geoId/31141" -"geoId/31153" -"geoId/31155" -"geoId/31157" -"geoId/31159" -"geoId/31177" -"geoId/32001" -"geoId/32003" -"geoId/32005" -"geoId/32007" -"geoId/32013" -"geoId/32019" -"geoId/32023" -"geoId/32031" -"geoId/32510" -"geoId/33001" -"geoId/33003" -"geoId/33005" -"geoId/33007" -"geoId/33009" -"geoId/33011" -"geoId/33013" -"geoId/33015" -"geoId/33017" -"geoId/33019" -"geoId/34001" -"geoId/34003" -"geoId/34005" -"geoId/34007" -"geoId/34009" -"geoId/34011" -"geoId/34013" -"geoId/34015" -"geoId/34017" -"geoId/34019" -"geoId/34021" -"geoId/34023" -"geoId/34025" -"geoId/34027" -"geoId/34029" -"geoId/34031" -"geoId/34033" -"geoId/34035" -"geoId/34037" -"geoId/34039" -"geoId/34041" -"geoId/35001" -"geoId/35005" -"geoId/35006" -"geoId/35009" -"geoId/35013" -"geoId/35015" -"geoId/35017" -"geoId/35025" -"geoId/35027" -"geoId/35028" -"geoId/35029" -"geoId/35031" -"geoId/35035" -"geoId/35039" -"geoId/35041" -"geoId/35043" -"geoId/35045" -"geoId/35047" -"geoId/35049" -"geoId/35053" -"geoId/35055" -"geoId/35057" -"geoId/35061" -"geoId/36001" -"geoId/36003" -"geoId/36005" -"geoId/36007" -"geoId/36009" -"geoId/36011" -"geoId/36013" -"geoId/36015" -"geoId/36017" -"geoId/36019" -"geoId/36021" -"geoId/36023" -"geoId/36025" -"geoId/36027" -"geoId/36029" -"geoId/36031" -"geoId/36033" -"geoId/36035" -"geoId/36037" -"geoId/36039" -"geoId/36043" -"geoId/36045" -"geoId/36047" -"geoId/36049" -"geoId/36051" -"geoId/36053" -"geoId/36055" -"geoId/36057" -"geoId/36059" -"geoId/36061" -"geoId/36063" -"geoId/36065" -"geoId/36067" -"geoId/36069" -"geoId/36071" -"geoId/36073" -"geoId/36075" -"geoId/36077" -"geoId/36079" -"geoId/36081" -"geoId/36083" -"geoId/36085" -"geoId/36087" -"geoId/36089" -"geoId/36091" -"geoId/36093" -"geoId/36095" -"geoId/36097" -"geoId/36099" -"geoId/36101" -"geoId/36103" -"geoId/36105" -"geoId/36107" -"geoId/36109" -"geoId/36111" -"geoId/36113" -"geoId/36115" -"geoId/36117" -"geoId/36119" -"geoId/36121" -"geoId/36123" -"geoId/37001" -"geoId/37003" -"geoId/37007" -"geoId/37009" -"geoId/37011" -"geoId/37013" -"geoId/37015" -"geoId/37017" -"geoId/37019" -"geoId/37021" -"geoId/37023" -"geoId/37025" -"geoId/37027" -"geoId/37031" -"geoId/37033" -"geoId/37035" -"geoId/37037" -"geoId/37039" -"geoId/37045" -"geoId/37047" -"geoId/37049" -"geoId/37051" -"geoId/37053" -"geoId/37055" -"geoId/37057" -"geoId/37059" -"geoId/37061" -"geoId/37063" -"geoId/37065" -"geoId/37067" -"geoId/37069" -"geoId/37071" -"geoId/37077" -"geoId/37079" -"geoId/37081" -"geoId/37083" -"geoId/37085" -"geoId/37087" -"geoId/37089" -"geoId/37091" -"geoId/37093" -"geoId/37097" -"geoId/37099" -"geoId/37101" -"geoId/37105" -"geoId/37107" -"geoId/37109" -"geoId/37111" -"geoId/37113" -"geoId/37115" -"geoId/37117" -"geoId/37119" -"geoId/37123" -"geoId/37125" -"geoId/37127" -"geoId/37129" -"geoId/37131" -"geoId/37133" -"geoId/37135" -"geoId/37139" -"geoId/37141" -"geoId/37145" -"geoId/37147" -"geoId/37149" -"geoId/37151" -"geoId/37153" -"geoId/37155" -"geoId/37157" -"geoId/37159" -"geoId/37161" -"geoId/37163" -"geoId/37165" -"geoId/37167" -"geoId/37169" -"geoId/37171" -"geoId/37175" -"geoId/37179" -"geoId/37181" -"geoId/37183" -"geoId/37185" -"geoId/37189" -"geoId/37191" -"geoId/37193" -"geoId/37195" -"geoId/37197" -"geoId/37199" -"geoId/38015" -"geoId/38017" -"geoId/38035" -"geoId/38059" -"geoId/38077" -"geoId/38089" -"geoId/38093" -"geoId/38101" -"geoId/38105" -"geoId/39001" -"geoId/39003" -"geoId/39005" -"geoId/39007" -"geoId/39009" -"geoId/39011" -"geoId/39013" -"geoId/39015" -"geoId/39017" -"geoId/39019" -"geoId/39021" -"geoId/39023" -"geoId/39025" -"geoId/39027" -"geoId/39029" -"geoId/39031" -"geoId/39033" -"geoId/39035" -"geoId/39037" -"geoId/39039" -"geoId/39041" -"geoId/39043" -"geoId/39045" -"geoId/39047" -"geoId/39049" -"geoId/39051" -"geoId/39053" -"geoId/39055" -"geoId/39057" -"geoId/39059" -"geoId/39061" -"geoId/39063" -"geoId/39065" -"geoId/39067" -"geoId/39069" -"geoId/39071" -"geoId/39073" -"geoId/39075" -"geoId/39077" -"geoId/39079" -"geoId/39081" -"geoId/39083" -"geoId/39085" -"geoId/39087" -"geoId/39089" -"geoId/39091" -"geoId/39093" -"geoId/39095" -"geoId/39097" -"geoId/39099" -"geoId/39101" -"geoId/39103" -"geoId/39105" -"geoId/39107" -"geoId/39109" -"geoId/39111" -"geoId/39113" -"geoId/39117" -"geoId/39119" -"geoId/39123" -"geoId/39125" -"geoId/39127" -"geoId/39129" -"geoId/39131" -"geoId/39133" -"geoId/39135" -"geoId/39137" -"geoId/39139" -"geoId/39141" -"geoId/39143" -"geoId/39145" -"geoId/39147" -"geoId/39149" -"geoId/39151" -"geoId/39153" -"geoId/39155" -"geoId/39157" -"geoId/39159" -"geoId/39161" -"geoId/39165" -"geoId/39167" -"geoId/39169" -"geoId/39171" -"geoId/39173" -"geoId/39175" -"geoId/40001" -"geoId/40009" -"geoId/40013" -"geoId/40015" -"geoId/40017" -"geoId/40019" -"geoId/40021" -"geoId/40023" -"geoId/40027" -"geoId/40031" -"geoId/40037" -"geoId/40039" -"geoId/40041" -"geoId/40047" -"geoId/40049" -"geoId/40051" -"geoId/40065" -"geoId/40071" -"geoId/40079" -"geoId/40081" -"geoId/40083" -"geoId/40087" -"geoId/40089" -"geoId/40091" -"geoId/40095" -"geoId/40097" -"geoId/40101" -"geoId/40109" -"geoId/40111" -"geoId/40113" -"geoId/40115" -"geoId/40119" -"geoId/40121" -"geoId/40123" -"geoId/40125" -"geoId/40131" -"geoId/40133" -"geoId/40135" -"geoId/40137" -"geoId/40139" -"geoId/40143" -"geoId/40145" -"geoId/40147" -"geoId/40149" -"geoId/40153" -"geoId/41003" -"geoId/41005" -"geoId/41007" -"geoId/41009" -"geoId/41011" -"geoId/41013" -"geoId/41015" -"geoId/41017" -"geoId/41019" -"geoId/41027" -"geoId/41029" -"geoId/41031" -"geoId/41033" -"geoId/41035" -"geoId/41039" -"geoId/41041" -"geoId/41043" -"geoId/41045" -"geoId/41047" -"geoId/41051" -"geoId/41053" -"geoId/41057" -"geoId/41059" -"geoId/41061" -"geoId/41065" -"geoId/41067" -"geoId/41071" -"geoId/42001" -"geoId/42003" -"geoId/42005" -"geoId/42007" -"geoId/42009" -"geoId/42011" -"geoId/42013" -"geoId/42015" -"geoId/42017" -"geoId/42019" -"geoId/42021" -"geoId/42025" -"geoId/42027" -"geoId/42029" -"geoId/42031" -"geoId/42033" -"geoId/42035" -"geoId/42037" -"geoId/42039" -"geoId/42041" -"geoId/42043" -"geoId/42045" -"geoId/42047" -"geoId/42049" -"geoId/42051" -"geoId/42055" -"geoId/42059" -"geoId/42061" -"geoId/42063" -"geoId/42065" -"geoId/42067" -"geoId/42069" -"geoId/42071" -"geoId/42073" -"geoId/42075" -"geoId/42077" -"geoId/42079" -"geoId/42081" -"geoId/42083" -"geoId/42085" -"geoId/42087" -"geoId/42089" -"geoId/42091" -"geoId/42093" -"geoId/42095" -"geoId/42097" -"geoId/42099" -"geoId/42101" -"geoId/42103" -"geoId/42105" -"geoId/42107" -"geoId/42109" -"geoId/42111" -"geoId/42115" -"geoId/42117" -"geoId/42119" -"geoId/42121" -"geoId/42123" -"geoId/42125" -"geoId/42127" -"geoId/42129" -"geoId/42131" -"geoId/42133" -"geoId/44001" -"geoId/44003" -"geoId/44005" -"geoId/44007" -"geoId/44009" -"geoId/45001" -"geoId/45003" -"geoId/45007" -"geoId/45009" -"geoId/45011" -"geoId/45013" -"geoId/45015" -"geoId/45019" -"geoId/45021" -"geoId/45023" -"geoId/45025" -"geoId/45027" -"geoId/45029" -"geoId/45031" -"geoId/45033" -"geoId/45035" -"geoId/45037" -"geoId/45039" -"geoId/45041" -"geoId/45043" -"geoId/45045" -"geoId/45047" -"geoId/45049" -"geoId/45051" -"geoId/45053" -"geoId/45055" -"geoId/45057" -"geoId/45059" -"geoId/45061" -"geoId/45063" -"geoId/45067" -"geoId/45069" -"geoId/45071" -"geoId/45073" -"geoId/45075" -"geoId/45077" -"geoId/45079" -"geoId/45081" -"geoId/45083" -"geoId/45085" -"geoId/45087" -"geoId/45089" -"geoId/45091" -"geoId/46005" -"geoId/46011" -"geoId/46013" -"geoId/46029" -"geoId/46035" -"geoId/46065" -"geoId/46081" -"geoId/46083" -"geoId/46093" -"geoId/46099" -"geoId/46103" -"geoId/46127" -"geoId/46135" -"geoId/47001" -"geoId/47003" -"geoId/47009" -"geoId/47011" -"geoId/47013" -"geoId/47017" -"geoId/47019" -"geoId/47021" -"geoId/47023" -"geoId/47025" -"geoId/47029" -"geoId/47031" -"geoId/47035" -"geoId/47037" -"geoId/47041" -"geoId/47043" -"geoId/47045" -"geoId/47047" -"geoId/47049" -"geoId/47051" -"geoId/47053" -"geoId/47055" -"geoId/47057" -"geoId/47059" -"geoId/47063" -"geoId/47065" -"geoId/47069" -"geoId/47071" -"geoId/47073" -"geoId/47075" -"geoId/47077" -"geoId/47079" -"geoId/47081" -"geoId/47085" -"geoId/47089" -"geoId/47091" -"geoId/47093" -"geoId/47097" -"geoId/47099" -"geoId/47103" -"geoId/47105" -"geoId/47107" -"geoId/47109" -"geoId/47111" -"geoId/47113" -"geoId/47115" -"geoId/47117" -"geoId/47119" -"geoId/47123" -"geoId/47125" -"geoId/47129" -"geoId/47131" -"geoId/47133" -"geoId/47139" -"geoId/47141" -"geoId/47143" -"geoId/47145" -"geoId/47147" -"geoId/47149" -"geoId/47151" -"geoId/47153" -"geoId/47155" -"geoId/47157" -"geoId/47159" -"geoId/47163" -"geoId/47165" -"geoId/47167" -"geoId/47171" -"geoId/47173" -"geoId/47177" -"geoId/47179" -"geoId/47181" -"geoId/47183" -"geoId/47185" -"geoId/47187" -"geoId/47189" -"geoId/48001" -"geoId/48003" -"geoId/48005" -"geoId/48007" -"geoId/48013" -"geoId/48015" -"geoId/48019" -"geoId/48021" -"geoId/48025" -"geoId/48027" -"geoId/48029" -"geoId/48035" -"geoId/48037" -"geoId/48039" -"geoId/48041" -"geoId/48049" -"geoId/48051" -"geoId/48053" -"geoId/48055" -"geoId/48057" -"geoId/48061" -"geoId/48067" -"geoId/48071" -"geoId/48073" -"geoId/48085" -"geoId/48089" -"geoId/48091" -"geoId/48097" -"geoId/48099" -"geoId/48113" -"geoId/48117" -"geoId/48121" -"geoId/48123" -"geoId/48133" -"geoId/48135" -"geoId/48139" -"geoId/48141" -"geoId/48143" -"geoId/48145" -"geoId/48147" -"geoId/48149" -"geoId/48157" -"geoId/48161" -"geoId/48163" -"geoId/48165" -"geoId/48167" -"geoId/48171" -"geoId/48177" -"geoId/48179" -"geoId/48181" -"geoId/48183" -"geoId/48185" -"geoId/48187" -"geoId/48189" -"geoId/48199" -"geoId/48201" -"geoId/48203" -"geoId/48209" -"geoId/48213" -"geoId/48215" -"geoId/48217" -"geoId/48219" -"geoId/48221" -"geoId/48223" -"geoId/48225" -"geoId/48227" -"geoId/48231" -"geoId/48233" -"geoId/48241" -"geoId/48245" -"geoId/48249" -"geoId/48251" -"geoId/48253" -"geoId/48257" -"geoId/48259" -"geoId/48265" -"geoId/48273" -"geoId/48277" -"geoId/48279" -"geoId/48281" -"geoId/48285" -"geoId/48287" -"geoId/48289" -"geoId/48291" -"geoId/48293" -"geoId/48299" -"geoId/48303" -"geoId/48309" -"geoId/48321" -"geoId/48323" -"geoId/48325" -"geoId/48329" -"geoId/48331" -"geoId/48337" -"geoId/48339" -"geoId/48341" -"geoId/48347" -"geoId/48349" -"geoId/48353" -"geoId/48355" -"geoId/48361" -"geoId/48363" -"geoId/48365" -"geoId/48367" -"geoId/48371" -"geoId/48373" -"geoId/48375" -"geoId/48381" -"geoId/48389" -"geoId/48395" -"geoId/48397" -"geoId/48401" -"geoId/48407" -"geoId/48409" -"geoId/48415" -"geoId/48419" -"geoId/48423" -"geoId/48427" -"geoId/48439" -"geoId/48441" -"geoId/48449" -"geoId/48451" -"geoId/48453" -"geoId/48457" -"geoId/48459" -"geoId/48463" -"geoId/48465" -"geoId/48467" -"geoId/48469" -"geoId/48471" -"geoId/48473" -"geoId/48477" -"geoId/48479" -"geoId/48481" -"geoId/48485" -"geoId/48489" -"geoId/48491" -"geoId/48493" -"geoId/48497" -"geoId/48499" -"geoId/48503" -"geoId/49003" -"geoId/49005" -"geoId/49007" -"geoId/49011" -"geoId/49013" -"geoId/49021" -"geoId/49035" -"geoId/49039" -"geoId/49041" -"geoId/49043" -"geoId/49045" -"geoId/49047" -"geoId/49049" -"geoId/49051" -"geoId/49053" -"geoId/49057" -"geoId/50001" -"geoId/50003" -"geoId/50005" -"geoId/50007" -"geoId/50011" -"geoId/50015" -"geoId/50017" -"geoId/50019" -"geoId/50021" -"geoId/50023" -"geoId/50025" -"geoId/50027" -"geoId/51001" -"geoId/51003" -"geoId/51005" -"geoId/51009" -"geoId/51013" -"geoId/51015" -"geoId/51019" -"geoId/51023" -"geoId/51025" -"geoId/51027" -"geoId/51029" -"geoId/51031" -"geoId/51033" -"geoId/51035" -"geoId/51041" -"geoId/51047" -"geoId/51051" -"geoId/51053" -"geoId/51059" -"geoId/51061" -"geoId/51065" -"geoId/51067" -"geoId/51069" -"geoId/51071" -"geoId/51073" -"geoId/51075" -"geoId/51077" -"geoId/51079" -"geoId/51083" -"geoId/51085" -"geoId/51087" -"geoId/51089" -"geoId/51093" -"geoId/51095" -"geoId/51099" -"geoId/51101" -"geoId/51105" -"geoId/51107" -"geoId/51109" -"geoId/51117" -"geoId/51121" -"geoId/51127" -"geoId/51137" -"geoId/51139" -"geoId/51141" -"geoId/51143" -"geoId/51145" -"geoId/51147" -"geoId/51149" -"geoId/51153" -"geoId/51155" -"geoId/51161" -"geoId/51163" -"geoId/51165" -"geoId/51167" -"geoId/51169" -"geoId/51171" -"geoId/51173" -"geoId/51175" -"geoId/51177" -"geoId/51179" -"geoId/51185" -"geoId/51187" -"geoId/51191" -"geoId/51193" -"geoId/51195" -"geoId/51197" -"geoId/51199" -"geoId/51510" -"geoId/51520" -"geoId/51540" -"geoId/51550" -"geoId/51570" -"geoId/51590" -"geoId/51600" -"geoId/51630" -"geoId/51650" -"geoId/51660" -"geoId/51670" -"geoId/51680" -"geoId/51683" -"geoId/51685" -"geoId/51690" -"geoId/51700" -"geoId/51710" -"geoId/51730" -"geoId/51740" -"geoId/51750" -"geoId/51760" -"geoId/51770" -"geoId/51775" -"geoId/51790" -"geoId/51800" -"geoId/51810" -"geoId/51820" -"geoId/51840" -"geoId/53001" -"geoId/53003" -"geoId/53005" -"geoId/53007" -"geoId/53009" -"geoId/53011" -"geoId/53015" -"geoId/53017" -"geoId/53021" -"geoId/53025" -"geoId/53027" -"geoId/53029" -"geoId/53031" -"geoId/53033" -"geoId/53035" -"geoId/53037" -"geoId/53039" -"geoId/53041" -"geoId/53045" -"geoId/53047" -"geoId/53049" -"geoId/53053" -"geoId/53055" -"geoId/53057" -"geoId/53061" -"geoId/53063" -"geoId/53065" -"geoId/53067" -"geoId/53071" -"geoId/53073" -"geoId/53075" -"geoId/53077" -"geoId/54003" -"geoId/54005" -"geoId/54009" -"geoId/54011" -"geoId/54019" -"geoId/54025" -"geoId/54027" -"geoId/54029" -"geoId/54033" -"geoId/54035" -"geoId/54037" -"geoId/54039" -"geoId/54041" -"geoId/54043" -"geoId/54045" -"geoId/54047" -"geoId/54049" -"geoId/54051" -"geoId/54053" -"geoId/54055" -"geoId/54057" -"geoId/54059" -"geoId/54061" -"geoId/54065" -"geoId/54067" -"geoId/54069" -"geoId/54077" -"geoId/54079" -"geoId/54081" -"geoId/54083" -"geoId/54091" -"geoId/54097" -"geoId/54099" -"geoId/54103" -"geoId/54107" -"geoId/54109" -"geoId/55001" -"geoId/55003" -"geoId/55005" -"geoId/55009" -"geoId/55013" -"geoId/55015" -"geoId/55017" -"geoId/55019" -"geoId/55021" -"geoId/55023" -"geoId/55025" -"geoId/55027" -"geoId/55029" -"geoId/55031" -"geoId/55033" -"geoId/55035" -"geoId/55039" -"geoId/55043" -"geoId/55045" -"geoId/55047" -"geoId/55049" -"geoId/55053" -"geoId/55055" -"geoId/55057" -"geoId/55059" -"geoId/55061" -"geoId/55063" -"geoId/55065" -"geoId/55067" -"geoId/55069" -"geoId/55071" -"geoId/55073" -"geoId/55075" -"geoId/55079" -"geoId/55081" -"geoId/55083" -"geoId/55085" -"geoId/55087" -"geoId/55089" -"geoId/55093" -"geoId/55095" -"geoId/55097" -"geoId/55101" -"geoId/55103" -"geoId/55105" -"geoId/55109" -"geoId/55111" -"geoId/55113" -"geoId/55115" -"geoId/55117" -"geoId/55119" -"geoId/55121" -"geoId/55123" -"geoId/55125" -"geoId/55127" -"geoId/55131" -"geoId/55133" -"geoId/55135" -"geoId/55137" -"geoId/55139" -"geoId/55141" -"geoId/56001" -"geoId/56005" -"geoId/56007" -"geoId/56013" -"geoId/56021" -"geoId/56023" -"geoId/56025" -"geoId/56029" -"geoId/56033" -"geoId/56037" -"geoId/56039" -"geoId/56041" diff --git a/statvar_imports/us_cdc/county_mortality/golden_data/golden_summary_report.csv b/statvar_imports/us_cdc/county_mortality/golden_data/golden_summary_report.csv deleted file mode 100644 index f0fcaf4892..0000000000 --- a/statvar_imports/us_cdc/county_mortality/golden_data/golden_summary_report.csv +++ /dev/null @@ -1,3 +0,0 @@ -"StatVar","MeasurementMethods","MinDate","observationPeriods","NumPlaces","Units","ScalingFactors" -"Count_MortalityEvent_Assault(Homicide)","[]","2018","[P1Y]","576","[]","[]" -"Count_MortalityEvent_Suicide","[]","2018","[P1Y]","1421","[]","[]" From d75882a01051f793fd85c8d60056a0e167b9778f Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Tue, 8 Sep 2026 16:28:04 +0000 Subject: [PATCH 06/14] cra_fix3 --- statvar_imports/us_cdc/county_mortality/README.md | 1 - statvar_imports/us_cdc/county_mortality/download.py | 2 +- statvar_imports/us_cdc/county_mortality/download_test.py | 4 ++++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index 9ba7fc9607..2ceeec877c 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -53,7 +53,6 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ | `manifest.json` | Automation manifest declaring scripts, inputs/outputs, cron schedule, and validation config. | | `validation_config.json` | Configuration file defining import validation rules (historical deleted records threshold). | | `test_data/` | Trimmed sample Delaware dataset and expected outputs for offline verification. | -| `runbook.md` | Operational runbook, architecture documentation, and troubleshooting guide. | --- diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py index 6aec3eab3f..8b911690fc 100644 --- a/statvar_imports/us_cdc/county_mortality/download.py +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -475,7 +475,7 @@ def is_state_downloaded( with open(single_file, "r", encoding="utf-8", errors="replace") as f: for line in f: for y in years: - if f",{y}," in line: + if line.startswith(f",{y},"): found_years.add(y) if len(found_years) == len(years): return True diff --git a/statvar_imports/us_cdc/county_mortality/download_test.py b/statvar_imports/us_cdc/county_mortality/download_test.py index be3da473b9..104bc81d3f 100644 --- a/statvar_imports/us_cdc/county_mortality/download_test.py +++ b/statvar_imports/us_cdc/county_mortality/download_test.py @@ -163,6 +163,10 @@ def test_is_state_downloaded(self): self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2025"])) + # Create file with only 2018 data, but Deaths column equals 2024 -> should NOT match 2024 + f.write_text("Notes,Year,County,Deaths\n" + ",2018,Kent County,2024\n" * 5) + self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) + # Test partitioned chunk files f.unlink() chunk_2024 = Path(temp_dir) / "UnderlyingCauseofDeath_County_10_2024.csv" From 7ff489c2e71af445914a3378424ec4e6c6ec0212 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Wed, 9 Sep 2026 06:28:19 +0000 Subject: [PATCH 07/14] cra fix 4 --- .../us_cdc/county_mortality/README.md | 9 ++ .../us_cdc/county_mortality/download.py | 85 +++++++++++++------ .../us_cdc/county_mortality/download_test.py | 26 ++++++ 3 files changed, 94 insertions(+), 26 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index 2ceeec877c..d5422708cf 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -11,6 +11,15 @@ This import acquires and processes county-level mortality statistics across all --- +### Prerequisites + +Ensure the required Python libraries are installed: +```bash +pip install requests beautifulsoup4 lxml retry absl-py +``` + +--- + ### Workflow The import consists of two fully automated steps: diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py index 8b911690fc..d06ff95f96 100644 --- a/statvar_imports/us_cdc/county_mortality/download.py +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -457,6 +457,24 @@ def save_tsv_as_csv(raw_tsv: str, output_csv_path: str) -> int: return row_count +def _has_data_rows(file_path: Path) -> bool: + """Checks whether a CSV file contains at least one observation data row beyond the header.""" + if not file_path.exists() or file_path.stat().st_size == 0: + return False + try: + with open(file_path, "r", encoding="utf-8", errors="replace") as f: + reader = csv.reader(f) + header = next(reader, None) + if not header: + return False + for row in reader: + if row and not row[0].startswith("---") and not row[0].startswith("Total"): + return True + return False + except Exception: + return False + + def is_state_downloaded( output_dir: str, state_fips: str, years: Optional[List[str]] = None ) -> bool: @@ -465,33 +483,46 @@ def is_state_downloaded( matches = list(Path(output_dir).glob(pattern)) if not matches: return False - if not all(f.stat().st_size > 100 for f in matches): - return False - if years: - # Check if the single combined file exists and contains all years - single_file = Path(output_dir) / f"UnderlyingCauseofDeath_County_{state_fips}.csv" - if single_file.exists(): - found_years = set() + + # Check if a single combined file exists and contains data rows for all requested years + single_file = Path(output_dir) / f"UnderlyingCauseofDeath_County_{state_fips}.csv" + if single_file.exists(): + found_years = set() + try: with open(single_file, "r", encoding="utf-8", errors="replace") as f: - for line in f: - for y in years: - if line.startswith(f",{y},"): - found_years.add(y) - if len(found_years) == len(years): + reader = csv.reader(f) + header = next(reader, None) + if header: + year_col_idx = 1 + if "Year" in header: + year_col_idx = header.index("Year") + for row in reader: + if len(row) > year_col_idx: + year_val = row[year_col_idx].strip() + if years is None or year_val in years: + found_years.add(year_val) + if years is None and found_years: return True - - # Otherwise, check if every year in years is covered by at least one chunk file - for y in years: - year_covered = False - for f in matches: - if f.name == single_file.name: - continue - if f"_{y}.csv" in f.name or f"_{y}_" in f.name: - year_covered = True - break - if not year_covered: - return False - return True + if years and len(found_years) == len(years): + return True + except Exception: + pass + + # If no specific years requested, check if any matching file has valid data rows + if not years: + return any(_has_data_rows(f) for f in matches) + + # Otherwise, check if every year in years is covered by at least one chunk file with data rows + for y in years: + year_covered = False + for f in matches: + if f.name == single_file.name: + continue + if (f"_{y}.csv" in f.name or f"_{y}_" in f.name) and _has_data_rows(f): + year_covered = True + break + if not year_covered: + return False return True @@ -508,7 +539,6 @@ def download_county_mortality_data( """Downloads CDC County Mortality data for specified states and years.""" os.makedirs(output_dir, exist_ok=True) downloader = CdcWonderCountyMortalityDownloader(timeout=timeout, delay=delay) - downloader.init_session() total_files = 0 total_rows = 0 @@ -530,6 +560,9 @@ def download_county_mortality_data( ) continue + if downloader.action_url is None: + downloader.init_session() + logging.info( "[%d/%d] Processing %s (FIPS %s) (Session batch item %d/%d)...", idx, diff --git a/statvar_imports/us_cdc/county_mortality/download_test.py b/statvar_imports/us_cdc/county_mortality/download_test.py index 104bc81d3f..69e1167da4 100644 --- a/statvar_imports/us_cdc/county_mortality/download_test.py +++ b/statvar_imports/us_cdc/county_mortality/download_test.py @@ -194,6 +194,10 @@ def test_is_state_downloaded(self): self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) f.unlink() + # Chunk file that is header-only (e.g. CDC header without data rows) -> should be False + chunk_2020.write_text("Notes,Year,Year Code,County,County Code,Deaths\n") + self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + @mock.patch.object(download.time, "sleep") @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "init_session") def test_execute_query_429_backoff(self, mock_init, mock_sleep): @@ -260,6 +264,28 @@ def side_effect(state_fips, years): state_11_csv = Path(temp_dir) / "UnderlyingCauseofDeath_County_11.csv" self.assertTrue(state_11_csv.exists()) + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "init_session") + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "download_state") + def test_download_county_mortality_data_lazy_session_skipped( + self, mock_download, mock_init + ): + with tempfile.TemporaryDirectory() as temp_dir: + # Pre-populate state 10 so it is skipped + f = Path(temp_dir) / "UnderlyingCauseofDeath_County_10.csv" + f.write_text("Notes,Year,County,Deaths\n" + ",2024,Kent County,10\n" * 5) + + download.download_county_mortality_data( + states=["10"], + years=["2024"], + output_dir=temp_dir, + skip_existing=True, + ) + + # Assert init_session was NOT called because all states were skipped + mock_init.assert_not_called() + mock_download.assert_not_called() + if __name__ == "__main__": unittest.main() + From e74c25ddbfe877b42bcc7a7370328514dffe0957 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Wed, 9 Sep 2026 09:35:02 +0000 Subject: [PATCH 08/14] Cra fix5 --- .../us_cdc/county_mortality/download.py | 8 ++++++ .../us_cdc/county_mortality/download_test.py | 26 +++++++++++++++++++ 2 files changed, 34 insertions(+) diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py index d06ff95f96..3bbd865479 100644 --- a/statvar_imports/us_cdc/county_mortality/download.py +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -573,6 +573,14 @@ def download_county_mortality_data( batch_size, ) + # Remove any existing/stale chunk or combined files for this state before re-downloading + # to prevent duplicate ingestion by stat_var_processor (which globs input_files/*.csv). + for stale_file in Path(output_dir).glob(f"UnderlyingCauseofDeath_County_{state_fips}*.csv"): + try: + stale_file.unlink() + except OSError as e: + logging.warning("Could not remove stale file %s: %s", stale_file, e) + try: results = downloader.download_state(state_fips, years) for chunk_label, tsv_data in results: diff --git a/statvar_imports/us_cdc/county_mortality/download_test.py b/statvar_imports/us_cdc/county_mortality/download_test.py index 69e1167da4..c72fc88d1d 100644 --- a/statvar_imports/us_cdc/county_mortality/download_test.py +++ b/statvar_imports/us_cdc/county_mortality/download_test.py @@ -285,7 +285,33 @@ def test_download_county_mortality_data_lazy_session_skipped( mock_init.assert_not_called() mock_download.assert_not_called() + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "init_session") + @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "download_state") + def test_download_county_mortality_data_purges_stale_files( + self, mock_download, mock_init + ): + with tempfile.TemporaryDirectory() as temp_dir: + # Pre-create an old chunk file that should be purged when re-downloading + old_chunk = Path(temp_dir) / "UnderlyingCauseofDeath_County_10_2018_2019.csv" + old_chunk.write_text("Notes,Year,County,Deaths\n" + ",2018,Kent County,10\n" * 5) + self.assertTrue(old_chunk.exists()) + + mock_download.return_value = [("all", "Notes\tCounty Code\tDeaths\n\t10001\t25\n")] + + download.download_county_mortality_data( + states=["10"], + years=["2018", "2024"], + output_dir=temp_dir, + skip_existing=False, + ) + + # Assert the old chunk file was deleted and new combined file was written + self.assertFalse(old_chunk.exists()) + combined_file = Path(temp_dir) / "UnderlyingCauseofDeath_County_10.csv" + self.assertTrue(combined_file.exists()) + if __name__ == "__main__": unittest.main() + From 6bd9cc1846e2494b6462ec9798084c45aed18e42 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Wed, 9 Sep 2026 10:08:15 +0000 Subject: [PATCH 09/14] cra fix6 --- .../us_cdc/county_mortality/README.md | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index d5422708cf..2fdbbafa1b 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -51,6 +51,23 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ --- +### Operational Notes & Downloader Architecture + +* **Rate-Limit Handling (HTTP 429)**: CDC WONDER enforces a 30-minute IP block when query thresholds are exceeded. The downloader detects HTTP 429 responses, pauses in complete silence for 31 minutes (`1,860s`), and automatically refreshes the session before resuming. Do not terminate the process during this cooldown. +* **Large State Partitioning**: High-population states (e.g. California `06`, Florida `12`, New York `36`, Texas `48`) exceed CDC WONDER's 75,000-row query cap when queried across all years at once. The downloader automatically queries these states in 2-year chunks (or 1-year chunks for Texas) to prevent HTTP 400 ("Too Much Data") and HTTP 504 timeouts. +* **Stale Partition Cleanup**: When a state is re-downloaded, the downloader purges existing chunk and combined files for that state before writing new data, preventing duplicate observations when `stat_var_processor.py` processes `input_files/*.csv`. +* **Failure Isolation**: If an individual state query encounters network or server errors, the downloader logs the failure, continues with the remaining states, and raises a summary exception at the end of the batch run to ensure non-zero exit code while preserving downloaded progress. +* **Lazy Session Initialization**: The session agreement with CDC WONDER is only initialized when at least one state actually requires downloading, avoiding unnecessary network calls during dry runs or when data is already cached. + +--- + +### Validation + +Validation is configured in `validation_config.json`: +* `check_deleted_records_percent`: Strictly enforces a historical deletion average threshold of `0.1%`. Per consensus on initial imports, golden regression files are omitted from initial submission. + +--- + ### Important Files | File | Description | From 13170bbf5c9372ba87d5b089e11370544187505d Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Thu, 10 Sep 2026 10:55:42 +0000 Subject: [PATCH 10/14] validation config changes --- .../us_cdc/county_mortality/README.md | 9 -- .../us_cdc/county_mortality/download.py | 142 ++++++++++++++---- .../us_cdc/county_mortality/download_test.py | 32 +++- .../us_cdc/county_mortality/manifest.json | 2 +- .../county_mortality/validation_config.json | 20 ++- 5 files changed, 159 insertions(+), 46 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index 2fdbbafa1b..49e5f608b6 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -11,15 +11,6 @@ This import acquires and processes county-level mortality statistics across all --- -### Prerequisites - -Ensure the required Python libraries are installed: -```bash -pip install requests beautifulsoup4 lxml retry absl-py -``` - ---- - ### Workflow The import consists of two fully automated steps: diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py index 3bbd865479..48fa6c7ba9 100644 --- a/statvar_imports/us_cdc/county_mortality/download.py +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -39,7 +39,9 @@ from absl import app, flags, logging from bs4 import BeautifulSoup import requests +from requests.adapters import HTTPAdapter from retry import retry +from urllib3.util import Retry script_dir = os.path.dirname(os.path.abspath(__file__)) DEFAULT_INPUT_DIR = os.path.join(script_dir, "input_files") @@ -165,7 +167,7 @@ def parse_year_list(year_str: str) -> List[str]: class CdcWonderCountyMortalityDownloader: - """Automates CDC WONDER sessions and queries for County-Level Mortality data across all causes.""" + """Automates CDC WONDER sessions and queries for County-Level Mortality data.""" def __init__( self, @@ -176,13 +178,30 @@ def __init__( self.landing_url = landing_url self.timeout = timeout self.delay = delay - self.session = requests.Session() - self.session.headers.update( - {"User-Agent": "Mozilla/5.0 (DataCommons CDC Importer; contact: support@datacommons.org)"} - ) + self.session = self._create_session() self.action_url: Optional[str] = None self.base_post_data: List[Tuple[str, str]] = [] + def _create_session(self) -> requests.Session: + """Creates a requests.Session with connection pooling and HTTP retries.""" + session = requests.Session() + session.headers.update({ + "User-Agent": ( + "Mozilla/5.0 (DataCommons CDC Importer; contact:" + " support@datacommons.org)" + ) + }) + retries = Retry( + total=3, + backoff_factor=1, + status_forcelist=[500, 502, 503, 504], + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retries) + session.mount("https://", adapter) + session.mount("http://", adapter) + return session + @retry( tries=3, delay=5, @@ -190,16 +209,13 @@ def __init__( exceptions=(requests.RequestException, ValueError), ) def init_session(self): - """Connects to landing page, agrees to data use terms, and stores pre-populated form state.""" + """Connects to landing page, agrees to terms, and stores pre-populated form state.""" if hasattr(self, "session") and self.session: try: self.session.close() except Exception: pass - self.session = requests.Session() - self.session.headers.update( - {"User-Agent": "Mozilla/5.0 (DataCommons CDC Importer; contact: support@datacommons.org)"} - ) + self.session = self._create_session() self.action_url = None self.base_post_data = [] @@ -314,11 +330,25 @@ def execute_query( for attempt in range(1, max_retries + 1): try: - res = self.session.post(self.action_url, data=payload, timeout=self.timeout) + logging.info( + "Dispatching HTTP POST to %s for state FIPS %s, years %s (attempt %d/%d)...", + self.action_url, + state_fips, + years, + attempt, + max_retries, + ) + res = self.session.post( + self.action_url, data=payload, timeout=self.timeout + ) if res.status_code == 429: retry_after = res.headers.get("Retry-After") - wait_seconds = int(retry_after) if retry_after and retry_after.isdigit() else 1860 + wait_seconds = ( + int(retry_after) + if retry_after and retry_after.isdigit() + else 1860 + ) logging.warning( "Encountered HTTP 429 (Too Many Requests). CDC WONDER enforces a 30-minute " "IP block. Waiting %d seconds (%d min) in complete silence for block to " @@ -333,9 +363,27 @@ def execute_query( payload = self._build_post_data(state_fips, years) continue - if res.status_code == 400 and ("too much data" in res.text or "simplify your filters" in res.text): + if res.status_code in (401, 403) or ( + res.text + and ( + "session has expired" in res.text.lower() + or "session timeout" in res.text.lower() + ) + ): + logging.warning( + "Session expired or rejected for FIPS %s. Re-initializing session...", + state_fips, + ) + self.init_session() + payload = self._build_post_data(state_fips, years) + continue + + if res.status_code == 400 and ( + "too much data" in res.text or "simplify your filters" in res.text + ): logging.warning( - "FIPS %s query returned 'too much data' (HTTP 400). Returning response for partitioning.", + "FIPS %s query returned 'too much data' (HTTP 400). " + "Returning response for partitioning.", state_fips, ) return res.text @@ -344,14 +392,20 @@ def execute_query( return res.text except requests.RequestException as e: - logging.warning("Query failed for FIPS %s (attempt %d/%d): %s", state_fips, attempt, max_retries, e) + logging.warning( + "Query failed for FIPS %s (attempt %d/%d): %s", + state_fips, + attempt, + max_retries, + e, + ) if attempt == max_retries: raise time.sleep(self.delay * attempt) - self.init_session() - payload = self._build_post_data(state_fips, years) - raise RuntimeError(f"Failed to fetch data for state FIPS {state_fips} after {max_retries} retries.") + raise RuntimeError( + f"Failed to fetch data for state FIPS {state_fips} after {max_retries} retries." + ) def download_state( self, state_fips: str, years: List[str] @@ -362,7 +416,12 @@ def download_state( need_partitioning = state_fips in LARGE_STATES if not need_partitioning: - logging.info("Querying full year range (%s) for state FIPS %s (%s)...", years, state_fips, state_name) + logging.info( + "Querying full year range (%s) for state FIPS %s (%s)...", + years, + state_fips, + state_name, + ) try: tsv_text = self.execute_query(state_fips, years) first_line = tsv_text.splitlines()[0] if tsv_text else "" @@ -371,7 +430,7 @@ def download_state( time.sleep(self.delay) return results logging.warning( - "%s response not TSV (likely exceeded 75k rows / too much data). Partitioning into chunks...", + "%s response not TSV (likely exceeded 75k rows). Partitioning into chunks...", state_name, ) need_partitioning = True @@ -404,13 +463,15 @@ def download_state( success = True else: logging.warning( - "Chunk %s response for %s did not contain valid TSV data. Splitting into single years.", + "Chunk %s response for %s did not contain valid TSV data. " + "Splitting into single years.", chunk_label, state_name, ) except Exception as e: logging.warning( - "Chunk %s for %s failed with %s. Falling back to single-year queries for this state.", + "Chunk %s for %s failed with %s. Falling back to single-year " + "queries for this state.", chunk_label, state_name, e, @@ -429,7 +490,9 @@ def download_state( sy_text = self.execute_query(state_fips, [single_year]) first_line = sy_text.splitlines()[0] if sy_text else "" if "County Code" not in first_line: - raise ValueError(f"Failed to query {state_name} even for single year {single_year}.") + raise ValueError( + f"Failed to query {state_name} even for single year {single_year}." + ) results.append((single_year, sy_text)) i += 1 @@ -549,7 +612,11 @@ def download_county_mortality_data( state_name = US_STATES.get(state_fips, f"FIPS-{state_fips}") if skip_existing and is_state_downloaded(output_dir, state_fips, years=years): - existing_files = list(Path(output_dir).glob(f"UnderlyingCauseofDeath_County_{state_fips}*.csv")) + existing_files = list( + Path(output_dir).glob( + f"UnderlyingCauseofDeath_County_{state_fips}*.csv" + ) + ) logging.info( "[%d/%d] Skipping %s (FIPS %s): %d existing file(s) found.", idx, @@ -600,7 +667,8 @@ def download_county_mortality_data( # Proactive session rotation after batch_size states if states_in_batch >= batch_size and idx < len(states): logging.info( - "Processed batch of %d states. Taking a %.1fs cooldown and refreshing session...", + "Processed batch of %d states. Taking a %.1fs cooldown and refreshing " + "session...", states_in_batch, batch_cooldown, ) @@ -609,12 +677,24 @@ def download_county_mortality_data( states_in_batch = 0 except Exception as e: - logging.error("Failed downloading state %s (FIPS %s): %s", state_name, state_fips, e) + logging.error( + "Failed downloading state %s (FIPS %s): %s", + state_name, + state_fips, + e, + ) failed_states.append(state_name) - logging.info("Download complete: Saved %d files with %d total rows in %s.", total_files, total_rows, output_dir) + logging.info( + "Download complete: Saved %d files with %d total rows in %s.", + total_files, + total_rows, + output_dir, + ) if failed_states: - raise RuntimeError(f"Failed to download data for states: {', '.join(failed_states)}") + raise RuntimeError( + f"Failed to download data for states: {', '.join(failed_states)}" + ) def main(_): @@ -625,7 +705,11 @@ def main(_): else: states = [s.strip().zfill(2) for s in FLAGS.states.split(",") if s.strip()] - logging.info("Starting CDC County Mortality live download for %d states, years: %s", len(states), years) + logging.info( + "Starting CDC County Mortality live download for %d states, years: %s", + len(states), + years, + ) download_county_mortality_data( states=states, years=years, diff --git a/statvar_imports/us_cdc/county_mortality/download_test.py b/statvar_imports/us_cdc/county_mortality/download_test.py index c72fc88d1d..60e8f2a8c2 100644 --- a/statvar_imports/us_cdc/county_mortality/download_test.py +++ b/statvar_imports/us_cdc/county_mortality/download_test.py @@ -159,11 +159,15 @@ def test_is_state_downloaded(self): self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) # Create file with both initial (2018) and latest (2024) -> should be True - f.write_text("Header,col1,col2,col3\n" + ",2018,val2,val3\n" * 5 + ",2024,val2,val3\n" * 5) + f.write_text( + "Header,col1,col2,col3\n" + + ",2018,val2,val3\n" * 5 + + ",2024,val2,val3\n" * 5 + ) self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2025"])) - # Create file with only 2018 data, but Deaths column equals 2024 -> should NOT match 2024 + # Create file with only 2018 data, Deaths column equals 2024 (should not match) f.write_text("Notes,Year,County,Deaths\n" + ",2018,Kent County,2024\n" * 5) self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) @@ -180,23 +184,39 @@ def test_is_state_downloaded(self): self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2024"])) # If intermediate year (e.g. 2021) is requested but missing chunk -> should be False - self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + self.assertFalse( + download.is_state_downloaded( + temp_dir, "10", years=["2018", "2021", "2024"] + ) + ) # Add intermediate chunk covering 2021 -> should now be True chunk_2020 = Path(temp_dir) / "UnderlyingCauseofDeath_County_10_2020_2021.csv" chunk_2020.write_text("Header,col1\n" + "val1,val2\n" * 10) - self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + self.assertTrue( + download.is_state_downloaded( + temp_dir, "10", years=["2018", "2021", "2024"] + ) + ) # If an incomplete single combined file also exists (e.g. from an aborted run), # but chunk files cover all requested years, it should still return True f = Path(temp_dir) / "UnderlyingCauseofDeath_County_10.csv" f.write_text("Header,col1\n,2018,val1\n" * 5) # only contains 2018 - self.assertTrue(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + self.assertTrue( + download.is_state_downloaded( + temp_dir, "10", years=["2018", "2021", "2024"] + ) + ) f.unlink() # Chunk file that is header-only (e.g. CDC header without data rows) -> should be False chunk_2020.write_text("Notes,Year,Year Code,County,County Code,Deaths\n") - self.assertFalse(download.is_state_downloaded(temp_dir, "10", years=["2018", "2021", "2024"])) + self.assertFalse( + download.is_state_downloaded( + temp_dir, "10", years=["2018", "2021", "2024"] + ) + ) @mock.patch.object(download.time, "sleep") @mock.patch.object(download.CdcWonderCountyMortalityDownloader, "init_session") diff --git a/statvar_imports/us_cdc/county_mortality/manifest.json b/statvar_imports/us_cdc/county_mortality/manifest.json index db2e5c594c..e05ca863a6 100644 --- a/statvar_imports/us_cdc/county_mortality/manifest.json +++ b/statvar_imports/us_cdc/county_mortality/manifest.json @@ -26,4 +26,4 @@ "validation_config_file": "validation_config.json" } ] -} \ No newline at end of file +} diff --git a/statvar_imports/us_cdc/county_mortality/validation_config.json b/statvar_imports/us_cdc/county_mortality/validation_config.json index 39a6784647..67fea15313 100644 --- a/statvar_imports/us_cdc/county_mortality/validation_config.json +++ b/statvar_imports/us_cdc/county_mortality/validation_config.json @@ -8,6 +8,24 @@ "params": { "threshold": 0.1 } + }, + { + "rule_id": "check_max_date_freshness", + "description": "Verify MaxDate is 2024 for all StatVars except rare historical exceptions with 2023", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar NOT IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", + "condition": "MaxDate = '2024'" + } + }, + { + "rule_id": "check_max_date_freshness_rare_causes", + "description": "Verify MaxDate is 2023 for rare historical exception causes", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", + "condition": "MaxDate = '2023'" + } } ] -} \ No newline at end of file +} From a73cc5d37be4c81acef705b4902a88061e5392ef Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Thu, 10 Sep 2026 17:26:04 +0000 Subject: [PATCH 11/14] Cra fixes7 --- .../us_cdc/county_mortality/README.md | 6 +- .../county_mortality_metadata.csv | 2 +- .../us_cdc/county_mortality/manifest.json | 4 +- .../underlyingcauseofdeath_county_output.csv | 142 +++++++++--------- .../underlyingcauseofdeath_county_output.tmcf | 1 - .../county_mortality/validation_config.json | 9 +- 6 files changed, 84 insertions(+), 80 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index 49e5f608b6..4300d0d039 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -55,7 +55,9 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ ### Validation Validation is configured in `validation_config.json`: -* `check_deleted_records_percent`: Strictly enforces a historical deletion average threshold of `0.1%`. Per consensus on initial imports, golden regression files are omitted from initial submission. +* `check_deleted_records_percent`: Strictly enforces a historical deletion average threshold of `0.1%`. Note that on the initial import run, this rule expectedly yields `DATA_ERROR` ("Differ summary is missing required field: 'previous_obs_count'") because there is no prior version to diff against; it is configured to safeguard future recurring refreshes. Per consensus on initial imports, golden regression files are omitted from initial submission. +* `check_max_date_freshness`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= 2024`) across 116 cause-of-death StatVars to ensure latest published data is ingested. +* `check_max_date_freshness_rare_causes`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= 2023`) for 2 rare historical exception causes with low observation frequency due to CDC sub-national cell suppression (< 10 deaths). --- @@ -68,7 +70,7 @@ Validation is configured in `validation_config.json`: | `county_mortality_metadata.csv` | Metadata specifying header row offsets, frequency, and output columns. | | `county_mortality_pvmap.csv` | Property-Value mapping resolving county FIPS and ICD-10 113 causes of death. | | `manifest.json` | Automation manifest declaring scripts, inputs/outputs, cron schedule, and validation config. | -| `validation_config.json` | Configuration file defining import validation rules (historical deleted records threshold). | +| `validation_config.json` | Configuration defining import validation rules (historical deletion and date freshness). | | `test_data/` | Trimmed sample Delaware dataset and expected outputs for offline verification. | --- diff --git a/statvar_imports/us_cdc/county_mortality/county_mortality_metadata.csv b/statvar_imports/us_cdc/county_mortality/county_mortality_metadata.csv index 5ebf2e3a4c..6c4034eeab 100644 --- a/statvar_imports/us_cdc/county_mortality/county_mortality_metadata.csv +++ b/statvar_imports/us_cdc/county_mortality/county_mortality_metadata.csv @@ -5,7 +5,7 @@ description,"County-level mortality statistics across all causes of death (ICD-1 start_date,2018 end_date,2024 release_frequency,P1Y -output_columns,"observationAbout,observationDate,variableMeasured,value,unit" +output_columns,"observationAbout,observationDate,variableMeasured,value" mapped_rows,1 mapped_columns,11 header_rows,1 diff --git a/statvar_imports/us_cdc/county_mortality/manifest.json b/statvar_imports/us_cdc/county_mortality/manifest.json index e05ca863a6..58413bc073 100644 --- a/statvar_imports/us_cdc/county_mortality/manifest.json +++ b/statvar_imports/us_cdc/county_mortality/manifest.json @@ -20,10 +20,12 @@ ], "source_files": [ "input_files/*.csv", - "counters/*.csv" + "counters/*.csv", + "validation_config.json" ], "cron_schedule": "0 5 * * 5", "validation_config_file": "validation_config.json" } ] } + diff --git a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv index 9d7b6dd21b..5efdba34ed 100644 --- a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv +++ b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.csv @@ -1,71 +1,71 @@ -observationAbout,observationDate,variableMeasured,value,unit -geoId/10001,2023,dcid:Count_MortalityEvent_Septicemia,26, -geoId/10001,2023,dcid:Count_MortalityEvent_OtherAndUnspecifiedInfectiousAndParasiticDiseasesAndTheirSequelae,42, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasms,407, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfColonRectumAndAnus,28, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLiverAndIntrahepaticBileDucts,18, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfPancreas,31, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfTracheaBronchusAndLung,100, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfBreast,36, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfProstate,14, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfBladder,20, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfMeningesBrainAndOtherPartsOfCentralNervousSystem,10, -geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLymphoidHematopoieticAndRelatedTissue,38, -geoId/10001,2023,dcid:Count_MortalityEvent_Non-HodgkinsLymphoma,15, -geoId/10001,2023,dcid:Count_MortalityEvent_Leukemia,18, -geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherAndUnspecifiedMalignantNeoplasms,57, -geoId/10001,2023,dcid:Count_MortalityEvent_InSituNeoplasmsBenignNeoplasmsAndNeoplasmsOfUncertainOrUnknownBehavior,12, -geoId/10001,2023,dcid:Count_MortalityEvent_DiabetesMellitus,78, -geoId/10001,2023,dcid:Count_MortalityEvent_NutritionalDeficiencies,11, -geoId/10001,2023,dcid:Count_MortalityEvent_Malnutrition,11, -geoId/10001,2023,dcid:Count_MortalityEvent_ParkinsonsDisease,16, -geoId/10001,2023,dcid:Count_MortalityEvent_AlzheimersDisease,72, -geoId/10001,2023,dcid:Count_MortalityEvent_MajorCardiovascularDiseases,612, -geoId/10001,2023,dcid:Count_MortalityEvent_DiseasesOfHeart,416, -geoId/10001,2023,dcid:Count_MortalityEvent_HypertensiveHeartDisease,51, -geoId/10001,2023,dcid:Count_MortalityEvent_IschemicHeartDiseases,223, -geoId/10001,2023,dcid:Count_MortalityEvent_AcuteMyocardialInfarction,33, -geoId/10001,2023,dcid:Count_MortalityEvent_OtherFormsOfChronicIschemicHeartDisease,187, -geoId/10001,2023,dcid:Count_MortalityEvent_AtheroscleroticCardiovascularDiseaseSoDescribed,67, -geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherFormsOfChronicIschemicHeartDisease,120, -geoId/10001,2023,dcid:Count_MortalityEvent_OtherHeartDiseases,128, -geoId/10001,2023,dcid:Count_MortalityEvent_HeartFailure,37, -geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherFormsOfHeartDisease,91, -geoId/10001,2023,dcid:Count_MortalityEvent_Essential(Primary)HypertensionAndHypertensiveRenalDisease,17, -geoId/10001,2023,dcid:Count_MortalityEvent_CerebrovascularDiseases,167, -geoId/10001,2023,dcid:Count_MortalityEvent_OtherDiseasesOfCirculatorySystem,11, -geoId/10001,2024,dcid:Count_MortalityEvent_Septicemia,37, -geoId/10001,2024,dcid:Count_MortalityEvent_OtherAndUnspecifiedInfectiousAndParasiticDiseasesAndTheirSequelae,38, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasms,420, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLipOralCavityAndPharynx,12, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfEsophagus,11, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfColonRectumAndAnus,32, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLiverAndIntrahepaticBileDucts,20, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfPancreas,38, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfTracheaBronchusAndLung,96, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfBreast,29, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfCorpusUteriAndUterusPartUnspecified,13, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfProstate,12, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfKidneyAndRenalPelvis,11, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfBladder,12, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfMeningesBrainAndOtherPartsOfCentralNervousSystem,11, -geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLymphoidHematopoieticAndRelatedTissue,46, -geoId/10001,2024,dcid:Count_MortalityEvent_Non-HodgkinsLymphoma,16, -geoId/10001,2024,dcid:Count_MortalityEvent_Leukemia,19, -geoId/10001,2024,dcid:Count_MortalityEvent_MultipleMyelomaAndImmunoproliferativeNeoplasms,11, -geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherAndUnspecifiedMalignantNeoplasms,50, -geoId/10001,2024,dcid:Count_MortalityEvent_InSituNeoplasmsBenignNeoplasmsAndNeoplasmsOfUncertainOrUnknownBehavior,14, -geoId/10001,2024,dcid:Count_MortalityEvent_DiabetesMellitus,75, -geoId/10001,2024,dcid:Count_MortalityEvent_ParkinsonsDisease,30, -geoId/10001,2024,dcid:Count_MortalityEvent_AlzheimersDisease,65, -geoId/10001,2024,dcid:Count_MortalityEvent_MajorCardiovascularDiseases,650, -geoId/10001,2024,dcid:Count_MortalityEvent_DiseasesOfHeart,444, -geoId/10001,2024,dcid:Count_MortalityEvent_HypertensiveHeartDisease,44, -geoId/10001,2024,dcid:Count_MortalityEvent_IschemicHeartDiseases,245, -geoId/10001,2024,dcid:Count_MortalityEvent_AcuteMyocardialInfarction,34, -geoId/10001,2024,dcid:Count_MortalityEvent_OtherFormsOfChronicIschemicHeartDisease,207, -geoId/10001,2024,dcid:Count_MortalityEvent_AtheroscleroticCardiovascularDiseaseSoDescribed,64, -geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherFormsOfChronicIschemicHeartDisease,143, -geoId/10001,2024,dcid:Count_MortalityEvent_OtherHeartDiseases,144, -geoId/10001,2024,dcid:Count_MortalityEvent_HeartFailure,45, -geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherFormsOfHeartDisease,96, +observationAbout,observationDate,variableMeasured,value +geoId/10001,2023,dcid:Count_MortalityEvent_Septicemia,26 +geoId/10001,2023,dcid:Count_MortalityEvent_OtherAndUnspecifiedInfectiousAndParasiticDiseasesAndTheirSequelae,42 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasms,407 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfColonRectumAndAnus,28 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLiverAndIntrahepaticBileDucts,18 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfPancreas,31 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfTracheaBronchusAndLung,100 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfBreast,36 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfProstate,14 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmOfBladder,20 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfMeningesBrainAndOtherPartsOfCentralNervousSystem,10 +geoId/10001,2023,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLymphoidHematopoieticAndRelatedTissue,38 +geoId/10001,2023,dcid:Count_MortalityEvent_Non-HodgkinsLymphoma,15 +geoId/10001,2023,dcid:Count_MortalityEvent_Leukemia,18 +geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherAndUnspecifiedMalignantNeoplasms,57 +geoId/10001,2023,dcid:Count_MortalityEvent_InSituNeoplasmsBenignNeoplasmsAndNeoplasmsOfUncertainOrUnknownBehavior,12 +geoId/10001,2023,dcid:Count_MortalityEvent_DiabetesMellitus,78 +geoId/10001,2023,dcid:Count_MortalityEvent_NutritionalDeficiencies,11 +geoId/10001,2023,dcid:Count_MortalityEvent_Malnutrition,11 +geoId/10001,2023,dcid:Count_MortalityEvent_ParkinsonsDisease,16 +geoId/10001,2023,dcid:Count_MortalityEvent_AlzheimersDisease,72 +geoId/10001,2023,dcid:Count_MortalityEvent_MajorCardiovascularDiseases,612 +geoId/10001,2023,dcid:Count_MortalityEvent_DiseasesOfHeart,416 +geoId/10001,2023,dcid:Count_MortalityEvent_HypertensiveHeartDisease,51 +geoId/10001,2023,dcid:Count_MortalityEvent_IschemicHeartDiseases,223 +geoId/10001,2023,dcid:Count_MortalityEvent_AcuteMyocardialInfarction,33 +geoId/10001,2023,dcid:Count_MortalityEvent_OtherFormsOfChronicIschemicHeartDisease,187 +geoId/10001,2023,dcid:Count_MortalityEvent_AtheroscleroticCardiovascularDiseaseSoDescribed,67 +geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherFormsOfChronicIschemicHeartDisease,120 +geoId/10001,2023,dcid:Count_MortalityEvent_OtherHeartDiseases,128 +geoId/10001,2023,dcid:Count_MortalityEvent_HeartFailure,37 +geoId/10001,2023,dcid:Count_MortalityEvent_AllOtherFormsOfHeartDisease,91 +geoId/10001,2023,dcid:Count_MortalityEvent_Essential(Primary)HypertensionAndHypertensiveRenalDisease,17 +geoId/10001,2023,dcid:Count_MortalityEvent_CerebrovascularDiseases,167 +geoId/10001,2023,dcid:Count_MortalityEvent_OtherDiseasesOfCirculatorySystem,11 +geoId/10001,2024,dcid:Count_MortalityEvent_Septicemia,37 +geoId/10001,2024,dcid:Count_MortalityEvent_OtherAndUnspecifiedInfectiousAndParasiticDiseasesAndTheirSequelae,38 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasms,420 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLipOralCavityAndPharynx,12 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfEsophagus,11 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfColonRectumAndAnus,32 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLiverAndIntrahepaticBileDucts,20 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfPancreas,38 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfTracheaBronchusAndLung,96 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfBreast,29 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfCorpusUteriAndUterusPartUnspecified,13 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfProstate,12 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfKidneyAndRenalPelvis,11 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmOfBladder,12 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfMeningesBrainAndOtherPartsOfCentralNervousSystem,11 +geoId/10001,2024,dcid:Count_MortalityEvent_MalignantNeoplasmsOfLymphoidHematopoieticAndRelatedTissue,46 +geoId/10001,2024,dcid:Count_MortalityEvent_Non-HodgkinsLymphoma,16 +geoId/10001,2024,dcid:Count_MortalityEvent_Leukemia,19 +geoId/10001,2024,dcid:Count_MortalityEvent_MultipleMyelomaAndImmunoproliferativeNeoplasms,11 +geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherAndUnspecifiedMalignantNeoplasms,50 +geoId/10001,2024,dcid:Count_MortalityEvent_InSituNeoplasmsBenignNeoplasmsAndNeoplasmsOfUncertainOrUnknownBehavior,14 +geoId/10001,2024,dcid:Count_MortalityEvent_DiabetesMellitus,75 +geoId/10001,2024,dcid:Count_MortalityEvent_ParkinsonsDisease,30 +geoId/10001,2024,dcid:Count_MortalityEvent_AlzheimersDisease,65 +geoId/10001,2024,dcid:Count_MortalityEvent_MajorCardiovascularDiseases,650 +geoId/10001,2024,dcid:Count_MortalityEvent_DiseasesOfHeart,444 +geoId/10001,2024,dcid:Count_MortalityEvent_HypertensiveHeartDisease,44 +geoId/10001,2024,dcid:Count_MortalityEvent_IschemicHeartDiseases,245 +geoId/10001,2024,dcid:Count_MortalityEvent_AcuteMyocardialInfarction,34 +geoId/10001,2024,dcid:Count_MortalityEvent_OtherFormsOfChronicIschemicHeartDisease,207 +geoId/10001,2024,dcid:Count_MortalityEvent_AtheroscleroticCardiovascularDiseaseSoDescribed,64 +geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherFormsOfChronicIschemicHeartDisease,143 +geoId/10001,2024,dcid:Count_MortalityEvent_OtherHeartDiseases,144 +geoId/10001,2024,dcid:Count_MortalityEvent_HeartFailure,45 +geoId/10001,2024,dcid:Count_MortalityEvent_AllOtherFormsOfHeartDisease,96 diff --git a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.tmcf b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.tmcf index a6afa2944d..684cf5999b 100644 --- a/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.tmcf +++ b/statvar_imports/us_cdc/county_mortality/test_data/underlyingcauseofdeath_county_output.tmcf @@ -3,6 +3,5 @@ observationAbout: C:underlyingcauseofdeath_county_output->observationAbout observationDate: C:underlyingcauseofdeath_county_output->observationDate variableMeasured: C:underlyingcauseofdeath_county_output->variableMeasured value: C:underlyingcauseofdeath_county_output->value -unit: C:underlyingcauseofdeath_county_output->unit typeOf: dcs:StatVarObservation observationPeriod: P1Y diff --git a/statvar_imports/us_cdc/county_mortality/validation_config.json b/statvar_imports/us_cdc/county_mortality/validation_config.json index 67fea15313..a38302e778 100644 --- a/statvar_imports/us_cdc/county_mortality/validation_config.json +++ b/statvar_imports/us_cdc/county_mortality/validation_config.json @@ -11,21 +11,22 @@ }, { "rule_id": "check_max_date_freshness", - "description": "Verify MaxDate is 2024 for all StatVars except rare historical exceptions with 2023", + "description": "Verify MaxDate is at least 2024 for all StatVars except rare historical exceptions with 2023", "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar NOT IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", - "condition": "MaxDate = '2024'" + "condition": "CAST(MaxDate AS INTEGER) >= 2024" } }, { "rule_id": "check_max_date_freshness_rare_causes", - "description": "Verify MaxDate is 2023 for rare historical exception causes", + "description": "Verify MaxDate is at least 2023 for rare historical exception causes", "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", - "condition": "MaxDate = '2023'" + "condition": "CAST(MaxDate AS INTEGER) >= 2023" } } ] } + From 0e29c393b56ed0fe8653fc06d1013c7dee58ac83 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Fri, 11 Sep 2026 08:16:24 +0000 Subject: [PATCH 12/14] cra --- .../us_cdc/county_mortality/README.md | 4 +- .../us_cdc/county_mortality/download.py | 71 ++++++++++++++++++- .../us_cdc/county_mortality/download_test.py | 61 ++++++++++++++++ .../county_mortality/validation_config.json | 8 +-- 4 files changed, 136 insertions(+), 8 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index 4300d0d039..cb7c25c3c5 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -56,8 +56,8 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ Validation is configured in `validation_config.json`: * `check_deleted_records_percent`: Strictly enforces a historical deletion average threshold of `0.1%`. Note that on the initial import run, this rule expectedly yields `DATA_ERROR` ("Differ summary is missing required field: 'previous_obs_count'") because there is no prior version to diff against; it is configured to safeguard future recurring refreshes. Per consensus on initial imports, golden regression files are omitted from initial submission. -* `check_max_date_freshness`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= 2024`) across 116 cause-of-death StatVars to ensure latest published data is ingested. -* `check_max_date_freshness_rare_causes`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= 2023`) for 2 rare historical exception causes with low observation frequency due to CDC sub-national cell suppression (< 10 deaths). +* `check_max_date_freshness`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 2)`) across 116 cause-of-death StatVars to ensure latest published data is ingested relative to annual publication lag. +* `check_max_date_freshness_rare_causes`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 3)`) for 2 rare historical exception causes with low observation frequency due to CDC sub-national cell suppression (< 10 deaths). --- diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py index 48fa6c7ba9..ab32521ab0 100644 --- a/statvar_imports/us_cdc/county_mortality/download.py +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -29,6 +29,7 @@ """ import csv +import datetime import io import os from pathlib import Path @@ -113,6 +114,7 @@ SINGLE_YEAR_STATES: set[str] = {"48"} FLAGS = flags.FLAGS +CURRENT_YEAR = datetime.date.today().year flags.DEFINE_string( "states", @@ -121,8 +123,8 @@ ) flags.DEFINE_string( "years", - "2018-2024", - "Year range ('2018-2024') or comma-separated years ('2018,2019,2020').", + f"2018-{CURRENT_YEAR}", + f"Year range (e.g. '2018-{CURRENT_YEAR}') or comma-separated years ('2018,2019,2020').", ) flags.DEFINE_string( "output_dir", @@ -181,6 +183,7 @@ def __init__( self.session = self._create_session() self.action_url: Optional[str] = None self.base_post_data: List[Tuple[str, str]] = [] + self.available_years: List[str] = [] def _create_session(self) -> requests.Session: """Creates a requests.Session with connection pooling and HTTP retries.""" @@ -275,8 +278,44 @@ def init_session(self): elif el.name == "textarea": self.base_post_data.append((name, el.text or "")) + # Discover available year options from the year selection dropdown (F_D158.V1) + self.available_years = [] + year_select = form_req.find("select", attrs={"name": "F_D158.V1"}) + if year_select: + for opt in year_select.find_all("option"): + val = opt.get("value", "").strip() + if val.isdigit(): + self.available_years.append(val) + if self.available_years: + logging.info( + "Discovered available years on CDC WONDER: %s", self.available_years + ) + logging.info("Successfully established CDC WONDER session with action: %s", self.action_url) + def filter_available_years(self, years: List[str]) -> List[str]: + """Filters requested years against available years discovered on CDC WONDER.""" + if not self.available_years: + return years + valid_years = [y for y in years if y in self.available_years] + if not valid_years: + logging.warning( + "None of the requested years %s exist on CDC WONDER (available: %s). " + "Falling back to all available years.", + years, + self.available_years, + ) + return list(self.available_years) + if len(valid_years) < len(years): + dropped = [y for y in years if y not in self.available_years] + logging.info( + "Filtered out unsupported years %s (not yet published on CDC WONDER). " + "Querying: %s", + dropped, + valid_years, + ) + return valid_years + def _build_post_data( self, state_fips: str, years: Optional[List[str]] = None ) -> List[Tuple[str, str]]: @@ -412,6 +451,8 @@ def download_state( ) -> List[Tuple[str, str]]: """Downloads county mortality data for a state, automatically partitioning if needed.""" state_name = US_STATES.get(state_fips, f"FIPS-{state_fips}") + if self.available_years: + years = self.filter_available_years(years) results: List[Tuple[str, str]] = [] need_partitioning = state_fips in LARGE_STATES @@ -516,6 +557,14 @@ def save_tsv_as_csv(raw_tsv: str, output_csv_path: str) -> int: csv_writer.writerow(row) row_count += 1 + if row_count <= 1: + if os.path.exists(temp_csv_path): + os.unlink(temp_csv_path) + raise ValueError( + f"Extracted TSV for {output_csv_path} contains no observation data " + f"(row_count={row_count})." + ) + os.replace(temp_csv_path, output_csv_path) return row_count @@ -629,6 +678,24 @@ def download_county_mortality_data( if downloader.action_url is None: downloader.init_session() + years = downloader.filter_available_years(years) + if skip_existing and is_state_downloaded( + output_dir, state_fips, years=years + ): + existing_files = list( + Path(output_dir).glob( + f"UnderlyingCauseofDeath_County_{state_fips}*.csv" + ) + ) + logging.info( + "[%d/%d] Skipping %s (FIPS %s): %d existing file(s) found.", + idx, + len(states), + state_name, + state_fips, + len(existing_files), + ) + continue logging.info( "[%d/%d] Processing %s (FIPS %s) (Session batch item %d/%d)...", diff --git a/statvar_imports/us_cdc/county_mortality/download_test.py b/statvar_imports/us_cdc/county_mortality/download_test.py index 60e8f2a8c2..a600aec767 100644 --- a/statvar_imports/us_cdc/county_mortality/download_test.py +++ b/statvar_imports/us_cdc/county_mortality/download_test.py @@ -16,10 +16,13 @@ import os from pathlib import Path +import sys import tempfile import unittest from unittest import mock +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) + import requests import download @@ -145,6 +148,15 @@ def test_save_tsv_as_csv(self): self.assertEqual(lines[0], "Notes,Year,County,County Code,Deaths") self.assertEqual(lines[1], ',2018,"Kent County, DE",10001,20') + def test_save_tsv_as_csv_empty_raises(self): + header_only_tsv = "Notes\tYear\tCounty\tCounty Code\tDeaths\n---\nCaveats:\n" + with tempfile.TemporaryDirectory() as temp_dir: + output_csv = os.path.join(temp_dir, "test_output.csv") + with self.assertRaises(ValueError): + download.save_tsv_as_csv(header_only_tsv, output_csv) + self.assertFalse(os.path.exists(output_csv)) + self.assertFalse(os.path.exists(f"{output_csv}.tmp")) + def test_is_state_downloaded(self): with tempfile.TemporaryDirectory() as temp_dir: self.assertFalse(download.is_state_downloaded(temp_dir, "10")) @@ -330,6 +342,55 @@ def test_download_county_mortality_data_purges_stale_files( combined_file = Path(temp_dir) / "UnderlyingCauseofDeath_County_10.csv" self.assertTrue(combined_file.exists()) + def test_filter_available_years(self): + downloader = download.CdcWonderCountyMortalityDownloader() + downloader.available_years = ["2018", "2019", "2020", "2021", "2022", "2023", "2024"] + + # Filters out unreleased future years + requested = ["2018", "2019", "2020", "2021", "2022", "2023", "2024", "2025", "2026"] + filtered = downloader.filter_available_years(requested) + self.assertEqual( + filtered, + ["2018", "2019", "2020", "2021", "2022", "2023", "2024"], + ) + + # Subset passes through untouched + subset = ["2020", "2021"] + self.assertEqual(downloader.filter_available_years(subset), ["2020", "2021"]) + + # No available_years means no filtering + downloader.available_years = [] + self.assertEqual(downloader.filter_available_years(requested), requested) + + @mock.patch.object(download.requests, "Session") + def test_init_session_discovers_available_years(self, mock_session_cls): + mock_session = mock.MagicMock() + mock_session_cls.return_value = mock_session + + mock_res1 = mock.MagicMock() + mock_res1.text = '
' + mock_res1.raise_for_status.return_value = None + + mock_res2 = mock.MagicMock() + mock_res2.text = """ +
+ +
+ """ + mock_res2.raise_for_status.return_value = None + + mock_session.get.return_value = mock_res1 + mock_session.post.return_value = mock_res2 + + downloader = download.CdcWonderCountyMortalityDownloader() + downloader.init_session() + self.assertEqual(downloader.available_years, ["2018", "2019", "2020"]) + if __name__ == "__main__": unittest.main() diff --git a/statvar_imports/us_cdc/county_mortality/validation_config.json b/statvar_imports/us_cdc/county_mortality/validation_config.json index a38302e778..aea23556b6 100644 --- a/statvar_imports/us_cdc/county_mortality/validation_config.json +++ b/statvar_imports/us_cdc/county_mortality/validation_config.json @@ -11,20 +11,20 @@ }, { "rule_id": "check_max_date_freshness", - "description": "Verify MaxDate is at least 2024 for all StatVars except rare historical exceptions with 2023", + "description": "Verify MaxDate is within allowable 2-year publication lag for all StatVars except rare historical exceptions", "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar NOT IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", - "condition": "CAST(MaxDate AS INTEGER) >= 2024" + "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 2)" } }, { "rule_id": "check_max_date_freshness_rare_causes", - "description": "Verify MaxDate is at least 2023 for rare historical exception causes", + "description": "Verify MaxDate is within allowable 3-year publication lag for rare historical exception causes", "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", - "condition": "CAST(MaxDate AS INTEGER) >= 2023" + "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 3)" } } ] From ebcd2d514f751200d24b9bb6077a8e625ae0e763 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Fri, 11 Sep 2026 09:50:17 +0000 Subject: [PATCH 13/14] namefix --- statvar_imports/us_cdc/county_mortality/README.md | 4 ++-- statvar_imports/us_cdc/county_mortality/manifest.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index cb7c25c3c5..a09e3c8891 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -1,8 +1,8 @@ -### CDC WONDER County-Level Mortality Across All Causes (`CDC_Mortality_Count`) +### CDC WONDER County-Level Mortality Across All Causes (`CDC_Mortality_UnderlyingCause`) This import acquires and processes county-level mortality statistics across all causes of death (ICD-10 113 Cause List) for all 50 US States and Washington D.C. from the CDC WONDER database. -- **Import Name**: `CDC_Mortality_Count` +- **Import Name**: `CDC_Mortality_UnderlyingCause` - **Source Database**: CDC WONDER Underlying Cause of Death (Database D158) - **Source URL**: `https://wonder.cdc.gov/ucd-icd10-expanded.html` - **Geographic Granularity**: County level (all ~3,143 US counties across all states) diff --git a/statvar_imports/us_cdc/county_mortality/manifest.json b/statvar_imports/us_cdc/county_mortality/manifest.json index 58413bc073..2a3312039f 100644 --- a/statvar_imports/us_cdc/county_mortality/manifest.json +++ b/statvar_imports/us_cdc/county_mortality/manifest.json @@ -1,7 +1,7 @@ { "import_specifications": [ { - "import_name": "CDC_Mortality_Count", + "import_name": "CDC_Mortality_UnderlyingCause", "curator_emails": [ "support@datacommons.org" ], From 5ce4105106705f47e8f6959c4885926f26005d77 Mon Sep 17 00:00:00 2001 From: goutamabrol Date: Fri, 11 Sep 2026 12:48:51 +0000 Subject: [PATCH 14/14] crafix --- .../us_cdc/county_mortality/README.md | 4 +- .../us_cdc/county_mortality/download.py | 61 ++++++++++--------- .../county_mortality/validation_config.json | 8 +-- 3 files changed, 38 insertions(+), 35 deletions(-) diff --git a/statvar_imports/us_cdc/county_mortality/README.md b/statvar_imports/us_cdc/county_mortality/README.md index a09e3c8891..2d94b88e6a 100644 --- a/statvar_imports/us_cdc/county_mortality/README.md +++ b/statvar_imports/us_cdc/county_mortality/README.md @@ -56,8 +56,8 @@ python3 ../../../tools/statvar_importer/stat_var_processor.py \ Validation is configured in `validation_config.json`: * `check_deleted_records_percent`: Strictly enforces a historical deletion average threshold of `0.1%`. Note that on the initial import run, this rule expectedly yields `DATA_ERROR` ("Differ summary is missing required field: 'previous_obs_count'") because there is no prior version to diff against; it is configured to safeguard future recurring refreshes. Per consensus on initial imports, golden regression files are omitted from initial submission. -* `check_max_date_freshness`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 2)`) across 116 cause-of-death StatVars to ensure latest published data is ingested relative to annual publication lag. -* `check_max_date_freshness_rare_causes`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 3)`) for 2 rare historical exception causes with low observation frequency due to CDC sub-national cell suppression (< 10 deaths). +* `check_max_date_freshness`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 3)`) across 116 cause-of-death StatVars to ensure latest published data is ingested relative to annual publication lag. +* `check_max_date_freshness_rare_causes`: Enforces date freshness (`CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 4)`) for 2 rare historical exception causes with low observation frequency due to CDC sub-national cell suppression (< 10 deaths). --- diff --git a/statvar_imports/us_cdc/county_mortality/download.py b/statvar_imports/us_cdc/county_mortality/download.py index ab32521ab0..94cf8f1036 100644 --- a/statvar_imports/us_cdc/county_mortality/download.py +++ b/statvar_imports/us_cdc/county_mortality/download.py @@ -638,6 +638,33 @@ def is_state_downloaded( return True +def _should_skip_state( + output_dir: str, + state_fips: str, + years: List[str], + state_name: str, + idx: int, + total_states: int, +) -> bool: + """Checks if state files already exist on disk and logs skip message.""" + if not is_state_downloaded(output_dir, state_fips, years=years): + return False + existing_files = list( + Path(output_dir).glob( + f"UnderlyingCauseofDeath_County_{state_fips}*.csv" + ) + ) + logging.info( + "[%d/%d] Skipping %s (FIPS %s): %d existing file(s) found.", + idx, + total_states, + state_name, + state_fips, + len(existing_files), + ) + return True + + def download_county_mortality_data( states: List[str], years: List[str], @@ -660,41 +687,17 @@ def download_county_mortality_data( for idx, state_fips in enumerate(states, start=1): state_name = US_STATES.get(state_fips, f"FIPS-{state_fips}") - if skip_existing and is_state_downloaded(output_dir, state_fips, years=years): - existing_files = list( - Path(output_dir).glob( - f"UnderlyingCauseofDeath_County_{state_fips}*.csv" - ) - ) - logging.info( - "[%d/%d] Skipping %s (FIPS %s): %d existing file(s) found.", - idx, - len(states), - state_name, - state_fips, - len(existing_files), - ) + if skip_existing and _should_skip_state( + output_dir, state_fips, years, state_name, idx, len(states) + ): continue if downloader.action_url is None: downloader.init_session() years = downloader.filter_available_years(years) - if skip_existing and is_state_downloaded( - output_dir, state_fips, years=years + if skip_existing and _should_skip_state( + output_dir, state_fips, years, state_name, idx, len(states) ): - existing_files = list( - Path(output_dir).glob( - f"UnderlyingCauseofDeath_County_{state_fips}*.csv" - ) - ) - logging.info( - "[%d/%d] Skipping %s (FIPS %s): %d existing file(s) found.", - idx, - len(states), - state_name, - state_fips, - len(existing_files), - ) continue logging.info( diff --git a/statvar_imports/us_cdc/county_mortality/validation_config.json b/statvar_imports/us_cdc/county_mortality/validation_config.json index aea23556b6..08a3dec889 100644 --- a/statvar_imports/us_cdc/county_mortality/validation_config.json +++ b/statvar_imports/us_cdc/county_mortality/validation_config.json @@ -11,20 +11,20 @@ }, { "rule_id": "check_max_date_freshness", - "description": "Verify MaxDate is within allowable 2-year publication lag for all StatVars except rare historical exceptions", + "description": "Verify MaxDate is within allowable 3-year publication lag for all StatVars except rare historical exceptions", "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar NOT IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", - "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 2)" + "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 3)" } }, { "rule_id": "check_max_date_freshness_rare_causes", - "description": "Verify MaxDate is within allowable 3-year publication lag for rare historical exception causes", + "description": "Verify MaxDate is within allowable 4-year publication lag for rare historical exception causes", "validator": "SQL_VALIDATOR", "params": { "query": "SELECT StatVar, MaxDate FROM stats WHERE StatVar IN ('Count_MortalityEvent_ChronicGlomerulonephritisNephritisAndNephritisNotSpecifiedAsAcuteOrChronicAndRenalSclerosisUnspecified', 'Count_MortalityEvent_AccidentalDischargeOfFirearms')", - "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 3)" + "condition": "CAST(MaxDate AS INTEGER) >= (EXTRACT(YEAR FROM CURRENT_DATE) - 4)" } } ]