diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md index eb6e1e8ea1..b2947f2286 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md @@ -19,6 +19,7 @@ python3 stat_var_processor.py --input_data='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/' --pv_map='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --config_file='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' +--output_counters='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' ``` #### Download the data: @@ -40,6 +41,7 @@ python3 stat_var_processor.py --pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_pvmap.csv --config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_output_counters.csv ``` ``` @@ -48,5 +50,6 @@ python3 stat_var_processor.py --pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_age_pvmap.csv --config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_age_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_age_output_counters.csv ``` diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py new file mode 100644 index 0000000000..7064b359d3 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py @@ -0,0 +1,82 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit and regression tests for commerce_ntia statvar import.""" + +import os +import subprocess +import sys +import tempfile +import unittest + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_DATA_DIR = os.path.abspath(os.path.join(_SCRIPT_DIR, '../../../')) +_TOOLS_DIR = os.path.join(_DATA_DIR, 'tools/statvar_importer') +sys.path.insert(0, _TOOLS_DIR) +sys.path.insert(0, os.path.join(_DATA_DIR, 'util')) +from counters import Counters +from mcf_diff import diff_mcf_files + + +class CommerceNtiaTest(unittest.TestCase): + + def setUp(self): + self.testdata_dir = os.path.join(_SCRIPT_DIR, 'testdata') + self.processor_path = os.path.join(_TOOLS_DIR, 'stat_var_processor.py') + self.pv_map = os.path.join(_SCRIPT_DIR, 'ntia_pvmap.csv') + self.metadata = os.path.join(_SCRIPT_DIR, 'ntia_metadata.csv') + + def test_stat_var_processor_ntia_output(self): + """Tests that stat_var_processor generates expected outputs for ntia-data.csv.""" + with tempfile.TemporaryDirectory() as tmp_dir: + output_path = os.path.join(tmp_dir, 'ntia_output') + cmd = [ + sys.executable, + self.processor_path, + f'--input_data={os.path.join(self.testdata_dir, "ntia-data.csv")}', + f'--pv_map={self.pv_map}', + f'--config_file={self.metadata}', + f'--output_path={output_path}', + ] + res = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(res.returncode, 0, + f'Processor failed: {res.stderr}') + + # Verify CSV + gen_csv = os.path.join(tmp_dir, 'ntia_output.csv') + exp_csv = os.path.join(self.testdata_dir, 'ntia_output.csv') + with open(gen_csv, + encoding='utf-8') as g, open(exp_csv, + encoding='utf-8') as e: + self.assertEqual(g.read().strip(), e.read().strip()) + + # Verify TMCF + gen_tmcf = os.path.join(tmp_dir, 'ntia_output.tmcf') + exp_tmcf = os.path.join(self.testdata_dir, 'ntia_output.tmcf') + with open(gen_tmcf, + encoding='utf-8') as g, open(exp_tmcf, + encoding='utf-8') as e: + self.assertEqual(g.read().strip(), e.read().strip()) + + # Verify StatVar MCF + gen_mcf = os.path.join(tmp_dir, 'ntia_output_stat_vars.mcf') + exp_mcf = os.path.join(self.testdata_dir, + 'ntia_output_stat_vars.mcf') + counters = Counters() + diff = diff_mcf_files(gen_mcf, exp_mcf, + {'show_diff_nodes_only': True}, counters) + self.assertEqual(len(diff), 0, f'MCF diff found: {diff}') + + +if __name__ == '__main__': + unittest.main() diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json index 2906cb50c3..beebb9d9c7 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json @@ -9,20 +9,23 @@ "provenance_description": "NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users.", "scripts": [ "preprocess.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output --output_counters=counters/ntia_output_counters.csv", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output --output_counters=counters/ntia_age_output_counters.csv" ], "source_files": [ - "input_files/ntia-analyze-table.csv" + "input_files/ntia-analyze-table.csv", + "counters/*.csv" ], "import_inputs": [ { "template_mcf": "output_files/ntia_output.tmcf", - "cleaned_csv": "output_files/ntia_output.csv" + "cleaned_csv": "output_files/ntia_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/ntia_age_output.tmcf", - "cleaned_csv": "output_files/ntia_age_output.csv" + "cleaned_csv": "output_files/ntia_age_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 06 * * 5" diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv index f9286c33c4..a29c91a193 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv @@ -7,7 +7,7 @@ CivilPerson,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,, internetAnywhere,populationType,Household,isInternetUser,True,internetUsageLocation,AnyLocation,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} internetAtHome,populationType,Household,isInternetUser,True,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} noInternetAtHome,populationType,Household,isInternetUser,False,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} -adultInternetUser,populationType,Household,,,isInternetUser,True,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +adultInternetUser,populationType,Person,Age,[15 - Years],isInternetUser,True,race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},,,, ispBundle,populationType,Household,internetSubscriptionType,InternetBundle,,,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} ,,,,,,,,,,,,,,,, ,,,,,,,,,,,,,,,, @@ -91,7 +91,7 @@ Uses the Internet at Someone Else's Home,#ignore,Ignore,,,,,,,,,,,,,, Browses the Web,DescriptionVar,Browses the Web,isInternetUser,True,internetUsagePurpose,WebBrowsingUsage,,,,,,,,,, Interacts with Household Equipment Using the Internet,#ignore,Ignore,,,,,,,,,,,,,, Participates in Online Video or Voice Calls or Conferencing,DescriptionVar,Participates in Online Video or Voice Calls or Conferencing,isInternetUser,True,internetUsagePurpose,VideoCallsUsage__AudioCallsUsage__ConferencingUsage,,,,,,,,,, -"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,StreamingUsage__MediaDownloadUsage,,,,,,,,,, +"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,MediaDownloadUsage__StreamingUsage,,,,,,,,,, Uses Health Monitoring Service that Connects to the Internet,DescriptionVar,Uses Health Monitoring Service that Connects to the Internet,isInternetUser,True,internetUsagePurpose,HealthMonitoringServicesUsage,,,,,,,,,, Uses Online Social Networks,DescriptionVar,Uses Online Social Networks,isInternetUser,True,internetUsagePurpose,SocialNetworksUsage,,,,,,,,,, Mobile Data Plan Used from Any Location,DescriptionVar,Mobile Data Plan Used from Any Location,isInternetUser,True,internetSubscriptionType,CellularDataPlan,,,,,,,,,, @@ -118,18 +118,18 @@ age1524Count,observationAbout,country/USA,measuredProperty,count,Age,[15 24 Year age2544Count,observationAbout,country/USA,measuredProperty,count,Age,[25 44 Years],value,{Number},name,"""{DescriptionVar}, age between 25 and 44 years""",,,,,, age4564Count,observationAbout,country/USA,measuredProperty,count,Age,[45 64 Years],value,{Number},name,"""{DescriptionVar}, age between 45 and 64 years""",,,,,, age65pCount,observationAbout,country/USA,measuredProperty,count,Age,[65 - Years],value,{Number},name,"""{DescriptionVar}, age 65 years and above""",,,,,, -workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",houseHolderAge,"""""" -workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",houseHolderAge,"""""" -workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",houseHolderAge,"""""" +workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",householderAge,"""""" +workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",householderAge,"""""" +workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",householderAge,"""""" incomeU25Count,observationAbout,country/USA,measuredProperty,count,income,[- 25000 USDollar],value,{Number},name,"""{DescriptionVar}, income under 25000 USD""",,,,,, income2549Count,observationAbout,country/USA,measuredProperty,count,income,[25000 49000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 25000 and 49000 USD""",,,,,, income5074Count,observationAbout,country/USA,measuredProperty,count,income,[50000 74000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 50000 and 74000 USD""",,,,,, income7599Count,observationAbout,country/USA,measuredProperty,count,income,[75000 99000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 75000 and 99000 USD""",,,,,, income100pCount,observationAbout,country/USA,measuredProperty,count,income,[100000 - USDollar],value,{Number},name,"""{DescriptionVar}, income above 100000 USD""",,,,,, -edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",houseHolderAge,"""""" -edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" -edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",houseHolderAge,"""""" -edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" +edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",householderAge,"""""" +edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",householderAge,"""""" +edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",householderAge,"""""" +edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",householderAge,"""""" sexMaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Male,value,{Number},name,"""{DescriptionVar}, male""",,,,,, sexFemaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Female,value,{Number},name,"""{DescriptionVar}, female""",,,,,, raceWhiteCount,observationAbout,country/USA,measuredProperty,count,Race,WhiteAloneNotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, White non-Hispanic""",,,,,, diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv index 8797927dc2..2cc9b8fd17 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv @@ -7,7 +7,7 @@ CivilPerson,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,, internetAnywhere,populationType,Household,isInternetUser,True,internetUsageLocation,AnyLocation,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} internetAtHome,populationType,Household,isInternetUser,True,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} noInternetAtHome,populationType,Household,isInternetUser,False,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} -adultInternetUser,populationType,Household,,,isInternetUser,True,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +adultInternetUser,populationType,Person,Age,[15 - Years],isInternetUser,True,race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},,,, ispBundle,populationType,Household,internetSubscriptionType,InternetBundle,,,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} ,,,,,,,,,,,,,,,, ,,,,,,,,,,,,,,,, @@ -91,7 +91,7 @@ Uses the Internet at Someone Else's Home,#ignore,Ignore,,,,,,,,,,,,,, Browses the Web,DescriptionVar,Browses the Web,isInternetUser,True,internetUsagePurpose,WebBrowsingUsage,,,,,,,,,, Interacts with Household Equipment Using the Internet,#ignore,Ignore,,,,,,,,,,,,,, Participates in Online Video or Voice Calls or Conferencing,DescriptionVar,Participates in Online Video or Voice Calls or Conferencing,isInternetUser,True,internetUsagePurpose,VideoCallsUsage__AudioCallsUsage__ConferencingUsage,,,,,,,,,, -"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,StreamingUsage__MediaDownloadUsage,,,,,,,,,, +"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,MediaDownloadUsage__StreamingUsage,,,,,,,,,, Uses Health Monitoring Service that Connects to the Internet,DescriptionVar,Uses Health Monitoring Service that Connects to the Internet,isInternetUser,True,internetUsagePurpose,HealthMonitoringServicesUsage,,,,,,,,,, Uses Online Social Networks,DescriptionVar,Uses Online Social Networks,isInternetUser,True,internetUsagePurpose,SocialNetworksUsage,,,,,,,,,, Mobile Data Plan Used from Any Location,DescriptionVar,Mobile Data Plan Used from Any Location,isInternetUser,True,internetSubscriptionType,CellularDataPlan,,,,,,,,,, @@ -118,18 +118,18 @@ age1524Count,observationAbout,country/USA,measuredProperty,count,Age,[15 24 Year age2544Count,observationAbout,country/USA,measuredProperty,count,Age,[25 44 Years],value,{Number},name,"""{DescriptionVar}, age between 25 and 44 years""",,,,,, age4564Count,observationAbout,country/USA,measuredProperty,count,Age,[45 64 Years],value,{Number},name,"""{DescriptionVar}, age between 45 and 64 years""",,,,,, age65pCount,observationAbout,country/USA,measuredProperty,count,Age,[65 - Years],value,{Number},name,"""{DescriptionVar}, age 65 years and above""",,,,,, -workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",houseHolderAge,"""""" -workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",houseHolderAge,"""""" -workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",houseHolderAge,"""""" +workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",householderAge,"""""" +workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",householderAge,"""""" +workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",householderAge,"""""" incomeU25Count,observationAbout,country/USA,measuredProperty,count,income,[- 25000 USDollar],value,{Number},name,"""{DescriptionVar}, income under 25000 USD""",,,,,, income2549Count,observationAbout,country/USA,measuredProperty,count,income,[25000 49000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 25000 and 49000 USD""",,,,,, income5074Count,observationAbout,country/USA,measuredProperty,count,income,[50000 74000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 50000 and 74000 USD""",,,,,, income7599Count,observationAbout,country/USA,measuredProperty,count,income,[75000 99000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 75000 and 99000 USD""",,,,,, income100pCount,observationAbout,country/USA,measuredProperty,count,income,[100000 - USDollar],value,{Number},name,"""{DescriptionVar}, income above 100000 USD""",,,,,, -edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",houseHolderAge,"""""" -edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" -edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",houseHolderAge,"""""" -edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" +edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",householderAge,"""""" +edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",householderAge,"""""" +edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",householderAge,"""""" +edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",householderAge,"""""" sexMaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Male,value,{Number},name,"""{DescriptionVar}, male""",,,,,, sexFemaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Female,value,{Number},name,"""{DescriptionVar}, female""",,,,,, raceWhiteCount,observationAbout,country/USA,measuredProperty,count,Race,WhiteAloneNotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, White non-Hispanic""",,,,,, diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 9bcf786a14..d5ea296137 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -12,10 +12,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, sys +import os +import sys import pandas as pd from absl import app, logging -from pathlib import Path import config script_dir = os.path.dirname(os.path.abspath(__file__)) @@ -27,15 +27,21 @@ Commerce_NTIA_URL = config.Commerce_NTIA_URL INPUT_DIR = os.path.join(script_dir, "input_files") -Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) - COMMON_COLUMNS = ["dataset", "variable", "description", "universe"] -AGE_COLUMNS = ["age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount"] +AGE_COLUMNS = [ + "age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount" +] INPUT_FILE = os.path.join(INPUT_DIR, "ntia-analyze-table.csv") INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") INPUT_FILE_2 = os.path.join(INPUT_DIR, "ntia-data.csv") +HEADERS = { + 'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +} + def move_column_left(df, column_to_move, target_column): """Moves the universe column to the left of variable column.""" @@ -50,45 +56,58 @@ def move_column_left(df, column_to_move, target_column): def preprocess_data(): try: + os.makedirs(INPUT_DIR, exist_ok=True) org_df = pd.read_csv(INPUT_FILE) - + df1 = org_df[COMMON_COLUMNS + AGE_COLUMNS].copy() df1['universeAgeResol'] = df1['universe'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df1['variableAgeResol'] = df1['variable'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df1_moved = move_column_left(df1, 'universe', 'variable') df1_moved.to_csv(INPUT_FILE_1, index=False) - df2_cols_to_keep = [col for col in org_df.columns if not col.startswith('age')] + df2_cols_to_keep = [ + col for col in org_df.columns if not col.startswith('age') + ] df2 = org_df[df2_cols_to_keep].copy() df2['universeAgeResol'] = df2['universe'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df2['variableAgeResol'] = df2['variable'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df2_moved = move_column_left(df2, 'universe', 'variable') df2_moved.to_csv(INPUT_FILE_2, index=False) except Exception as e: - logging.fatal(f"An error occurred while preprocessing the input data: {e}") - return None + logging.fatal( + f"An error occurred while preprocessing the input data: {e}") + sys.exit(1) + def main(argv): try: - download_file(url=Commerce_NTIA_URL, - output_folder=INPUT_DIR, - unzip=False, - headers= None, - tries= 3, - delay= 5, - backoff= 2) + success = download_file( + url=Commerce_NTIA_URL, + output_folder=INPUT_DIR, + unzip=False, + headers=HEADERS, + tries=3, + delay=5, + backoff=2, + ) + if not success or not os.path.exists(INPUT_FILE): + logging.fatal("Failed to download Commerce_NTIA file.") + sys.exit(1) except Exception as e: logging.fatal(f"Failed to download Commerce_NTIA file: {e}") + sys.exit(1) + preprocess_data() + if __name__ == "__main__": app.run(main)