From 1f02f1bf2f32378cfb8a791d8e92f870d0b5d62e Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Tue, 25 Aug 2026 09:49:43 +0000 Subject: [PATCH 1/3] Changes in preproces.py to remove SSL and add header to downlaod --- .../commerce_ntia/manifest.json | 7 +- .../commerce_ntia/preprocess.py | 76 ++++++++++++++++--- 2 files changed, 68 insertions(+), 15 deletions(-) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json index 2906cb50c3..9ae9709ef6 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json @@ -9,11 +9,12 @@ "provenance_description": "NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users.", "scripts": [ "preprocess.py", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output", - "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output" + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output --output_counters=counters/ntia_output_counters.csv", + "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output --output_counters=counters/ntia_age_output_counters.csv" ], "source_files": [ - "input_files/ntia-analyze-table.csv" + "input_files/ntia-analyze-table.csv", + "counters/*.csv" ], "import_inputs": [ { diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 9bcf786a14..23b7f0de15 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -12,17 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, sys +import os, sys, time import pandas as pd +import requests +import urllib3 from absl import app, logging from pathlib import Path import config -script_dir = os.path.dirname(os.path.abspath(__file__)) - -sys.path.append(os.path.join(script_dir, '../../../util')) +# Suppress InsecureRequestWarning when verify=False is used +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) -from download_util_script import download_file +script_dir = os.path.dirname(os.path.abspath(__file__)) Commerce_NTIA_URL = config.Commerce_NTIA_URL @@ -36,6 +37,53 @@ INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") INPUT_FILE_2 = os.path.join(INPUT_DIR, "ntia-data.csv") +HEADERS = { + 'User-Agent': ( + 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' + ), + 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', +} + + +def download_dataset(url: str, + output_file: str, + tries: int = 3, + delay: int = 5, + backoff: int = 2) -> bool: + """Downloads dataset with browser headers, SSL verification disabled, and retries.""" + current_delay = delay + for attempt in range(1, tries + 1): + try: + logging.info( + f"Attempt {attempt}/{tries}: Downloading from {url} (verify=False)" + ) + response = requests.get( + url, + headers=HEADERS, + verify=False, + timeout=60, + stream=True, + ) + response.raise_for_status() + with open(output_file, "wb") as f: + for chunk in response.iter_content(chunk_size=8192): + f.write(chunk) + logging.info(f"Successfully downloaded dataset to {output_file}") + return True + except Exception as e: + logging.warning(f"Download attempt {attempt} failed: {e}") + if os.path.exists(output_file): + try: + os.remove(output_file) + except OSError: + pass + if attempt < tries: + time.sleep(current_delay) + current_delay *= backoff + logging.error(f"All {tries} download attempts failed for {url}") + return False + def move_column_left(df, column_to_move, target_column): """Moves the universe column to the left of variable column.""" @@ -79,15 +127,19 @@ def preprocess_data(): def main(argv): try: - download_file(url=Commerce_NTIA_URL, - output_folder=INPUT_DIR, - unzip=False, - headers= None, - tries= 3, - delay= 5, - backoff= 2) + success = download_dataset( + url=Commerce_NTIA_URL, + output_file=INPUT_FILE, + tries=3, + delay=5, + backoff=2, + ) + if not success or not os.path.exists(INPUT_FILE): + logging.fatal("Failed to download Commerce_NTIA file.") + sys.exit(1) except Exception as e: logging.fatal(f"Failed to download Commerce_NTIA file: {e}") + sys.exit(1) preprocess_data() if __name__ == "__main__": From 9b89b6c4b49a53bf936333ccc4d51639df526f9c Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Tue, 25 Aug 2026 11:54:55 +0000 Subject: [PATCH 2/3] Modifications in process script --- .../commerce_ntia/preprocess.py | 63 +++++-------------- 1 file changed, 14 insertions(+), 49 deletions(-) diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 23b7f0de15..177e78f2fa 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -12,19 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, sys, time +import os, sys +from pathlib import Path import pandas as pd -import requests -import urllib3 from absl import app, logging -from pathlib import Path import config -# Suppress InsecureRequestWarning when verify=False is used -urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) - script_dir = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(os.path.join(script_dir, '../../../util')) + +from download_util_script import download_file + Commerce_NTIA_URL = config.Commerce_NTIA_URL INPUT_DIR = os.path.join(script_dir, "input_files") @@ -46,45 +45,6 @@ } -def download_dataset(url: str, - output_file: str, - tries: int = 3, - delay: int = 5, - backoff: int = 2) -> bool: - """Downloads dataset with browser headers, SSL verification disabled, and retries.""" - current_delay = delay - for attempt in range(1, tries + 1): - try: - logging.info( - f"Attempt {attempt}/{tries}: Downloading from {url} (verify=False)" - ) - response = requests.get( - url, - headers=HEADERS, - verify=False, - timeout=60, - stream=True, - ) - response.raise_for_status() - with open(output_file, "wb") as f: - for chunk in response.iter_content(chunk_size=8192): - f.write(chunk) - logging.info(f"Successfully downloaded dataset to {output_file}") - return True - except Exception as e: - logging.warning(f"Download attempt {attempt} failed: {e}") - if os.path.exists(output_file): - try: - os.remove(output_file) - except OSError: - pass - if attempt < tries: - time.sleep(current_delay) - current_delay *= backoff - logging.error(f"All {tries} download attempts failed for {url}") - return False - - def move_column_left(df, column_to_move, target_column): """Moves the universe column to the left of variable column.""" cols = df.columns.tolist() @@ -123,13 +83,16 @@ def preprocess_data(): except Exception as e: logging.fatal(f"An error occurred while preprocessing the input data: {e}") - return None + sys.exit(1) + def main(argv): try: - success = download_dataset( + success = download_file( url=Commerce_NTIA_URL, - output_file=INPUT_FILE, + output_folder=INPUT_DIR, + unzip=False, + headers=HEADERS, tries=3, delay=5, backoff=2, @@ -140,7 +103,9 @@ def main(argv): except Exception as e: logging.fatal(f"Failed to download Commerce_NTIA file: {e}") sys.exit(1) + preprocess_data() + if __name__ == "__main__": app.run(main) From fddcf934770fbb8dedd5136e09b9dfab9ba7281e Mon Sep 17 00:00:00 2001 From: Krishnam Maheshwari Date: Tue, 8 Sep 2026 12:24:58 +0000 Subject: [PATCH 3/3] Chnages --- .../commerce_ntia/README.md | 3 + .../commerce_ntia/commerce_ntia_test.py | 82 +++++++++++++++++++ .../commerce_ntia/manifest.json | 6 +- .../commerce_ntia/ntia_age_pvmap.csv | 18 ++-- .../commerce_ntia/ntia_pvmap.csv | 18 ++-- .../commerce_ntia/preprocess.py | 42 +++++----- 6 files changed, 129 insertions(+), 40 deletions(-) create mode 100644 statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md index eb6e1e8ea1..b2947f2286 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md @@ -19,6 +19,7 @@ python3 stat_var_processor.py --input_data='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/' --pv_map='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --config_file='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' +--output_counters='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA//' ``` #### Download the data: @@ -40,6 +41,7 @@ python3 stat_var_processor.py --pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_pvmap.csv --config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_output_counters.csv ``` ``` @@ -48,5 +50,6 @@ python3 stat_var_processor.py --pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_age_pvmap.csv --config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_age_output +--output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_age_output_counters.csv ``` diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py new file mode 100644 index 0000000000..7064b359d3 --- /dev/null +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py @@ -0,0 +1,82 @@ +# Copyright 2025 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Unit and regression tests for commerce_ntia statvar import.""" + +import os +import subprocess +import sys +import tempfile +import unittest + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +_DATA_DIR = os.path.abspath(os.path.join(_SCRIPT_DIR, '../../../')) +_TOOLS_DIR = os.path.join(_DATA_DIR, 'tools/statvar_importer') +sys.path.insert(0, _TOOLS_DIR) +sys.path.insert(0, os.path.join(_DATA_DIR, 'util')) +from counters import Counters +from mcf_diff import diff_mcf_files + + +class CommerceNtiaTest(unittest.TestCase): + + def setUp(self): + self.testdata_dir = os.path.join(_SCRIPT_DIR, 'testdata') + self.processor_path = os.path.join(_TOOLS_DIR, 'stat_var_processor.py') + self.pv_map = os.path.join(_SCRIPT_DIR, 'ntia_pvmap.csv') + self.metadata = os.path.join(_SCRIPT_DIR, 'ntia_metadata.csv') + + def test_stat_var_processor_ntia_output(self): + """Tests that stat_var_processor generates expected outputs for ntia-data.csv.""" + with tempfile.TemporaryDirectory() as tmp_dir: + output_path = os.path.join(tmp_dir, 'ntia_output') + cmd = [ + sys.executable, + self.processor_path, + f'--input_data={os.path.join(self.testdata_dir, "ntia-data.csv")}', + f'--pv_map={self.pv_map}', + f'--config_file={self.metadata}', + f'--output_path={output_path}', + ] + res = subprocess.run(cmd, capture_output=True, text=True) + self.assertEqual(res.returncode, 0, + f'Processor failed: {res.stderr}') + + # Verify CSV + gen_csv = os.path.join(tmp_dir, 'ntia_output.csv') + exp_csv = os.path.join(self.testdata_dir, 'ntia_output.csv') + with open(gen_csv, + encoding='utf-8') as g, open(exp_csv, + encoding='utf-8') as e: + self.assertEqual(g.read().strip(), e.read().strip()) + + # Verify TMCF + gen_tmcf = os.path.join(tmp_dir, 'ntia_output.tmcf') + exp_tmcf = os.path.join(self.testdata_dir, 'ntia_output.tmcf') + with open(gen_tmcf, + encoding='utf-8') as g, open(exp_tmcf, + encoding='utf-8') as e: + self.assertEqual(g.read().strip(), e.read().strip()) + + # Verify StatVar MCF + gen_mcf = os.path.join(tmp_dir, 'ntia_output_stat_vars.mcf') + exp_mcf = os.path.join(self.testdata_dir, + 'ntia_output_stat_vars.mcf') + counters = Counters() + diff = diff_mcf_files(gen_mcf, exp_mcf, + {'show_diff_nodes_only': True}, counters) + self.assertEqual(len(diff), 0, f'MCF diff found: {diff}') + + +if __name__ == '__main__': + unittest.main() diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json index 9ae9709ef6..beebb9d9c7 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json @@ -19,11 +19,13 @@ "import_inputs": [ { "template_mcf": "output_files/ntia_output.tmcf", - "cleaned_csv": "output_files/ntia_output.csv" + "cleaned_csv": "output_files/ntia_output.csv", + "node_mcf": "output_files/*.mcf" }, { "template_mcf": "output_files/ntia_age_output.tmcf", - "cleaned_csv": "output_files/ntia_age_output.csv" + "cleaned_csv": "output_files/ntia_age_output.csv", + "node_mcf": "output_files/*.mcf" } ], "cron_schedule": "0 06 * * 5" diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv index f9286c33c4..a29c91a193 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv @@ -7,7 +7,7 @@ CivilPerson,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,, internetAnywhere,populationType,Household,isInternetUser,True,internetUsageLocation,AnyLocation,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} internetAtHome,populationType,Household,isInternetUser,True,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} noInternetAtHome,populationType,Household,isInternetUser,False,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} -adultInternetUser,populationType,Household,,,isInternetUser,True,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +adultInternetUser,populationType,Person,Age,[15 - Years],isInternetUser,True,race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},,,, ispBundle,populationType,Household,internetSubscriptionType,InternetBundle,,,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} ,,,,,,,,,,,,,,,, ,,,,,,,,,,,,,,,, @@ -91,7 +91,7 @@ Uses the Internet at Someone Else's Home,#ignore,Ignore,,,,,,,,,,,,,, Browses the Web,DescriptionVar,Browses the Web,isInternetUser,True,internetUsagePurpose,WebBrowsingUsage,,,,,,,,,, Interacts with Household Equipment Using the Internet,#ignore,Ignore,,,,,,,,,,,,,, Participates in Online Video or Voice Calls or Conferencing,DescriptionVar,Participates in Online Video or Voice Calls or Conferencing,isInternetUser,True,internetUsagePurpose,VideoCallsUsage__AudioCallsUsage__ConferencingUsage,,,,,,,,,, -"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,StreamingUsage__MediaDownloadUsage,,,,,,,,,, +"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,MediaDownloadUsage__StreamingUsage,,,,,,,,,, Uses Health Monitoring Service that Connects to the Internet,DescriptionVar,Uses Health Monitoring Service that Connects to the Internet,isInternetUser,True,internetUsagePurpose,HealthMonitoringServicesUsage,,,,,,,,,, Uses Online Social Networks,DescriptionVar,Uses Online Social Networks,isInternetUser,True,internetUsagePurpose,SocialNetworksUsage,,,,,,,,,, Mobile Data Plan Used from Any Location,DescriptionVar,Mobile Data Plan Used from Any Location,isInternetUser,True,internetSubscriptionType,CellularDataPlan,,,,,,,,,, @@ -118,18 +118,18 @@ age1524Count,observationAbout,country/USA,measuredProperty,count,Age,[15 24 Year age2544Count,observationAbout,country/USA,measuredProperty,count,Age,[25 44 Years],value,{Number},name,"""{DescriptionVar}, age between 25 and 44 years""",,,,,, age4564Count,observationAbout,country/USA,measuredProperty,count,Age,[45 64 Years],value,{Number},name,"""{DescriptionVar}, age between 45 and 64 years""",,,,,, age65pCount,observationAbout,country/USA,measuredProperty,count,Age,[65 - Years],value,{Number},name,"""{DescriptionVar}, age 65 years and above""",,,,,, -workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",houseHolderAge,"""""" -workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",houseHolderAge,"""""" -workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",houseHolderAge,"""""" +workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",householderAge,"""""" +workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",householderAge,"""""" +workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",householderAge,"""""" incomeU25Count,observationAbout,country/USA,measuredProperty,count,income,[- 25000 USDollar],value,{Number},name,"""{DescriptionVar}, income under 25000 USD""",,,,,, income2549Count,observationAbout,country/USA,measuredProperty,count,income,[25000 49000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 25000 and 49000 USD""",,,,,, income5074Count,observationAbout,country/USA,measuredProperty,count,income,[50000 74000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 50000 and 74000 USD""",,,,,, income7599Count,observationAbout,country/USA,measuredProperty,count,income,[75000 99000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 75000 and 99000 USD""",,,,,, income100pCount,observationAbout,country/USA,measuredProperty,count,income,[100000 - USDollar],value,{Number},name,"""{DescriptionVar}, income above 100000 USD""",,,,,, -edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",houseHolderAge,"""""" -edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" -edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",houseHolderAge,"""""" -edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" +edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",householderAge,"""""" +edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",householderAge,"""""" +edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",householderAge,"""""" +edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",householderAge,"""""" sexMaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Male,value,{Number},name,"""{DescriptionVar}, male""",,,,,, sexFemaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Female,value,{Number},name,"""{DescriptionVar}, female""",,,,,, raceWhiteCount,observationAbout,country/USA,measuredProperty,count,Race,WhiteAloneNotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, White non-Hispanic""",,,,,, diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv index 8797927dc2..2cc9b8fd17 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_pvmap.csv @@ -7,7 +7,7 @@ CivilPerson,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,, internetAnywhere,populationType,Household,isInternetUser,True,internetUsageLocation,AnyLocation,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} internetAtHome,populationType,Household,isInternetUser,True,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} noInternetAtHome,populationType,Household,isInternetUser,False,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} -adultInternetUser,populationType,Household,,,isInternetUser,True,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} +adultInternetUser,populationType,Person,Age,[15 - Years],isInternetUser,True,race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},,,, ispBundle,populationType,Household,internetSubscriptionType,InternetBundle,,,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus} ,,,,,,,,,,,,,,,, ,,,,,,,,,,,,,,,, @@ -91,7 +91,7 @@ Uses the Internet at Someone Else's Home,#ignore,Ignore,,,,,,,,,,,,,, Browses the Web,DescriptionVar,Browses the Web,isInternetUser,True,internetUsagePurpose,WebBrowsingUsage,,,,,,,,,, Interacts with Household Equipment Using the Internet,#ignore,Ignore,,,,,,,,,,,,,, Participates in Online Video or Voice Calls or Conferencing,DescriptionVar,Participates in Online Video or Voice Calls or Conferencing,isInternetUser,True,internetUsagePurpose,VideoCallsUsage__AudioCallsUsage__ConferencingUsage,,,,,,,,,, -"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,StreamingUsage__MediaDownloadUsage,,,,,,,,,, +"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,MediaDownloadUsage__StreamingUsage,,,,,,,,,, Uses Health Monitoring Service that Connects to the Internet,DescriptionVar,Uses Health Monitoring Service that Connects to the Internet,isInternetUser,True,internetUsagePurpose,HealthMonitoringServicesUsage,,,,,,,,,, Uses Online Social Networks,DescriptionVar,Uses Online Social Networks,isInternetUser,True,internetUsagePurpose,SocialNetworksUsage,,,,,,,,,, Mobile Data Plan Used from Any Location,DescriptionVar,Mobile Data Plan Used from Any Location,isInternetUser,True,internetSubscriptionType,CellularDataPlan,,,,,,,,,, @@ -118,18 +118,18 @@ age1524Count,observationAbout,country/USA,measuredProperty,count,Age,[15 24 Year age2544Count,observationAbout,country/USA,measuredProperty,count,Age,[25 44 Years],value,{Number},name,"""{DescriptionVar}, age between 25 and 44 years""",,,,,, age4564Count,observationAbout,country/USA,measuredProperty,count,Age,[45 64 Years],value,{Number},name,"""{DescriptionVar}, age between 45 and 64 years""",,,,,, age65pCount,observationAbout,country/USA,measuredProperty,count,Age,[65 - Years],value,{Number},name,"""{DescriptionVar}, age 65 years and above""",,,,,, -workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",houseHolderAge,"""""" -workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",houseHolderAge,"""""" -workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",houseHolderAge,"""""" +workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",householderAge,"""""" +workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",householderAge,"""""" +workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",householderAge,"""""" incomeU25Count,observationAbout,country/USA,measuredProperty,count,income,[- 25000 USDollar],value,{Number},name,"""{DescriptionVar}, income under 25000 USD""",,,,,, income2549Count,observationAbout,country/USA,measuredProperty,count,income,[25000 49000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 25000 and 49000 USD""",,,,,, income5074Count,observationAbout,country/USA,measuredProperty,count,income,[50000 74000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 50000 and 74000 USD""",,,,,, income7599Count,observationAbout,country/USA,measuredProperty,count,income,[75000 99000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 75000 and 99000 USD""",,,,,, income100pCount,observationAbout,country/USA,measuredProperty,count,income,[100000 - USDollar],value,{Number},name,"""{DescriptionVar}, income above 100000 USD""",,,,,, -edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",houseHolderAge,"""""" -edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" -edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",houseHolderAge,"""""" -edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",houseHolderAge,"""""" +edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",householderAge,"""""" +edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",householderAge,"""""" +edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",householderAge,"""""" +edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",householderAge,"""""" sexMaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Male,value,{Number},name,"""{DescriptionVar}, male""",,,,,, sexFemaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Female,value,{Number},name,"""{DescriptionVar}, female""",,,,,, raceWhiteCount,observationAbout,country/USA,measuredProperty,count,Race,WhiteAloneNotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, White non-Hispanic""",,,,,, diff --git a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py index 177e78f2fa..d5ea296137 100644 --- a/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py +++ b/statvar_imports/ntia_internet_use_survey/commerce_ntia/preprocess.py @@ -12,8 +12,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -import os, sys -from pathlib import Path +import os +import sys import pandas as pd from absl import app, logging import config @@ -27,20 +27,18 @@ Commerce_NTIA_URL = config.Commerce_NTIA_URL INPUT_DIR = os.path.join(script_dir, "input_files") -Path(INPUT_DIR).mkdir(parents=True, exist_ok=True) - COMMON_COLUMNS = ["dataset", "variable", "description", "universe"] -AGE_COLUMNS = ["age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount"] +AGE_COLUMNS = [ + "age314Count", "age1524Count", "age2544Count", "age4564Count", "age65pCount" +] INPUT_FILE = os.path.join(INPUT_DIR, "ntia-analyze-table.csv") INPUT_FILE_1 = os.path.join(INPUT_DIR, "ntia-data-age-only.csv") INPUT_FILE_2 = os.path.join(INPUT_DIR, "ntia-data.csv") HEADERS = { - 'User-Agent': ( - 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' - '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36' - ), + 'User-Agent': ('Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 ' + '(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36'), 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8', } @@ -58,31 +56,35 @@ def move_column_left(df, column_to_move, target_column): def preprocess_data(): try: + os.makedirs(INPUT_DIR, exist_ok=True) org_df = pd.read_csv(INPUT_FILE) - + df1 = org_df[COMMON_COLUMNS + AGE_COLUMNS].copy() df1['universeAgeResol'] = df1['universe'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df1['variableAgeResol'] = df1['variable'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df1_moved = move_column_left(df1, 'universe', 'variable') df1_moved.to_csv(INPUT_FILE_1, index=False) - df2_cols_to_keep = [col for col in org_df.columns if not col.startswith('age')] + df2_cols_to_keep = [ + col for col in org_df.columns if not col.startswith('age') + ] df2 = org_df[df2_cols_to_keep].copy() df2['universeAgeResol'] = df2['universe'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df2['variableAgeResol'] = df2['variable'].apply( - lambda x: 'CivilPerson' if x == 'isPerson' else ('Adult' if x == 'isAdult' else None) - ) + lambda x: 'CivilPerson' + if x == 'isPerson' else ('Adult' if x == 'isAdult' else None)) df2_moved = move_column_left(df2, 'universe', 'variable') df2_moved.to_csv(INPUT_FILE_2, index=False) except Exception as e: - logging.fatal(f"An error occurred while preprocessing the input data: {e}") + logging.fatal( + f"An error occurred while preprocessing the input data: {e}") sys.exit(1)