Changes in preproces.py to remove SSL and add header to downlaod

statvar_imports/ntia_internet_use_survey/commerce_ntia/README.md

@@ -19,6 +19,7 @@ python3 stat_var_processor.py
 --input_data='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/input_files/<input_file.csv>' 
 --pv_map='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/<filename of pv_map.csv>' --config_file='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/<filename of metadata.csv>' --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf 
 --output_path='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/<output_folder_name>/<filename>'
+--output_counters='../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/<counters_folder_name>/<filename_counters.csv>'
 ```
 
 #### Download the data: 
@@ -40,6 +41,7 @@ python3 stat_var_processor.py
 --pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_pvmap.csv 
 --config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf 
 --output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_output
+--output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_output_counters.csv
 ```
 
 ```
@@ -48,5 +50,6 @@ python3 stat_var_processor.py
 --pv_map=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_age_pvmap.csv 
 --config_file=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf 
 --output_path=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/output_files/ntia_age_output
+--output_counters=../../statvar_imports/ntia_internet_use_survey/Commerce_NTIA/counters/ntia_age_output_counters.csv
 ```
 

statvar_imports/ntia_internet_use_survey/commerce_ntia/commerce_ntia_test.py

@@ -0,0 +1,82 @@
+# Copyright 2025 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#    https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Unit and regression tests for commerce_ntia statvar import."""
+
+import os
+import subprocess
+import sys
+import tempfile
+import unittest
+
+_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+_DATA_DIR = os.path.abspath(os.path.join(_SCRIPT_DIR, '../../../'))
+_TOOLS_DIR = os.path.join(_DATA_DIR, 'tools/statvar_importer')
+sys.path.insert(0, _TOOLS_DIR)
+sys.path.insert(0, os.path.join(_DATA_DIR, 'util'))
+from counters import Counters
+from mcf_diff import diff_mcf_files
+
+
+class CommerceNtiaTest(unittest.TestCase):
+
+    def setUp(self):
+        self.testdata_dir = os.path.join(_SCRIPT_DIR, 'testdata')
+        self.processor_path = os.path.join(_TOOLS_DIR, 'stat_var_processor.py')
+        self.pv_map = os.path.join(_SCRIPT_DIR, 'ntia_pvmap.csv')
+        self.metadata = os.path.join(_SCRIPT_DIR, 'ntia_metadata.csv')
+
+    def test_stat_var_processor_ntia_output(self):
+        """Tests that stat_var_processor generates expected outputs for ntia-data.csv."""
+        with tempfile.TemporaryDirectory() as tmp_dir:
+            output_path = os.path.join(tmp_dir, 'ntia_output')
+            cmd = [
+                sys.executable,
+                self.processor_path,
+                f'--input_data={os.path.join(self.testdata_dir, "ntia-data.csv")}',
+                f'--pv_map={self.pv_map}',
+                f'--config_file={self.metadata}',
+                f'--output_path={output_path}',
+            ]
+            res = subprocess.run(cmd, capture_output=True, text=True)
+            self.assertEqual(res.returncode, 0,
+                             f'Processor failed: {res.stderr}')
+
+            # Verify CSV
+            gen_csv = os.path.join(tmp_dir, 'ntia_output.csv')
+            exp_csv = os.path.join(self.testdata_dir, 'ntia_output.csv')
+            with open(gen_csv,
+                      encoding='utf-8') as g, open(exp_csv,
+                                                   encoding='utf-8') as e:
+                self.assertEqual(g.read().strip(), e.read().strip())
+
+            # Verify TMCF
+            gen_tmcf = os.path.join(tmp_dir, 'ntia_output.tmcf')
+            exp_tmcf = os.path.join(self.testdata_dir, 'ntia_output.tmcf')
+            with open(gen_tmcf,
+                      encoding='utf-8') as g, open(exp_tmcf,
+                                                   encoding='utf-8') as e:
+                self.assertEqual(g.read().strip(), e.read().strip())
+
+            # Verify StatVar MCF
+            gen_mcf = os.path.join(tmp_dir, 'ntia_output_stat_vars.mcf')
+            exp_mcf = os.path.join(self.testdata_dir,
+                                   'ntia_output_stat_vars.mcf')
+            counters = Counters()
+            diff = diff_mcf_files(gen_mcf, exp_mcf,
+                                  {'show_diff_nodes_only': True}, counters)
+            self.assertEqual(len(diff), 0, f'MCF diff found: {diff}')
+
+
+if __name__ == '__main__':
+    unittest.main()

statvar_imports/ntia_internet_use_survey/commerce_ntia/manifest.json

@@ -9,20 +9,23 @@
             "provenance_description": "NTIA programs and policymaking focus largely on expanding broadband Internet access and adoption in America, expanding the use of spectrum by all users.",
             "scripts": [
                 "preprocess.py",
-                "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output",
-                "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output"
+                "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data.csv --pv_map=ntia_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_output --output_counters=counters/ntia_output_counters.csv",
+                "../../../tools/statvar_importer/stat_var_processor.py --input_data=input_files/ntia-data-age-only.csv --pv_map=ntia_age_pvmap.csv --config_file=ntia_metadata.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=output_files/ntia_age_output --output_counters=counters/ntia_age_output_counters.csv"
             ],
             "source_files": [
-                "input_files/ntia-analyze-table.csv"
+                "input_files/ntia-analyze-table.csv",
+                "counters/*.csv"
             ],
             "import_inputs": [
                 {
                     "template_mcf": "output_files/ntia_output.tmcf",
-                    "cleaned_csv": "output_files/ntia_output.csv"
+                    "cleaned_csv": "output_files/ntia_output.csv",
+                    "node_mcf": "output_files/*.mcf"
                 },
                 {
                     "template_mcf": "output_files/ntia_age_output.tmcf",
-                    "cleaned_csv": "output_files/ntia_age_output.csv"
+                    "cleaned_csv": "output_files/ntia_age_output.csv",
+                    "node_mcf": "output_files/*.mcf"
                 }
             ],
             "cron_schedule": "0 06 * * 5"

statvar_imports/ntia_internet_use_survey/commerce_ntia/ntia_age_pvmap.csv

@@ -7,7 +7,7 @@ CivilPerson,age,{Age},armedForcesStatus,Civilian,,,,,,,,,,,,
 internetAnywhere,populationType,Household,isInternetUser,True,internetUsageLocation,AnyLocation,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus}
 internetAtHome,populationType,Household,isInternetUser,True,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus}
 noInternetAtHome,populationType,Household,isInternetUser,False,internetUsageLocation,Home,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus}
-adultInternetUser,populationType,Household,,,isInternetUser,True,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus}
+adultInternetUser,populationType,Person,Age,[15 - Years],isInternetUser,True,race,{Race},gender,{Gender},educationalAttainment,{EducationalAttainment},employmentStatus,{EmploymentStatus},,,,
 ispBundle,populationType,Household,internetSubscriptionType,InternetBundle,,,householderAge,{Age},householderRace,{Race},householderGender,{Gender},householderEducationalAttainment,{EducationalAttainment},householderWorkStatus,{EmploymentStatus}
 ,,,,,,,,,,,,,,,,
 ,,,,,,,,,,,,,,,,
@@ -91,7 +91,7 @@ Uses the Internet at Someone Else's Home,#ignore,Ignore,,,,,,,,,,,,,,
 Browses the Web,DescriptionVar,Browses the Web,isInternetUser,True,internetUsagePurpose,WebBrowsingUsage,,,,,,,,,,
 Interacts with Household Equipment Using the Internet,#ignore,Ignore,,,,,,,,,,,,,,
 Participates in Online Video or Voice Calls or Conferencing,DescriptionVar,Participates in Online Video or Voice Calls or Conferencing,isInternetUser,True,internetUsagePurpose,VideoCallsUsage__AudioCallsUsage__ConferencingUsage,,,,,,,,,,
-"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,StreamingUsage__MediaDownloadUsage,,,,,,,,,,
+"Streams or Downloads Music, Radio, Podcasts, etc.",DescriptionVar,"Streams or Downloads Music, Radio, Podcasts, etc.",isInternetUser,True,internetUsagePurpose,MediaDownloadUsage__StreamingUsage,,,,,,,,,,
 Uses Health Monitoring Service that Connects to the Internet,DescriptionVar,Uses Health Monitoring Service that Connects to the Internet,isInternetUser,True,internetUsagePurpose,HealthMonitoringServicesUsage,,,,,,,,,,
 Uses Online Social Networks,DescriptionVar,Uses Online Social Networks,isInternetUser,True,internetUsagePurpose,SocialNetworksUsage,,,,,,,,,,
 Mobile Data Plan Used from Any Location,DescriptionVar,Mobile Data Plan Used from Any Location,isInternetUser,True,internetSubscriptionType,CellularDataPlan,,,,,,,,,,
@@ -118,18 +118,18 @@ age1524Count,observationAbout,country/USA,measuredProperty,count,Age,[15 24 Year
 age2544Count,observationAbout,country/USA,measuredProperty,count,Age,[25 44 Years],value,{Number},name,"""{DescriptionVar}, age between 25 and 44 years""",,,,,,
 age4564Count,observationAbout,country/USA,measuredProperty,count,Age,[45 64 Years],value,{Number},name,"""{DescriptionVar}, age between 45 and 64 years""",,,,,,
 age65pCount,observationAbout,country/USA,measuredProperty,count,Age,[65 - Years],value,{Number},name,"""{DescriptionVar}, age 65 years and above""",,,,,,
-workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",houseHolderAge,""""""
-workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",houseHolderAge,""""""
-workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",houseHolderAge,""""""
+workEmployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Employed,value,{Number},name,"""{DescriptionVar}, employed""",Age,"""""",age,"""""",householderAge,""""""
+workUnemployedCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,Unemployed,value,{Number},name,"""{DescriptionVar}, unemployed""",Age,"""""",age,"""""",householderAge,""""""
+workNILFCount,observationAbout,country/USA,measuredProperty,count,EmploymentStatus,NotInLaborForce,value,{Number},name,"""{DescriptionVar}, not in Labour Force""",Age,"""""",age,"""""",householderAge,""""""
 incomeU25Count,observationAbout,country/USA,measuredProperty,count,income,[- 25000 USDollar],value,{Number},name,"""{DescriptionVar}, income under 25000 USD""",,,,,,
 income2549Count,observationAbout,country/USA,measuredProperty,count,income,[25000 49000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 25000 and 49000 USD""",,,,,,
 income5074Count,observationAbout,country/USA,measuredProperty,count,income,[50000 74000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 50000 and 74000 USD""",,,,,,
 income7599Count,observationAbout,country/USA,measuredProperty,count,income,[75000 99000 USDollar],value,{Number},name,"""{DescriptionVar}, income between 75000 and 99000 USD""",,,,,,
 income100pCount,observationAbout,country/USA,measuredProperty,count,income,[100000 - USDollar],value,{Number},name,"""{DescriptionVar}, income above 100000 USD""",,,,,,
-edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",houseHolderAge,""""""
-edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",houseHolderAge,""""""
-edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",houseHolderAge,""""""
-edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",houseHolderAge,""""""
+edNoDiplomaCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,NoDiploma,value,{Number},name,"""{DescriptionVar}, education without Diploma""",Age,"""""",age,"""""",householderAge,""""""
+edHSGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,HighSchoolGraduate,value,{Number},name,"""{DescriptionVar}, education up to Higher Secondary Graduate""",Age,"""""",age,"""""",householderAge,""""""
+edSomeCollegeCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,SomeCollegeOrAssociatesDegree,value,{Number},name,"""{DescriptionVar}, education up to some college""",Age,"""""",age,"""""",householderAge,""""""
+edCollegeGradCount,observationAbout,country/USA,measuredProperty,count,EducationalAttainment,CollegeGraduate,value,{Number},name,"""{DescriptionVar}, education up to college Graduate""",Age,"""""",age,"""""",householderAge,""""""
 sexMaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Male,value,{Number},name,"""{DescriptionVar}, male""",,,,,,
 sexFemaleCount,observationAbout,country/USA,measuredProperty,count,Gender,Female,value,{Number},name,"""{DescriptionVar}, female""",,,,,,
 raceWhiteCount,observationAbout,country/USA,measuredProperty,count,Race,WhiteAloneNotHispanicOrLatino,value,{Number},name,"""{DescriptionVar}, White non-Hispanic""",,,,,,