diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md index 19d09757af..96a71547f9 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/README.md @@ -10,19 +10,22 @@ This import pipeline processes various socio-economic and agricultural statistic ## Configuration -The `rbi_download.py` script relies on a configuration file named `configs.py` to fetch the URLs, filenames, and categories for the data to be downloaded. +The `rbi_download.py` script relies on a local, version-controlled JSON configuration file named `configs.json` to specify URLs, filenames, and categories for the data to be downloaded. -Sample data of the config file: -URLS_CONFIG=[ +Sample structure of `configs.json`: +```json +{ + "URLS_CONFIG": [ { - "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_xxxxxxxxxxxxx.XLSX", - "category": "agriculture", - "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" - }, - ... - ] + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_xxxxxxxxxxxxx.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" + } + ] +} +``` -This approach makes the import process semi-automatic: if the download URLs change in future RBI releases (which commonly happens due to shifting table numbers or publication structures), only this configuration file needs to be updated, rather than modifying the Python script itself. +This makes the import process **Semi-Automated**: if download URLs change in future RBI releases (due to updated publication links or revised tables), only `configs.json` needs to be updated without modifying script logic. Additionally, `rbi_download.py` employs connection pooling with `requests.Session()` + `HTTPAdapter(Retry(...))`, validates file magic bytes (`PK\x03\x04`), and atomically writes downloaded files to prevent corrupted/partial files. ## Data Acquisition and Initial Preprocessing @@ -288,3 +291,10 @@ python3 stat_var_processor.py \ --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf \ --output_path=../../statvar_imports/database_on_indian_economy/india_rbi_state_statistics/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output ``` + +## Validation Configuration and Thresholds + +This import uses `validation_config.json` with a 7% record deletion threshold (`DELETED_RECORDS_PERCENT: 7`). +- **Rationale & Analysis**: Up to 7% of records may be deleted or modified across historical revisions and cleanups of unmapped sub-divisions and regional reporting structures across RBI state tables. +- **Justification Document**: For detailed root cause analysis, table breakdown, and justification, see the [RBI State Statistics Deletion Threshold Justification Doc](https://docs.google.com/document/d/19k3zyNk_KR9D4dVpXdGvlXKk6AMOzoYU2I1khJSsX8w/edit?resourcekey=0-JR3lDK9eNfo3ftEV6YpGxQ&tab=t.0#heading=h.iiqodx2yoqc8). + diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json new file mode 100644 index 0000000000..52ef784515 --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/configs.json @@ -0,0 +1,334 @@ +{ + "URLS_CONFIG": [ + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/53T_111220253720381F6FEB46769E9A3BA1775126F7.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_sown_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/54T_111220257E02BB1F59FC46F2B936CC773BBC0874.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_net_sown_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/55T_11122025AFBC97CC9BA44E54BC31B3E58DBF2E21.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_gross_irrigated_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/56T_1112202564A24EAD0AA44A278DB069916CA8FECB.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_net_irrigated_area.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/57T_111220253F1BCFDCDF5047B495821F992E4D87B6.XLSX", + "category": "agriculture", + "filename": "state_wise_pattern_of_land_use_cropping_intensity.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/58T_111220250E85BAD71CD64324A6F4E28DD0A05129.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/59T_1112202571AAC5A686F74224A1DF55D2B9D62D77.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/60T_111220251CE8EB785C6E40FAA8BA4EE805D01B93.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/61T_111220251C3BA74BCC3742A7BA09518C4DDD0BE5.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_foodgrains_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/62T_11122025F9D0951B74274E99A9C236A5CFAC1858.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/63T_1112202514B06350BA2849F98C3551060541C568.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/64T_111220251AE568D097A740F18A52EFEB30BDDA02.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/65T_11122025FF1A1A01BD604A1A9A63E572FF11B5B5.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/66T_1112202551DDDB3EE6624BE9B30EB1692CC0CBA6.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_non_foodgrains_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/67T_1112202594892EEE85FF4BAC9AAB5DC35661E62A.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/68T_11122025D202C9C812754B4FBC0F25A348940AA8.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/69T_111220258B2ACEEE047E482AA3EDF0C09F7E3A86.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/70T_11122025A43EF0C6E0154357B8E404D9CF42998C.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_foodgrains_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/71T_111220252D54659F49874E689923BF46C29E01A5.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/72T_11122025D573A69D2317423FBCFB65F5622AF929.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_fruits.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/73T_11122025E35C4283EAE7430A88B1BF32B236A976.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_total_vegetables.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/74T_11122025EBF01225C48441C2A23B436CCC589BF1.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_fruits.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/75T_11122025C8A307B91418448DBF5B5E54997E7355.XLSX", + "category": "agriculture", + "filename": "state_wise_production_of_total_vegetables.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/76T_1112202535D900531AEA4C71B96F562B9AE59835.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/77T_11122025DA3881A5159A4760B53AC35C5BFBA8D9.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/78T_111220252516A0D23AA94A6B809CFB3B1953DC7B.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/79T_1112202567B7934A72A04BF988AAC0656AA9DB66.XLSX", + "category": "agriculture", + "filename": "state_wise_area_of_non_foodgrains_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/80T_111220251421B93FB22A4B8BAE3E52809636B603.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_rice.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/81T_11122025F795A58AD4964BFD9C1AF74F0557164E.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_wheat.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/82T_1112202534205116BBCA4010AC1792E31FADC9C9.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_coarse_cereals.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/83T_111220255A0477BFBDFD45AF93EDA200B2A5A66F.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_pulses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/84T_11122025D106F74C0F98465DAF95013CB77F3A20.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_total_foodgrains.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/85T_111220250039FAA1932944E5B7627528B147FAFC.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_oilseeds.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/86T_111220258B6992228C504C2D84CBD88574260E87.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_cotton_lintt.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/87T_1112202547D1DA4285204EAFA0BAAA6CA47834C7.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_sugarcane.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/88T_1112202507582D4179654DC292A6311E3BDB2AB5.XLSX", + "category": "agriculture", + "filename": "state_wise_estimates_of_yield_raw_jute_mesta.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/89T_11122025CC34A78CE2D64C1D880A9B4B6FA1135D.XLSX", + "category": "agriculture", + "filename": "state_wise_per_hectare_consumption_of_fertiliser_n_p_k.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/90T_1112202558892C39CD554E4098A8EB79C63D47D9.XLSX", + "category": "agriculture", + "filename": "state_wise_meat_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/91T_11122025A7A12D69EA1645129226BEAE5783E1A4.XLSX", + "category": "agriculture", + "filename": "state_wise_milk_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/92T_11122025867917E0AC5644ADB035966AAF9D220B.XLSX", + "category": "agriculture", + "filename": "state_wise_egg_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/93T_1112202561B6558711FE4088998165985924266E.XLSX", + "category": "agriculture", + "filename": "state_wise_fish_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/94T_111220254142688D61EB497BA75DAFDDB72F7160.XLSX", + "category": "agriculture", + "filename": "state_wise_wool_production.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/99T_111220251AFF6E04A39B48E499CE9CDE9E66838B.XLSX", + "category": "environment", + "filename": "state_wise_forest_cover.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/101T_11122025C61D6934B6D74891B15B91679CFE1689.XLSX", + "category": "environment", + "filename": "state_wise_tree_cover.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/102T_11122025C0C87B2909234302A1F4A52A9A42E4DF.XLSX", + "category": "environment", + "filename": "sub_division_wise_annual_rainfall.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/105T_11122025454345A0E067418D9326DD30AE67BF6F.XLSX", + "category": "environment", + "filename": "state_wise_expenditure_on_relief_on_natural_calamities.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/106T_111220251B177E8F85CA412DB39DCE90BCBD724A.XLSX", + "category": "environment", + "filename": "state_wise_sustainable_development_goals_score_SDGs.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/138T_111220252D21BC51E5A842E4B25E954391C10A41.XLSX", + "category": "infrastructure", + "filename": "state_wise_per_capita_availability_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/139T_111220256569FF1C11E248B7A8ED64D31C999776.XLSX", + "category": "infrastructure", + "filename": "state_wise_availability_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/140T_111220254D8DA0B69B444492B6E9BAF30F3395C8.XLSX", + "category": "infrastructure", + "filename": "state_wise_installed_capacity_of_power.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/141T_11122025615E98A72EA6475CAACFFD12FFC83057.XLSX", + "category": "infrastructure", + "filename": "state_wise_power_requirement.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/144T_111220250497208D070C44219818788DB5BCE35A.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_national_highways.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/145T_111220259E033C52098E4732A6101C940DEF1222.XLSX", + "category": "infrastructure", + "filename": "state_wise_railway_route.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/146T_11122025B89449F4159B420E8305A9509F27FE82.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_roads.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/147T_111220255C153B31AB2E4835ABEEAC91D7494BFA.XLSX", + "category": "infrastructure", + "filename": "state_wise_length_of_state_highways.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/148T_11122025D741C3CB2CC24F448DC99D907589DD71.XLSX", + "category": "infrastructure", + "filename": "state_wise_electricity_transmission_distribution_losses.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/149T_1112202538D6A609C55B4FEFBBC44BB6C5ED20F7.XLSX", + "category": "infrastructure", + "filename": "state_wise_telephones_per_100_population.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/150T_11122025C30EFE7568EA472C8AB0290288B160C0.XLSX", + "category": "infrastructure", + "filename": "state_wise_road_constructed_under_PMGSY.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/108T_111220251942D16B3BED4D73BE851D15D2329168.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_general.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/109T_111220250E067E49350B48659E5A50C3D357BB46.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_food_and_beverages.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/110T_11122025784C1BDC2482496B9E25DA1BA4B38A3D.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_fuel_and_light.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/111T_111220252F7D9704AB664C35A58770BAC73518DE.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_inflation_CPI_housing_urban.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/112T_11122025771E55EB688044CE8A85E52A901B5D22.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_construction_workers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/113T_111220256FEBCC253D4A47419892925F897E2537.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in _rural_india_men_general_agricultural_labourers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/114T_111220259CA566982E7F4BF38B3FDAC0D7587F2E.XLSX", + "category": "price_and_wages", + "filename": "state_wise_Average Daily Wage Rates in Rural_india_men_horticulture_workers.xlsx" + }, + { + "url": "https://rbidocs.rbi.org.in/rdocs/Publications/DOCs/115T_1112202585CF99F21E654CED8990A734A043B3DA.XLSX", + "category": "price_and_wages", + "filename": "state_wise_average_daily_wage_rates_in_rural_india_men_non_agricultural_labourers.xlsx" + } + ] +} \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv index 0a1f8a72a5..ebd44f700d 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/environment_pvmap.csv @@ -6,6 +6,7 @@ Table 105: STATE-WISE EXPENDITURE ON RELIEF ON NATURAL CALAMITIES,measuredProper Table 106: STATE-WISE SUSTAINABLE DEVELOPMENT GOALS (SDGs) SCORE,measuredProperty,score,populationType,SustainableDevelopmentGoals,name,"""Sustainable Development Goals Score""",,,,,,,, State/ Union Territory,observationAbout,{Data},,,,,,,,,,,, State/Union Territory,observationAbout,{Data},,,,,,,,,,,, +"Haryana, Delhi & Chandigarh",observationAbout,country/IND 1987,observationDate,1987,value,{Number},,,,,,,,,, 1989,observationDate,1989,value,{Number},,,,,,,,,, 1991,observationDate,1991,value,{Number},,,,,,,,,, diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json index efeca24ba2..d7f96314e7 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/manifest.json @@ -15,83 +15,103 @@ "input_files/agriculture/*.xlsx", "input_files/environment/*.xlsx", "input_files/infrastructure/*.xlsx", - "input_files/price_and_wages/*.xlsx" + "input_files/price_and_wages/*.xlsx", + "counters/*.csv" ], "import_inputs": [ { "template_mcf": "output_files/agriculture/agriculture_output.tmcf", - "cleaned_csv": "output_files/agriculture/agriculture_output.csv" + "cleaned_csv": "output_files/agriculture/agriculture_output.csv", + "node_mcf": "output_files/agriculture/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv" + "cleaned_csv": "output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_forest_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_forest_cover_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_sdg_score_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv" + "cleaned_csv": "output_files/environment/state_wise_sdg_score_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/state_wise_tree_cover_output.tmcf", - "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv" + "cleaned_csv": "output_files/environment/state_wise_tree_cover_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/environment/sub_division_wise_annual_rainfall_output.tmcf", - "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv" + "cleaned_csv": "output_files/environment/sub_division_wise_annual_rainfall_output.csv", + "node_mcf": "output_files/environment/*.mcf" }, { "template_mcf": "output_files/price_and_wages/price_and_wages_output.tmcf", - "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv" + "cleaned_csv": "output_files/price_and_wages/price_and_wages_output.csv", + "node_mcf": "output_files/price_and_wages/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_availability_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_availability_of_power_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_installed_capacity_of_power_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_national_highways_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_national_highways_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_roads_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_roads_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_length_of_state_highways_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_length_of_state_highways_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_per_capita_availability_of_power_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_power_requirement_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_power_requirement_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_railway_route_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_railway_route_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" }, { "template_mcf": "output_files/infrastructure/state_wise_telephones_per_100_population_output.tmcf", - "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv" + "cleaned_csv": "output_files/infrastructure/state_wise_telephones_per_100_population_output.csv", + "node_mcf": "output_files/infrastructure/*.mcf" } ], - "cron_schedule": "0 10 * * 1" + "cron_schedule": "0 10 * * 1", + "validation_config_file": "validation_config.json" } ] } \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py index e30edc2db1..7d6da73694 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download.py @@ -12,97 +12,302 @@ # See the License for the specific language governing permissions and # limitations under the License. +import json import os -from absl import app, logging, flags +import pathlib +import re import sys +import tempfile +import time +from absl import app, flags, logging from google.cloud import storage import pandas as pd +import requests +from requests.adapters import HTTPAdapter +import urllib3 +from urllib3.util import Retry + +# Suppress unverified HTTPS warnings for RBI endpoints +urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning) SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(os.path.join(SCRIPT_DIR, '../../../util')) INPUT_DIR = os.path.join(SCRIPT_DIR, "input_files") +DEFAULT_CONFIG_JSON = os.path.join(SCRIPT_DIR, 'configs.json') -from download_util_script import _retry_method +flags.DEFINE_string('config_file_path', DEFAULT_CONFIG_JSON, + 'Config file path (local json or gs:// path)') -flags.DEFINE_string( - 'config_file_path', - 'gs://unresolved_mcf/india_rbi/state_statistics/configs.py', - 'Config file path') +DEFAULT_HEADERS = { + 'User-Agent': ('Mozilla/5.0 (Windows NT 10.0; Win64; x64) ' + 'AppleWebKit/537.36 (KHTML, like Gecko) ' + 'Chrome/124.0.0.0 Safari/537.36'), + 'Accept': + 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8', + 'Accept-Language': + 'en-US,en;q=0.9', +} -def reads_config_file(): - _FLAGS = flags.FLAGS - config_file_path = _FLAGS.config_file_path +XLSX_ZIP_SIGNATURE = b'PK\x03\x04' + + +def create_retry_session( + retries: int = 4, + backoff_factor: float = 2.0, + status_forcelist: tuple = (429, 500, 502, 503, 504), +) -> requests.Session: + """Creates a requests.Session with connection pooling, retries, and browser headers.""" + session = requests.Session() + retry_strategy = Retry( + total=retries, + backoff_factor=backoff_factor, + status_forcelist=status_forcelist, + allowed_methods=["HEAD", "GET"], + raise_on_status=False, + ) + adapter = HTTPAdapter(max_retries=retry_strategy, + pool_connections=10, + pool_maxsize=10) + session.mount("https://", adapter) + session.mount("http://", adapter) + session.headers.update(DEFAULT_HEADERS) + session.verify = False + return session + + +def _load_local_config(file_path): + if not os.path.exists(file_path): + return None try: - storage_client = storage.Client() - bucket_name = config_file_path.split('/')[2] - bucket = storage_client.bucket(bucket_name) - blob_name = '/'.join(config_file_path.split('/')[3:]) - blob = bucket.blob(blob_name) - file_contents = blob.download_as_text() - local_vars = {} - exec(file_contents, {}, local_vars) - return local_vars + with open(file_path, 'r', encoding='utf-8') as f: + return json.load(f) except Exception as e: - logging.fatal(f"Cannot extract url and related configs: {e}") + logging.warning(f"Failed to read local config file '{file_path}': {e}") + return None + -def download_files(URL_CONFIG): +def reads_config_file(): + _FLAGS = flags.FLAGS try: - for config in URL_CONFIG: - config_url = config.get("url") - category_name = config.get("category") - file_name = config.get("filename") - file_path = os.path.join(INPUT_DIR, category_name, file_name) - os.makedirs(os.path.join(INPUT_DIR, category_name), exist_ok=True) - file_response = _retry_method(config_url, headers=None, tries=3, delay=5, backoff=2) - if file_response: - with open(file_path, 'wb') as f: - f.write(file_response.content) - logging.info(f"Downloaded the file {file_name} successfully.") - except Exception as e: - logging.fatal(f"Download error: {str(e)}") + config_file_path = _FLAGS.config_file_path + except (flags.UnparsedFlagAccessError, AttributeError): + try: + config_file_path = _FLAGS['config_file_path'].value + except Exception: + config_file_path = DEFAULT_CONFIG_JSON + + # 1. If it's a local file path (or doesn't start with gs://), try loading it + if config_file_path and not config_file_path.startswith('gs://'): + config = _load_local_config(config_file_path) + if config and 'URLS_CONFIG' in config: + logging.info(f"Loaded config from local path: {config_file_path}") + return config + logging.warning( + f"Could not load config from local file '{config_file_path}'") + + # 2. If it's a GCS path, try loading from GCS + if config_file_path and config_file_path.startswith('gs://'): + try: + storage_client = storage.Client() + parts = config_file_path[5:].split('/', 1) + bucket_name = parts[0] + blob_name = parts[1] if len(parts) > 1 else '' + bucket = storage_client.bucket(bucket_name) + blob = bucket.blob(blob_name) + file_contents = blob.download_as_text() + if config_file_path.endswith('.json'): + return json.loads(file_contents) + local_vars = {} + exec(file_contents, {}, local_vars) + return local_vars + except Exception as e: + logging.warning( + f"Cannot extract url and related configs from GCS '{config_file_path}': {e}. " + "Falling back to local configuration.") + + # 3. Fall back to local configs.json in SCRIPT_DIR + config = _load_local_config(DEFAULT_CONFIG_JSON) + if config and 'URLS_CONFIG' in config: + logging.info( + f"Loaded fallback configuration from {DEFAULT_CONFIG_JSON}") + return config + + logging.error( + "Cannot extract url and related configs: all sources failed.") + raise RuntimeError( + "Cannot extract url and related configs: all sources failed.") + + +def download_files(URL_CONFIG, session=None, delay=0.5): + if not URL_CONFIG: + logging.warning("No URL configurations provided to download.") + return + + if session is None: + session = create_retry_session() + + for config in URL_CONFIG: + config_url = config.get("url") + category_name = config.get("category") + file_name = config.get("filename") + if not config_url or not category_name or not file_name: + logging.warning(f"Skipping incomplete config: {config}") + continue + + target_dir = os.path.join(INPUT_DIR, category_name) + os.makedirs(target_dir, exist_ok=True) + file_path = os.path.join(target_dir, file_name) + + if os.path.exists(file_path): + if os.path.getsize(file_path) > 0: + with open(file_path, 'rb') as f: + magic = f.read(4) + if magic == XLSX_ZIP_SIGNATURE: + logging.info(f"Skipping existing valid file: {file_name}") + continue + # Remove stale, empty, or corrupt file + try: + os.remove(file_path) + except OSError: + pass + + try: + logging.info(f"Attempting GET request to: {config_url}") + response = session.get(config_url, timeout=45) + + if response.status_code == 404: + logging.warning( + f"Table not found (HTTP 404) at {config_url}, skipping {file_name}" + ) + continue + + response.raise_for_status() + content = response.content + + if not content: + logging.error( + f"Empty response received for {file_name} from {config_url}" + ) + continue + + if not content.startswith(XLSX_ZIP_SIGNATURE): + snippet = content[:50].decode('utf-8', errors='replace') + logging.error( + f"Invalid file format received for {file_name} from {config_url}: " + f"expected XLSX ZIP signature, got: {snippet!r}") + continue + + # Write atomically to a temporary file, then move into place + with tempfile.NamedTemporaryFile('wb', + dir=target_dir, + delete=False) as tmp_file: + tmp_file.write(content) + temp_path = tmp_file.name + + os.replace(temp_path, file_path) + logging.info( + f"Downloaded the file {file_name} successfully ({len(content)} bytes)." + ) + + if delay > 0: + time.sleep(delay) + + except Exception as e: + logging.error( + f"Failed to download table {file_name} from {config_url}: {e}") + + +def _apply_map(df, func): + """Version-agnostic element-wise DataFrame mapping (uses df.map if available, else df.applymap).""" + if hasattr(df, 'map'): + return df.map(func) + return df.applymap(func) def preprocess_files(directory_path): if not os.path.isdir(directory_path): - logging.info(f"Error: Directory not found at '{directory_path}'") + logging.fatal(f"Error: Directory not found at '{directory_path}'") return - + xlsx_files = [f for f in os.listdir(directory_path) if f.endswith('.xlsx')] if not xlsx_files: - logging.info(f"No XLSX files found in the directory: {directory_path}") + logging.fatal( + f"No XLSX files found in the directory: {directory_path}") return - logging.info(f"Found {len(xlsx_files)} XLSX files to process in '{directory_path}'.") + logging.info( + f"Found {len(xlsx_files)} XLSX files to process in '{directory_path}'." + ) - for file_name in xlsx_files: + for file_name in sorted(xlsx_files): file_path = os.path.join(directory_path, file_name) logging.info(f"Processing file: {file_path}") try: - all_sheets_data = pd.read_excel(file_path, sheet_name=None, header=None) - with pd.ExcelWriter(file_path, engine='openpyxl') as writer: - for sheet_name, df in all_sheets_data.items(): - df = df.map(lambda x: str(x).replace('*', '').replace('@', '').strip()) - mask = df.eq('State/Union Territory').any(axis=1) - state_positions = df[mask] == 'State/Union Territory' - - def safe_to_numeric(val): - val_str = str(val) - if val_str.isdigit(): - return int(val_str) - try: - float_val = float(val_str) - return float_val - except ValueError: - return val - - df_num = df[mask].applymap(safe_to_numeric) - converted = df_num.mask(state_positions, 'State/Union Territory') + all_sheets_data = pd.read_excel(file_path, + sheet_name=None, + header=None, + engine='openpyxl') + + def clean_cell(val): + if pd.isna(val): + return val + if isinstance(val, (int, float)): + return val + val_str = str(val).replace('*', '').replace('@', '').strip() + if not val_str or val_str.lower() == 'nan': + return float('nan') + return val_str + + def safe_to_numeric(val): + if pd.isna(val): + return val + if isinstance(val, (int, float)): + return val + val_str = str(val).strip() + if not val_str or val_str.lower() == 'nan': + return float('nan') + if val_str.isdigit(): + return int(val_str) + try: + float_val = float(val_str) + if float_val.is_integer(): + return int(float_val) + return float_val + except ValueError: + return val + + def is_state_header(val): + if pd.isna(val): + return False + return bool( + re.search(r'state\s*/\s*union\s*territory', str(val), + re.IGNORECASE)) + + for sheet_name, df in all_sheets_data.items(): + df = _apply_map(df, clean_cell) + is_state_cell = _apply_map(df, is_state_header) + mask = is_state_cell.any(axis=1) + + if mask.any(): + state_positions = is_state_cell.loc[mask] + df_num = _apply_map(df[mask], safe_to_numeric) + converted = df_num.mask(state_positions, df[mask]) + df = df.astype(object) df.loc[mask, :] = converted - df.to_excel(writer, sheet_name=sheet_name, index=False, header=False) + all_sheets_data[sheet_name] = df + + with tempfile.TemporaryDirectory(dir=directory_path) as output_dir: + output_path = os.path.join(output_dir, file_name) + with pd.ExcelWriter(output_path, engine='openpyxl') as writer: + for sheet_name, df in all_sheets_data.items(): + df.to_excel(writer, + sheet_name=sheet_name, + index=False, + header=False) + os.replace(output_path, file_path) except Exception as e: - logging.fatal(f"Error processing {file_name}: {e}") + logging.error(f"Error processing {file_name}: {e}") logging.info("All specified XLSX files have been processed.") @@ -112,11 +317,13 @@ def main(_): RBI_URL = configs['URLS_CONFIG'] download_files(RBI_URL) logging.info("Download process Completed successfully") - directories = ['agriculture', 'environment', 'infrastructure', 'price_and_wages'] + directories = [ + 'agriculture', 'environment', 'infrastructure', 'price_and_wages' + ] for directory in directories: preprocess_files(os.path.join(INPUT_DIR, directory)) logging.info("Pre-process Completed successfully") -if __name__ == "__main__": +if __name__ == "__main__": app.run(main) diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py new file mode 100644 index 0000000000..15cce4876f --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_download_test.py @@ -0,0 +1,302 @@ +# Copyright 2026 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import json +import os +import pathlib +import sys +import tempfile +from unittest import mock + +from absl.testing import absltest, flagsaver +import pandas as pd +import requests + +_SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__)) +sys.path.append(_SCRIPT_DIR) + +import rbi_download + + +class PreprocessFilesTest(absltest.TestCase): + + def test_preserves_workbook_and_original_error(self): + with tempfile.TemporaryDirectory() as directory: + workbook = pathlib.Path(directory) / 'source.xlsx' + workbook.write_bytes(b'original workbook') + sheets = {'Sheet1': pd.DataFrame([['State/Union Territory']])} + + with mock.patch.object( + rbi_download.pd, 'read_excel', return_value=sheets), \ + mock.patch.object( + pd.DataFrame, + 'map', + side_effect=ValueError('transform failed')), \ + mock.patch.object(rbi_download.logging, 'error') as mock_error: + rbi_download.preprocess_files(directory) + + self.assertEqual(workbook.read_bytes(), b'original workbook') + mock_error.assert_called_once_with( + 'Error processing source.xlsx: transform failed') + + def test_preprocess_files_converts_numeric_headers(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2015*', '2016@', '2017-18'], + ['Andhra Pradesh', '10.5', '20.0', '30.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertEqual(processed.iloc[0, 0], 'State/Union Territory') + self.assertEqual(processed.iloc[0, 1], 2015) + self.assertEqual(processed.iloc[0, 2], 2016) + self.assertEqual(processed.iloc[0, 3], '2017-18') + self.assertEqual(processed.iloc[1, 0], 'Andhra Pradesh') + + def test_preprocess_files_preserves_nan(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2015*', None], + ['Andhra Pradesh', None, '30.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertTrue(pd.isna(processed.iloc[0, 2])) + self.assertTrue(pd.isna(processed.iloc[1, 1])) + + def test_preprocess_files_fatal_on_missing_dir(self): + with mock.patch.object(rbi_download.logging, 'fatal') as fatal: + rbi_download.preprocess_files('/non/existent/directory/path') + fatal.assert_called_once() + self.assertIn("Directory not found", fatal.call_args[0][0]) + + def test_preprocess_files_fatal_on_empty_dir(self): + with tempfile.TemporaryDirectory() as directory: + with mock.patch.object(rbi_download.logging, 'fatal') as fatal: + rbi_download.preprocess_files(directory) + fatal.assert_called_once() + self.assertIn("No XLSX files found", fatal.call_args[0][0]) + + def test_preprocess_files_handles_flexible_state_header(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame([['State / Union Territory', '2018-19'], + ['State/Union Territory', '2019-20'], + ['Karnataka', '120.5']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + self.assertEqual(processed.iloc[0, 0], 'State / Union Territory') + self.assertEqual(processed.iloc[1, 0], 'State/Union Territory') + self.assertEqual(processed.iloc[0, 1], '2018-19') + self.assertEqual(processed.iloc[1, 1], '2019-20') + self.assertEqual(processed.iloc[2, 0], 'Karnataka') + self.assertEqual(processed.iloc[2, 1], '120.5') + + def test_preprocess_files_avoids_literal_nan_and_preserves_numeric(self): + with tempfile.TemporaryDirectory() as directory: + file_path = pathlib.Path(directory) / 'source.xlsx' + initial_df = pd.DataFrame( + [['State/Union Territory', '2020', '2021', '2022'], + ['Bihar', 100, 200.5, ' '], ['Assam', 'nan', 'NAN', '300']]) + with pd.ExcelWriter(file_path, engine='openpyxl') as writer: + initial_df.to_excel(writer, + sheet_name='Sheet1', + index=False, + header=False) + + rbi_download.preprocess_files(directory) + + processed = pd.read_excel(file_path, + sheet_name='Sheet1', + header=None) + # Empty/whitespace and 'nan' strings become NaN, NOT literal string 'nan' + self.assertTrue(pd.isna(processed.iloc[1, 3])) + self.assertTrue(pd.isna(processed.iloc[2, 1])) + self.assertTrue(pd.isna(processed.iloc[2, 2])) + # Numeric values preserved + self.assertEqual(processed.iloc[1, 1], 100) + self.assertEqual(processed.iloc[1, 2], 200.5) + + +class ReadsConfigFileTest(absltest.TestCase): + + def test_reads_local_config_file(self): + with tempfile.NamedTemporaryFile('w', suffix='.json', + delete=False) as f: + json.dump( + { + 'URLS_CONFIG': [{ + 'url': 'http://example.com/test.xlsx', + 'category': 'test', + 'filename': 'test.xlsx' + }] + }, f) + temp_path = f.name + try: + with flagsaver.flagsaver(config_file_path=temp_path): + configs = rbi_download.reads_config_file() + self.assertIn('URLS_CONFIG', configs) + self.assertEqual(len(configs['URLS_CONFIG']), 1) + self.assertEqual(configs['URLS_CONFIG'][0]['filename'], + 'test.xlsx') + finally: + os.remove(temp_path) + + def test_falls_back_to_local_configs_json(self): + with flagsaver.flagsaver(config_file_path='gs://nonexistent_bucket/configs.json'), \ + mock.patch.object(rbi_download.storage, 'Client', side_effect=Exception('GCS unavailable')): + configs = rbi_download.reads_config_file() + self.assertIn('URLS_CONFIG', configs) + self.assertGreater(len(configs['URLS_CONFIG']), 0) + + +class DownloadFilesTest(absltest.TestCase): + + def test_skips_existing_valid_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + cat_dir = os.path.join(temp_dir, 'agri') + os.makedirs(cat_dir, exist_ok=True) + existing_file = os.path.join(cat_dir, 'existing.xlsx') + with open(existing_file, 'wb') as f: + f.write(b'PK\x03\x04valid_zip_content') + + configs = [{ + 'url': 'http://example.com/existing.xlsx', + 'category': 'agri', + 'filename': 'existing.xlsx' + }] + mock_session = mock.MagicMock() + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + mock_session.get.assert_not_called() + + def test_redownloads_invalid_existing_file(self): + with tempfile.TemporaryDirectory() as temp_dir: + cat_dir = os.path.join(temp_dir, 'agri') + os.makedirs(cat_dir, exist_ok=True) + corrupt_file = os.path.join(cat_dir, 'corrupt.xlsx') + with open(corrupt_file, 'wb') as f: + f.write(b'error page') + + configs = [{ + 'url': 'http://example.com/corrupt.xlsx', + 'category': 'agri', + 'filename': 'corrupt.xlsx' + }] + mock_session = mock.MagicMock() + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.content = b'PK\x03\x04valid_content' + mock_session.get.return_value = mock_response + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + mock_session.get.assert_called_once() + self.assertEqual( + pathlib.Path(corrupt_file).read_bytes(), + b'PK\x03\x04valid_content') + + def test_download_files_uses_browser_headers_in_session(self): + session = rbi_download.create_retry_session() + headers = session.headers + self.assertIn('User-Agent', headers) + self.assertIn('Mozilla', headers['User-Agent']) + + def test_rejects_html_error_response(self): + with tempfile.TemporaryDirectory() as temp_dir: + configs = [{ + 'url': 'http://example.com/blocked.xlsx', + 'category': 'agri', + 'filename': 'blocked.xlsx' + }] + mock_session = mock.MagicMock() + mock_response = mock.MagicMock() + mock_response.status_code = 200 + mock_response.content = b'Access Denied' + mock_session.get.return_value = mock_response + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + target_file = pathlib.Path(temp_dir) / 'agri' / 'blocked.xlsx' + self.assertFalse(target_file.exists()) + + def test_isolates_download_failures(self): + with tempfile.TemporaryDirectory() as temp_dir: + configs = [ + { + 'url': 'http://example.com/fail.xlsx', + 'category': 'agri', + 'filename': 'fail.xlsx' + }, + { + 'url': 'http://example.com/success.xlsx', + 'category': 'agri', + 'filename': 'success.xlsx' + }, + ] + mock_session = mock.MagicMock() + mock_success = mock.MagicMock() + mock_success.status_code = 200 + mock_success.content = b'PK\x03\x04success content' + mock_session.get.side_effect = [ + requests.exceptions.RequestException('404 Not Found'), + mock_success + ] + + with mock.patch.object(rbi_download, 'INPUT_DIR', temp_dir): + rbi_download.download_files(configs, + session=mock_session, + delay=0) + + success_file = pathlib.Path(temp_dir) / 'agri' / 'success.xlsx' + self.assertTrue(success_file.exists()) + self.assertEqual(success_file.read_bytes(), + b'PK\x03\x04success content') + + +if __name__ == '__main__': + absltest.main() diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv index a67a4fa4f1..ed2aea2d1f 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/rbi_places_resolver.csv @@ -8,7 +8,7 @@ Chandigarh,wikidataId/Q43433 Chhattisgarh,wikidataId/Q1168 Dadra & Nagar Haveli,wikidataId/Q46107 Daman & Diu,wikidataId/Q66710 -NCT of Delhi,wikidataId/Q1352 +NCT of Delhi,wikidataId/Q9357528 Delhi,wikidataId/Q1353 Goa,wikidataId/Q1171 Gujarat,wikidataId/Q1061 @@ -38,9 +38,10 @@ Uttarakhand,wikidataId/Q1499 West Bengal,wikidataId/Q1356 ALL INDIA,country/IND All-India,country/IND +All India,country/IND Ladakh,wikidataId/Q200667 -Kolkata**,wikidataId/Q1348 -Chennai**,wikidataId/Q1352 +Kolkata,wikidataId/Q1348 +Chennai,wikidataId/Q1352 Mumbai,wikidataId/Q1156 TamilNadu,wikidataId/Q1445 Andaman & NicobarIslands,wikidataId/Q40888 \ No newline at end of file diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh index 29f9bdd623..f73ef6e360 100644 --- a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/run.sh @@ -2,43 +2,52 @@ set -e SCRIPT_PATH=$(realpath "$(dirname "$0")") +COUNTERS_PATH=$SCRIPT_PATH/counters +mkdir -p "$COUNTERS_PATH" -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/agriculture/*.xlsx --pv_map=$SCRIPT_PATH/agriculture_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/agriculture/agriculture_output || { echo "Error: Processing Agriculture files failed!"; exit 1; } +# Existing statvar MCF path with local fallback +LOCAL_MCF="$SCRIPT_PATH/stat_vars.mcf" +if [ -f "$LOCAL_MCF" ]; then + EXISTING_MCF="$LOCAL_MCF" +else + EXISTING_MCF="${EXISTING_STATVAR_MCF:-gs://unresolved_mcf/scripts/statvar/stat_vars.mcf}" +fi -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/agriculture/*.xlsx" --pv_map=$SCRIPT_PATH/agriculture_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/agriculture_counters.csv --output_path=$SCRIPT_PATH/output_files/agriculture/agriculture_output || { echo "Error: Processing Agriculture files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_forest_cover.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_forest_cover_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_forest_cover_output || { echo "Error: Processing State-wise Forest Cover failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_tree_cover.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_tree_cover_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_tree_cover_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/sub_division_wise_annual_rainfall.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/sub_division_wise_annual_rainfall_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/sub_division_wise_annual_rainfall_output || { echo "Error: Processing Sub-Division-wise Annual Rainfall failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/environment_sdg_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_expenditure_on_relief_on_natural_calamities.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_expenditure_on_relief_on_natural_calamities_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_expenditure_on_relief_on_natural_calamities_output || { echo "Error: Processing State-wise Expenditure on Relief on Natural Calamities failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/price_and_wages/*.xlsx --pv_map=$SCRIPT_PATH/price_wages_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output || { echo "Error: Processing price_and_wages files failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/environment/state_wise_sustainable_development_goals_score_SDGs.xlsx" --pv_map=$SCRIPT_PATH/environment_pvmap.csv --config_file=$SCRIPT_PATH/environment_sdg_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_sdg_score_counters.csv --output_path=$SCRIPT_PATH/output_files/environment/state_wise_sdg_score_output || { echo "Error: Processing State-wise Sustainable Development Goals Score failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/price_and_wages/*.xlsx" --pv_map=$SCRIPT_PATH/price_wages_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/price_and_wages_counters.csv --output_path=$SCRIPT_PATH/output_files/price_and_wages/price_and_wages_output || { echo "Error: Processing price_and_wages files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_per_capita_availability_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_per_capita_availability_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_per_capita_availability_of_power_output || { echo "Error: Processing State-wise Per Capita Availability of Power failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_availability_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_availability_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_availability_of_power_output || { echo "Error: Processing State-wise Availability of Power files failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_installed_capacity_of_power.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_installed_capacity_of_power_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_installed_capacity_of_power_output || { echo "Error: Processing State-wise Installed Capacity of Power failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_power_requirement.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_power_requirement_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_power_requirement_output || { echo "Error: Processing State-wise Power Requirement failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_national_highways.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_national_highways_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_national_highways_output || { echo "Error: Processing State-wise Length of National Highways failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output || { echo "Error: Processing State-wise Tree Cover failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_railway_route.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_railway_route_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_railway_route_output || { echo "Error: Processing State-wise Railway Route failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_roads.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_roads_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_roads_output || { echo "Error: Processing State-wise Length of Roads failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_length_of_state_highways.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/infrastructure_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_length_of_state_highways_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_length_of_state_highways_output || { echo "Error: Processing State-wise Length of State Highways failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_electricity_transmission_distribution_losses.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_electricity_transmission_distribution_losses_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_electricity_transmission_distribution_losses_output || { echo "Error: Processing State-wise Electricity Transmission & Distribution Losses failed!"; exit 1; } -python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data=$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf=gs://unresolved_mcf/scripts/statvar/stat_vars.mcf --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_telephones_per_100_population.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_telephones_per_100_population_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_telephones_per_100_population_output || { echo "Error: Processing State-wise Telephones per 100 Population failed!"; exit 1; } + +python3 $SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py --input_data="$SCRIPT_PATH/input_files/infrastructure/state_wise_road_constructed_under_PMGSY.xlsx" --pv_map=$SCRIPT_PATH/infrastructure_pvmap.csv --config_file=$SCRIPT_PATH/rbi_metadata.csv --places_resolved_csv=$SCRIPT_PATH/rbi_places_resolver.csv --existing_statvar_mcf="$EXISTING_MCF" --output_counters=$COUNTERS_PATH/state_wise_road_constructed_under_pmgsy_counters.csv --output_path=$SCRIPT_PATH/output_files/infrastructure/state_wise_road_constructed_under_pmgsy_output || { echo "Error: Processing State-wise Road Constructed under PMGSY failed!"; exit 1; } echo "All processing steps completed successfully." exit 0 - diff --git a/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json new file mode 100644 index 0000000000..9655dc9f61 --- /dev/null +++ b/statvar_imports/database_on_indian_economy/india_rbi_state_statistics/validation_config.json @@ -0,0 +1,22 @@ +{ + "schema_version": "1.0", + "rules": [ + { + "rule_id": "check_deleted_records_percent", + "description": "Checks that the percentage of deleted records is within the 7% threshold.", + "validator": "DELETED_RECORDS_PERCENT", + "params": { + "threshold": 7 + } + }, + { + "rule_id": "check_max_date_freshness", + "description": "Verifies that each table meets its expected freshness threshold according to RBI publishing cadence.", + "validator": "SQL_VALIDATOR", + "params": { + "query": "SELECT CASE WHEN any_value(StatVar) IN ('Length_Transportation_Road', 'Length_Transportation_StateHighway') THEN MAX(MaxDate) >= '2020' WHEN any_value(StatVar) IN ('Area_LandCover_Forest', 'Score_SustainableDevelopmentGoals', 'Area_LandCover_Tree', 'Loss_Electricity_TransmissionAndDistribution_AsAFractionOf_Annual_Generation_Electricity') THEN MAX(MaxDate) >= '2023' ELSE MAX(MaxDate) >= '2024' END AS is_fresh, COUNT(*) AS row_cnt FROM stats", + "condition": "is_fresh = TRUE AND row_cnt > 0" + } + } + ] +}