From 44dfe2acaffb3097f0635310b20d97e2e2219f37 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 7 Sep 2026 07:21:32 +0000 Subject: [PATCH 1/2] Migrate gsutil usage to gcloud storage --- scripts/earthengine/README.md | 8 ++++---- scripts/oecd/regional_demography/README.md | 2 +- scripts/us_bts/latch/download_script.py | 4 ++-- scripts/us_eia/opendata/generate_jsonl_for_bq.py | 2 +- scripts/us_nces/README.md | 8 ++++---- scripts/who/gho/README.md | 2 +- .../boundaries/country_boundaries_exploration.ipynb | 2 +- .../brazil_rural_development_program/run.sh | 2 +- statvar_imports/fbi/fbigovcrime/preprocess.py | 8 ++++---- .../india_ndap/india_nss_health_ailments/README.md | 2 +- 10 files changed, 20 insertions(+), 20 deletions(-) diff --git a/scripts/earthengine/README.md b/scripts/earthengine/README.md index 81b0edf136..0d0f5d5daa 100644 --- a/scripts/earthengine/README.md +++ b/scripts/earthengine/README.md @@ -11,7 +11,7 @@ The scripts use [Google Earth Engine](https://earthengine.google.com/) to extract flooded regions as a [geoTIFF](https://en.wikipedia.org/wiki/GeoTIFF) file. These `.tif` files are exported to [Google Cloud Storage](https://cloud.google.com/storage) (GCS). -The raster files are then copied over using `gsutil` to be processed into `.csv` locally. +The raster files are then copied over using `gcloud storage` to be processed into `.csv` locally. The tools can be installed as follows: @@ -36,7 +36,7 @@ earthengine authenticate --quiet ``` ### Google Cloud SDK -Install the `gsutil` command to copy files from GCS to the local +Install the `gcloud storage` command to copy files from GCS to the local machine. 1. Install GCS tools using the command: @@ -55,7 +55,7 @@ gcloud auth login To copy files on GCS, create a storage bucket with the following command or on the cloud console. ``` -gsutil mb gs:/// +gcloud storage buckets create gs:/// ``` ## Extract geoTIFF from EarthEngine (EE) @@ -99,7 +99,7 @@ in the previous step into a StatVarObservation MCF nodes for following: ``` # Download the geoTiff from GCS -gsutil cp gs:////*.tif . +gcloud storage cp gs:////*.tif . # Convert the geoTiff into raster python3 raster_to_csv.py \ --input_geotiff=ee_image_dynamic_world-band_water-r_max-mask_land-s_1000-from_2022-01-01.tif \ diff --git a/scripts/oecd/regional_demography/README.md b/scripts/oecd/regional_demography/README.md index 7aae61112c..9e5191bc83 100644 --- a/scripts/oecd/regional_demography/README.md +++ b/scripts/oecd/regional_demography/README.md @@ -39,7 +39,7 @@ In each subdirectory, the source CSVs are saved as `REGION_DEMOGR_{subdirectory Population raw data is stored [here](https://pantheon.corp.google.com/storage/browser/_details/datcom-source-data/oecd/regional_demography/population/REGION_DEMOGR_population.csv?authuser=0&project=datcom-204919) since it's quite big. To copy raw CSV into the population directory, run: ```bash -gsutil cp gs://datcom-source-data/oecd/regional_demography/population/REGION_DEMOGR_population.csv . +gcloud storage cp gs://datcom-source-data/oecd/regional_demography/population/REGION_DEMOGR_population.csv . ``` Population raw CSV is missing data for region_id CL16. To add the missing data, we created manual_curated_population.csv with command diff --git a/scripts/us_bts/latch/download_script.py b/scripts/us_bts/latch/download_script.py index 3b18cb27b6..7791b7d64e 100644 --- a/scripts/us_bts/latch/download_script.py +++ b/scripts/us_bts/latch/download_script.py @@ -120,7 +120,7 @@ def _download_paginated_socrata_file( def create_download_configs(gcs_urls_config_file: str) -> List[Dict]: """Reads the URL config JSON from GCS and generates the download configurations.""" try: - result = subprocess.run(['gsutil', 'cat', gcs_urls_config_file], + result = subprocess.run(['gcloud', 'storage', 'cat', gcs_urls_config_file], capture_output=True, text=True, check=True, @@ -131,7 +131,7 @@ def create_download_configs(gcs_urls_config_file: str) -> List[Dict]: ) except (subprocess.CalledProcessError, FileNotFoundError) as e: logging.fatal( - f"Failed to read GCS config file '{gcs_urls_config_file}': {e}. Ensure 'gsutil' is in your PATH and the file exists." + f"Failed to read GCS config file '{gcs_urls_config_file}': {e}. Ensure 'gcloud' is in your PATH and the file exists." ) return [] except json.JSONDecodeError as e: diff --git a/scripts/us_eia/opendata/generate_jsonl_for_bq.py b/scripts/us_eia/opendata/generate_jsonl_for_bq.py index fef4957c05..cac02000da 100644 --- a/scripts/us_eia/opendata/generate_jsonl_for_bq.py +++ b/scripts/us_eia/opendata/generate_jsonl_for_bq.py @@ -19,7 +19,7 @@ To import to bigquery: - run this script: `python3 generate_jsonl_for_bq.py` -- copy tmp_bq_import/ to gcs: `gsutil -m cp -r bq_import gs://us_eia/` +- copy tmp_bq_import/ to gcs: `gcloud storage cp --recursive bq_import gs://us_eia/` - load data into bigquery: ``` bq load \ diff --git a/scripts/us_nces/README.md b/scripts/us_nces/README.md index 79ca250537..a3ff050d95 100644 --- a/scripts/us_nces/README.md +++ b/scripts/us_nces/README.md @@ -22,13 +22,13 @@ This dataset has Population Estimates for the National Center for Education Stat Note: Give one year at a time for District and Public Schools as there are large number of column values. -### upload the files manually to GCP bucket for processing using gsutil command. +### upload the files manually to GCP bucket for processing using gcloud storage command. Ex private : - gsutil cp -r /scripts/us_nces/demographics/private_school/input_files gs://unresolved_mcf/us_nces/demographics/private_school/semi_automation_input_files/ + gcloud storage cp --recursive /scripts/us_nces/demographics/private_school/input_files gs://unresolved_mcf/us_nces/demographics/private_school/semi_automation_input_files/ Ex public : - gsutil cp -r /scripts/us_nces/demographics/public_school/input_files gs://unresolved_mcf/us_nces/demographics/public_school/semi_automation_input_files/ + gcloud storage cp --recursive /scripts/us_nces/demographics/public_school/input_files gs://unresolved_mcf/us_nces/demographics/public_school/semi_automation_input_files/ Ex district : - gsutil cp -r /scripts/us_nces/demographics/school_district/input_files gs://unresolved_mcf/us_nces/demographics/school_district/semi_automation_input_files/ + gcloud storage cp --recursive /scripts/us_nces/demographics/school_district/input_files gs://unresolved_mcf/us_nces/demographics/school_district/semi_automation_input_files/ ### Note: The only manual part here is after downloading the input files and then uploading them to gcp bucket. Once they're uploaded, Each import requires its own sh command to copy the files from Google Cloud to a local folder called gcs_folder/input_files. From there, a script automatically picks up these files to process them. Finally, it generates the output and saves it in gcs_folder/output_files diff --git a/scripts/who/gho/README.md b/scripts/who/gho/README.md index 03e37fdc17..e126aac05d 100644 --- a/scripts/who/gho/README.md +++ b/scripts/who/gho/README.md @@ -42,7 +42,7 @@ The following are the files generated by the [import_data.py](import_data.py) sc 1. Get the raw data in one of 2 ways: - To get the pre-downloaded data from our Google Cloud Storage, run and then unzip the file: ``` - gsutil cp gs://datcom-source-data/who/gho/indicator_data.zip . + gcloud storage cp gs://datcom-source-data/who/gho/indicator_data.zip . ``` - To download data using the WHO API (this can take about 1 hour), run: ``` diff --git a/scripts/world_bank/boundaries/country_boundaries_exploration.ipynb b/scripts/world_bank/boundaries/country_boundaries_exploration.ipynb index 94e8d74018..825315d277 100644 --- a/scripts/world_bank/boundaries/country_boundaries_exploration.ipynb +++ b/scripts/world_bank/boundaries/country_boundaries_exploration.ipynb @@ -156,7 +156,7 @@ "z = zipfile.ZipFile(io.BytesIO(r.content))\n", "z.extractall(TMP_PATH)\n", "\n", - "!gsutil -m cp -r {TMP_PATH} {GCS_BUCKET}" + "!gcloud storage cp --recursive {TMP_PATH} {GCS_BUCKET}" ], "execution_count": 16, "outputs": [ diff --git a/statvar_imports/brazil_visdata/brazil_rural_development_program/run.sh b/statvar_imports/brazil_visdata/brazil_rural_development_program/run.sh index 1c9aa38670..ffe5077b91 100644 --- a/statvar_imports/brazil_visdata/brazil_rural_development_program/run.sh +++ b/statvar_imports/brazil_visdata/brazil_rural_development_program/run.sh @@ -5,7 +5,7 @@ SCRIPT_PATH=$(realpath "$(dirname "$0")") mkdir -p "input_files" -gsutil cp "gs://unresolved_mcf/country/brazil/VISDATA/Benefits_RuralDevelopmentProgram/latest/input_files/*.csv" "$SCRIPT_PATH/input_files" +gcloud storage cp "gs://unresolved_mcf/country/brazil/VISDATA/Benefits_RuralDevelopmentProgram/latest/input_files/*.csv" "$SCRIPT_PATH/input_files" PROCESSOR_SCRIPT="$SCRIPT_PATH/../../../tools/statvar_importer/stat_var_processor.py" INPUT_BASE_PATH="$SCRIPT_PATH/input_files" diff --git a/statvar_imports/fbi/fbigovcrime/preprocess.py b/statvar_imports/fbi/fbigovcrime/preprocess.py index 54f798e12f..37f0d8b51b 100644 --- a/statvar_imports/fbi/fbigovcrime/preprocess.py +++ b/statvar_imports/fbi/fbigovcrime/preprocess.py @@ -59,19 +59,19 @@ def copy_and_process_files_from_gcs(gcs_bucket, gcs_prefix, local_base_dir): logging.info(f"Listing files from: {gcs_source_path_wildcard}") try: - result = subprocess.run(['gsutil', 'ls', gcs_source_path_wildcard], capture_output=True, text=True, check=True) + result = subprocess.run(['gcloud', 'storage', 'ls', gcs_source_path_wildcard], capture_output=True, text=True, check=True) gcs_files = result.stdout.strip().split('\n') if not gcs_files or gcs_files == ['']: logging.fatal(f"No .xlsx files found at '{gcs_source_path_wildcard}'") raise RuntimeError('No .xlsx files found') return except subprocess.CalledProcessError as e: - logging.fatal(f"Error listing files from GCS with gsutil: {e.stderr}") + logging.fatal(f"Error listing files from GCS with gcloud storage: {e.stderr}") raise RuntimeError('Error listing files from GCS') return except FileNotFoundError: - logging.fatal("gsutil command not found. Please ensure the Google Cloud SDK is installed and in your PATH.") - raise RuntimeError('gsutil command not found') + logging.fatal("gcloud storage command not found. Please ensure the Google Cloud SDK is installed and in your PATH.") + raise RuntimeError('gcloud storage command not found') return logging.info(f"Found {len(gcs_files)} xlsx files to copy and process.") diff --git a/statvar_imports/india_ndap/india_nss_health_ailments/README.md b/statvar_imports/india_ndap/india_nss_health_ailments/README.md index 5897ceb226..e18b55d5ff 100644 --- a/statvar_imports/india_ndap/india_nss_health_ailments/README.md +++ b/statvar_imports/india_ndap/india_nss_health_ailments/README.md @@ -133,7 +133,7 @@ If the download script fails or logs HTTP `401 Unauthorized` / `403 Forbidden` e ```bash gcloud storage cp download_config.json gs://unresolved_mcf/india_ndap/NDAP_NSS_Health/latest/download_config.json # or - gsutil cp download_config.json gs://unresolved_mcf/india_ndap/NDAP_NSS_Health/latest/download_config.json + gcloud storage cp download_config.json gs://unresolved_mcf/india_ndap/NDAP_NSS_Health/latest/download_config.json ``` 3. **Verify Download**: * Re-run `python3 download_script.py` and verify `india_nss_health_ailments.csv` downloads successfully. From af48959deec82131c531ede2046636130d9f4f50 Mon Sep 17 00:00:00 2001 From: gurusai-voleti Date: Mon, 7 Sep 2026 13:17:11 +0530 Subject: [PATCH 2/2] Update README.md --- statvar_imports/india_ndap/india_nss_health_ailments/README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/statvar_imports/india_ndap/india_nss_health_ailments/README.md b/statvar_imports/india_ndap/india_nss_health_ailments/README.md index e18b55d5ff..6c0540c52c 100644 --- a/statvar_imports/india_ndap/india_nss_health_ailments/README.md +++ b/statvar_imports/india_ndap/india_nss_health_ailments/README.md @@ -132,8 +132,6 @@ If the download script fails or logs HTTP `401 Unauthorized` / `403 Forbidden` e * Upload the updated configuration file to GCS: ```bash gcloud storage cp download_config.json gs://unresolved_mcf/india_ndap/NDAP_NSS_Health/latest/download_config.json - # or - gcloud storage cp download_config.json gs://unresolved_mcf/india_ndap/NDAP_NSS_Health/latest/download_config.json ``` 3. **Verify Download**: * Re-run `python3 download_script.py` and verify `india_nss_health_ailments.csv` downloads successfully.