diff --git a/codecarbon/core/config.py b/codecarbon/core/config.py index 7cacea41d..c94835ad8 100644 --- a/codecarbon/core/config.py +++ b/codecarbon/core/config.py @@ -1,10 +1,13 @@ import configparser import os from pathlib import Path -from typing import List, Optional, Union +from typing import Dict, List, Optional, Tuple, Union from codecarbon.external.logger import logger +CONFIG_FILE_NAME = ".codecarbon.config" +CONFIG_SECTION = "codecarbon" + def clean_env_key(k: str) -> str: """ @@ -36,7 +39,7 @@ def parse_env_config() -> dict: dict: existing relevant environment variables mapped into a dict """ return { - "codecarbon": { + CONFIG_SECTION: { clean_env_key(k): v for k, v in os.environ.items() if k.lower().startswith("codecarbon_") @@ -110,10 +113,81 @@ def normalize_gpu_ids( return None +def read_config_file(path: str) -> dict: + """ + Read the `[codecarbon]` section of a single configuration file. + + A file that does not exist, or that cannot be read, is ignored, like + configparser.ConfigParser.read does. + + Args: + path (str): Path of the configuration file to read. + + Returns: + dict: The keys defined in this file only, empty if there is nothing to read. + """ + config = configparser.ConfigParser() + config.read(path) + if not config.has_section(CONFIG_SECTION): + return {} + return dict(config[CONFIG_SECTION]) + + +def log_config_sources(sources: Dict[str, str]) -> None: + """ + Log a single line telling, for every configuration key, which layer set it. + + Only the key names and their source are logged, never the values, because + this configuration holds secrets such as `api_key` and `experiment_id`. + + Args: + sources (dict): The key to source mapping built by + get_hierarchical_config_with_sources. + """ + if not sources: + return + logger.info( + "Codecarbon configuration: " + + ", ".join(f"{key} from {source}" for key, source in sorted(sources.items())) + ) + + +def get_hierarchical_config_with_sources() -> Tuple[dict, Dict[str, str]]: + """ + Same as get_hierarchical_config, but also tells where each key comes from. + + The layers are read from the lowest to the highest priority: the global + config file, then the local config file, then the environment variables. + The last layer defining a key wins, and is the one reported as its source. + A source is either the path of the config file that set the key, or the + name of the environment variable, e.g. `CODECARBON_PROJECT_NAME`. + + Returns: + tuple: The (config, sources) dicts, which share the same keys. `config` + maps a key to its value (**all values are strings**) and `sources` maps + a key to the layer it was read from. + """ + global_path = str((Path.home() / CONFIG_FILE_NAME).expanduser().resolve()) + local_path = str((Path.cwd() / CONFIG_FILE_NAME).expanduser().resolve()) + + config = {} + sources = {} + + for path in (global_path, local_path): + for key, value in read_config_file(path).items(): + config[key] = value + sources[key] = path + + for key, value in parse_env_config()[CONFIG_SECTION].items(): + config[key] = value + sources[key] = f"CODECARBON_{key.upper()}" + + return config, sources + + def get_hierarchical_config(): """ - Get the user-defined codecarbon configuration ConfigParser dictionnary - (actually a configparser.SectionProxy instance). + Get the user-defined codecarbon configuration as a dict. ``` >>> from codecarbon.core.config import get_hierarchical_config @@ -121,10 +195,7 @@ def get_hierarchical_config(): >>> print(conf) ``` - `conf` works like a regular dict + methods getint(key) getfloat(key) - and getboolean(key) to automatically parse strings into those types. - - All values (outputs of get(key)) are strings. + All values are strings. It looks for, and reads, a config file .codecarbon.config in the user's $HOME. It then looks for, reads, and updates the previous configuration from a config @@ -133,29 +204,13 @@ def get_hierarchical_config(): with `CODECARBON_` (for instance if `CODECARBON_PROJECT_NAME` is `your-project` then the resulting configuration key `project_name` will have value `your-project`) + The source of every resolved key is logged, so that a surprising value can be + traced back to the file or the environment variable that set it. + Returns: dict: The final configuration dict parsed from global, local and environment configurations. **All values are strings**. """ - - config = configparser.ConfigParser() - - cwd = Path.cwd() - home = Path.home() - global_path = str((home / ".codecarbon.config").expanduser().resolve()) - local_path = str((cwd / ".codecarbon.config").expanduser().resolve()) - if Path(global_path).exists(): - logger.info( - f"Codecarbon is taking the configuration from global file: {global_path}" - ) - if Path(local_path).exists(): - logger.info(f"Some variables are overriden by the local file: {local_path}") - elif Path(local_path).exists(): - logger.info( - f"Codecarbon is taking the configuration from the local file {local_path}" - ) - - config.read([global_path, local_path]) - config.read_dict(parse_env_config()) - - return dict(config["codecarbon"]) + config, sources = get_hierarchical_config_with_sources() + log_config_sources(sources) + return config diff --git a/docs/how-to/configuration.md b/docs/how-to/configuration.md index 9f6766aa1..9075470e9 100644 --- a/docs/how-to/configuration.md +++ b/docs/how-to/configuration.md @@ -76,6 +76,19 @@ Yields attributes: } ``` +!!! note "Where does a value come from?" + + CodeCarbon logs one line listing every configuration key read from the files + and the environment, with the layer it came from, so you can trace a + surprising value back to its source: + + ``` + [codecarbon INFO @ 09:30:00] Codecarbon configuration: experiment_id from CODECARBON_EXPERIMENT_ID, measure_power_secs from /home/victor/.codecarbon.config + ``` + + Only the key names and their source are logged, never the values, as this + configuration holds secrets such as `api_key`. + !!! note "Note" If you're wondering about the configuration files' syntax, be aware diff --git a/tests/test_config.py b/tests/test_config.py index 9a11515e6..e913d9f36 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,5 +1,6 @@ import os import unittest +from pathlib import Path from textwrap import dedent from unittest import mock from unittest.mock import patch @@ -7,6 +8,7 @@ from codecarbon.core.config import ( clean_env_key, get_hierarchical_config, + get_hierarchical_config_with_sources, normalize_gpu_ids, parse_env_config, parse_gpu_ids, @@ -170,6 +172,65 @@ def test_read_confs_and_parse_envs(self): } self.assertDictEqual(conf, target) + @mock.patch.dict( + os.environ, + { + "CODECARBON_EXPERIMENT_ID": "SUCCESS:overwritten", + }, + ) + def test_config_sources(self): + global_conf = dedent("""\ + [codecarbon] + no_overwrite=path/to/somewhere + local_overwrite=ERROR:not overwritten + experiment_id=ERROR:not overwritten + """) + local_conf = dedent("""\ + [codecarbon] + local_overwrite=SUCCESS:overwritten + local_new_key=cool value + """) + + global_path = str((Path.home() / ".codecarbon.config").expanduser().resolve()) + local_path = str((Path.cwd() / ".codecarbon.config").expanduser().resolve()) + + with patch( + "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf) + ): + conf, sources = get_hierarchical_config_with_sources() + + self.assertEqual(conf["local_overwrite"], "SUCCESS:overwritten") + self.assertEqual(conf["experiment_id"], "SUCCESS:overwritten") + # Every key has a source, and it is the last layer that set it. + self.assertEqual(set(conf), set(sources)) + self.assertEqual(sources["no_overwrite"], global_path) + self.assertEqual(sources["local_overwrite"], local_path) + self.assertEqual(sources["local_new_key"], local_path) + self.assertEqual(sources["experiment_id"], "CODECARBON_EXPERIMENT_ID") + self.assertEqual( + sources["allow_multiple_runs"], "CODECARBON_ALLOW_MULTIPLE_RUNS" + ) + + def test_config_sources_are_logged_without_values(self): + global_conf = dedent("""\ + [codecarbon] + api_key=do-not-log-me + measure_power_secs=10 + """) + + with patch("builtins.open", new_callable=get_custom_mock_open(global_conf, "")): + with self.assertLogs("codecarbon", level="INFO") as logs: + get_hierarchical_config() + + message = "\n".join(logs.output) + global_path = str((Path.home() / ".codecarbon.config").expanduser().resolve()) + self.assertIn(f"api_key from {global_path}", message) + self.assertIn(f"measure_power_secs from {global_path}", message) + self.assertIn( + "allow_multiple_runs from CODECARBON_ALLOW_MULTIPLE_RUNS", message + ) + self.assertNotIn("do-not-log-me", message) + def test_empty_conf(self): global_conf = "" local_conf = ""