Log the source of every configuration key

codecarbon/core/config.py

@@ -1,10 +1,13 @@
 import configparser
 import os
 from pathlib import Path
-from typing import List, Optional, Union
+from typing import Dict, List, Optional, Tuple, Union
 
 from codecarbon.external.logger import logger
 
+CONFIG_FILE_NAME = ".codecarbon.config"
+CONFIG_SECTION = "codecarbon"
+
 
 def clean_env_key(k: str) -> str:
     """
@@ -36,7 +39,7 @@ def parse_env_config() -> dict:
         dict: existing relevant environment variables mapped into a dict
     """
     return {
-        "codecarbon": {
+        CONFIG_SECTION: {
             clean_env_key(k): v
             for k, v in os.environ.items()
             if k.lower().startswith("codecarbon_")
@@ -110,21 +113,89 @@ def normalize_gpu_ids(
     return None
 
 
+def read_config_file(path: str) -> dict:
+    """
+    Read the `[codecarbon]` section of a single configuration file.
+
+    A file that does not exist, or that cannot be read, is ignored, like
+    configparser.ConfigParser.read does.
+
+    Args:
+        path (str): Path of the configuration file to read.
+
+    Returns:
+        dict: The keys defined in this file only, empty if there is nothing to read.
+    """
+    config = configparser.ConfigParser()
+    config.read(path)
+    if not config.has_section(CONFIG_SECTION):
+        return {}
+    return dict(config[CONFIG_SECTION])
+
+
+def log_config_sources(sources: Dict[str, str]) -> None:
+    """
+    Log a single line telling, for every configuration key, which layer set it.
+
+    Only the key names and their source are logged, never the values, because
+    this configuration holds secrets such as `api_key` and `experiment_id`.
+
+    Args:
+        sources (dict): The key to source mapping built by
+            get_hierarchical_config_with_sources.
+    """
+    if not sources:
+        return
+    logger.info(
+        "Codecarbon configuration: "
+        + ", ".join(f"{key} from {source}" for key, source in sorted(sources.items()))
+    )
+
+
+def get_hierarchical_config_with_sources() -> Tuple[dict, Dict[str, str]]:
+    """
+    Same as get_hierarchical_config, but also tells where each key comes from.
+
+    The layers are read from the lowest to the highest priority: the global
+    config file, then the local config file, then the environment variables.
+    The last layer defining a key wins, and is the one reported as its source.
+    A source is either the path of the config file that set the key, or the
+    name of the environment variable, e.g. `CODECARBON_PROJECT_NAME`.
+
+    Returns:
+        tuple: The (config, sources) dicts, which share the same keys. `config`
+        maps a key to its value (**all values are strings**) and `sources` maps
+        a key to the layer it was read from.
+    """
+    global_path = str((Path.home() / CONFIG_FILE_NAME).expanduser().resolve())
+    local_path = str((Path.cwd() / CONFIG_FILE_NAME).expanduser().resolve())
+
+    config = {}
+    sources = {}
+
+    for path in (global_path, local_path):
+        for key, value in read_config_file(path).items():
+            config[key] = value
+            sources[key] = path
+
+    for key, value in parse_env_config()[CONFIG_SECTION].items():
+        config[key] = value
+        sources[key] = f"CODECARBON_{key.upper()}"
+
+    return config, sources
+
+
 def get_hierarchical_config():
     """
-    Get the user-defined codecarbon configuration ConfigParser dictionnary
-    (actually a configparser.SectionProxy instance).
+    Get the user-defined codecarbon configuration as a dict.
 
     ```
     >>> from codecarbon.core.config import get_hierarchical_config
     >>> conf = get_hierarchical_config()
     >>> print(conf)
     ```
 
-    `conf` works like a regular dict + methods getint(key) getfloat(key)
-    and getboolean(key) to automatically parse strings into those types.
-
-    All values (outputs of get(key)) are strings.
+    All values are strings.
 
     It looks for, and reads, a config file .codecarbon.config in the user's $HOME.
     It then looks for, reads, and updates the previous configuration from a config
@@ -133,29 +204,13 @@ def get_hierarchical_config():
     with `CODECARBON_` (for instance if `CODECARBON_PROJECT_NAME` is `your-project`
     then the resulting configuration key `project_name` will have value `your-project`)
 
+    The source of every resolved key is logged, so that a surprising value can be
+    traced back to the file or the environment variable that set it.
+
     Returns:
         dict: The final configuration dict parsed from global,
         local and environment configurations. **All values are strings**.
     """
-
-    config = configparser.ConfigParser()
-
-    cwd = Path.cwd()
-    home = Path.home()
-    global_path = str((home / ".codecarbon.config").expanduser().resolve())
-    local_path = str((cwd / ".codecarbon.config").expanduser().resolve())
-    if Path(global_path).exists():
-        logger.info(
-            f"Codecarbon is taking the configuration from global file: {global_path}"
-        )
-        if Path(local_path).exists():
-            logger.info(f"Some variables are overriden by the local file: {local_path}")
-    elif Path(local_path).exists():
-        logger.info(
-            f"Codecarbon is taking the configuration from the local file {local_path}"
-        )
-
-    config.read([global_path, local_path])
-    config.read_dict(parse_env_config())
-
-    return dict(config["codecarbon"])
+    config, sources = get_hierarchical_config_with_sources()
+    log_config_sources(sources)
+    return config

docs/how-to/configuration.md

@@ -76,6 +76,19 @@ Yields attributes:
 }
 ```
 
+!!! note "Where does a value come from?"
+
+    CodeCarbon logs one line listing every configuration key read from the files
+    and the environment, with the layer it came from, so you can trace a
+    surprising value back to its source:
+
+    ```
+    [codecarbon INFO @ 09:30:00] Codecarbon configuration: experiment_id from CODECARBON_EXPERIMENT_ID, measure_power_secs from /home/victor/.codecarbon.config
+    ```
+
+    Only the key names and their source are logged, never the values, as this
+    configuration holds secrets such as `api_key`.
+
 !!! note "Note"
 
     If you're wondering about the configuration files' syntax, be aware

tests/test_config.py

@@ -1,12 +1,14 @@
 import os
 import unittest
+from pathlib import Path
 from textwrap import dedent
 from unittest import mock
 from unittest.mock import patch
 
 from codecarbon.core.config import (
     clean_env_key,
     get_hierarchical_config,
+    get_hierarchical_config_with_sources,
     normalize_gpu_ids,
     parse_env_config,
     parse_gpu_ids,
@@ -170,6 +172,65 @@ def test_read_confs_and_parse_envs(self):
             }
             self.assertDictEqual(conf, target)
 
+    @mock.patch.dict(
+        os.environ,
+        {
+            "CODECARBON_EXPERIMENT_ID": "SUCCESS:overwritten",
+        },
+    )
+    def test_config_sources(self):
+        global_conf = dedent("""\
+            [codecarbon]
+            no_overwrite=path/to/somewhere
+            local_overwrite=ERROR:not overwritten
+            experiment_id=ERROR:not overwritten
+            """)
+        local_conf = dedent("""\
+            [codecarbon]
+            local_overwrite=SUCCESS:overwritten
+            local_new_key=cool value
+            """)
+
+        global_path = str((Path.home() / ".codecarbon.config").expanduser().resolve())
+        local_path = str((Path.cwd() / ".codecarbon.config").expanduser().resolve())
+
+        with patch(
+            "builtins.open", new_callable=get_custom_mock_open(global_conf, local_conf)
+        ):
+            conf, sources = get_hierarchical_config_with_sources()
+
+        self.assertEqual(conf["local_overwrite"], "SUCCESS:overwritten")
+        self.assertEqual(conf["experiment_id"], "SUCCESS:overwritten")
+        # Every key has a source, and it is the last layer that set it.
+        self.assertEqual(set(conf), set(sources))
+        self.assertEqual(sources["no_overwrite"], global_path)
+        self.assertEqual(sources["local_overwrite"], local_path)
+        self.assertEqual(sources["local_new_key"], local_path)
+        self.assertEqual(sources["experiment_id"], "CODECARBON_EXPERIMENT_ID")
+        self.assertEqual(
+            sources["allow_multiple_runs"], "CODECARBON_ALLOW_MULTIPLE_RUNS"
+        )
+
+    def test_config_sources_are_logged_without_values(self):
+        global_conf = dedent("""\
+            [codecarbon]
+            api_key=do-not-log-me
+            measure_power_secs=10
+            """)
+
+        with patch("builtins.open", new_callable=get_custom_mock_open(global_conf, "")):
+            with self.assertLogs("codecarbon", level="INFO") as logs:
+                get_hierarchical_config()
+
+        message = "\n".join(logs.output)
+        global_path = str((Path.home() / ".codecarbon.config").expanduser().resolve())
+        self.assertIn(f"api_key from {global_path}", message)
+        self.assertIn(f"measure_power_secs from {global_path}", message)
+        self.assertIn(
+            "allow_multiple_runs from CODECARBON_ALLOW_MULTIPLE_RUNS", message
+        )
+        self.assertNotIn("do-not-log-me", message)
+
     def test_empty_conf(self):
         global_conf = ""
         local_conf = ""