From f59ac223fdb0ba15e51521faf0e875585c363394 Mon Sep 17 00:00:00 2001 From: David Berenstein Date: Thu, 20 Aug 2026 08:13:16 +0200 Subject: [PATCH] fix: skip uncorrectable RAPL counter wraps When `max_energy_range_uj` is unreadable, `max_energy_reading` is 0, so the wrap-around branch still ran but corrected nothing, emitting a large negative energy delta straight into the CPU and total energy. Drop the sample instead, matching the EMI backend's behaviour. The same guard also covers driver resets and the transient read-error fallback. A failed read previously returned `Energy(0)`, which looked like a wrap-around and, with a readable `max_energy_range_uj`, injected a spurious positive delta of up to one counter range. `_get_value` now returns None, and the sample is skipped and re-baselined. The uncorrectable-wrap warning is emitted once per file instead of on every measurement cycle. Co-Authored-By: Claude Opus 5 (1M context) --- codecarbon/core/rapl.py | 31 ++++++-- tests/test_rapl_permissions.py | 127 +++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+), 5 deletions(-) diff --git a/codecarbon/core/rapl.py b/codecarbon/core/rapl.py index 5c742cf2b..fd9f4ebc3 100644 --- a/codecarbon/core/rapl.py +++ b/codecarbon/core/rapl.py @@ -1,4 +1,5 @@ from dataclasses import dataclass, field +from typing import Optional from codecarbon.core.units import Energy, Power, Time from codecarbon.external.logger import logger @@ -16,10 +17,11 @@ class RAPLFile: energy_delta: Energy = field(default_factory=lambda: Energy(0)) # Power based on reading power: Power = field(default_factory=lambda: Power(0)) - # Last energy reading in kWh - last_energy: Energy = field(default_factory=lambda: Energy(0)) + # Last energy reading in kWh, None if it could not be read + last_energy: Optional[Energy] = field(default_factory=lambda: Energy(0)) # Max value energy can hold before it wraps max_energy_reading: Energy = field(default_factory=lambda: Energy(0)) + _warned_uncorrectable_wrap: bool = field(default=False, init=False) def __post_init__(self): self.last_energy = self._get_value() @@ -44,9 +46,9 @@ def __post_init__(self): ) self.max_energy_reading = Energy.from_ujoules(0) - def _get_value(self) -> Energy: + def _get_value(self) -> Optional[Energy]: """ - Reads the value in the file at the path + Reads the value in the file at the path, or None if it cannot be read. """ try: with open(self.path, "r") as f: @@ -62,21 +64,40 @@ def _get_value(self) -> Energy: ) else: logger.debug("Unable to read RAPL value from %s: %s", self.path, e) - return Energy.from_ujoules(0) + return None def start(self) -> None: self.last_energy = self._get_value() + def _skip_sample(self, new_last_energy: Optional[Energy]) -> None: + self.energy_delta = Energy(0) + self.power = Power(0) + self.last_energy = new_last_energy + def delta(self, duration: Time) -> None: """ Compute the energy used since last call. """ new_last_energy = energy = self._get_value() + if energy is None or self.last_energy is None: + # no baseline to compute a delta from: re-baseline silently + self._skip_sample(new_last_energy) + return if self.last_energy > energy: logger.debug( f"In RAPLFile : Current energy value ({energy}) is lower than previous value ({self.last_energy}). Assuming wrap-around! Source file : {self.path}" ) energy = energy + self.max_energy_reading + if self.last_energy > energy: + # still backwards after correction: skip rather than report a negative delta + if not self._warned_uncorrectable_wrap: + self._warned_uncorrectable_wrap = True + logger.warning( + "In RAPLFile : counter went backwards and cannot be corrected for %s; skipping this sample (warned once).", + self.path, + ) + self._skip_sample(new_last_energy) + return self.power = self.power.from_energies_and_delay( energy, self.last_energy, duration ) diff --git a/tests/test_rapl_permissions.py b/tests/test_rapl_permissions.py index 4c522435d..f83bdf4e7 100644 --- a/tests/test_rapl_permissions.py +++ b/tests/test_rapl_permissions.py @@ -110,3 +110,130 @@ def emit(self, record): os.chmod(energy1, stat.S_IMODE(mode_before) or 0o644) except Exception: pass + + +def test_rapl_wraparound_without_max_skips_sample(tmp_path): + """A wrap-around with an unknown max range must not yield a negative delta.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Time + from codecarbon.external.logger import logger as codecarbon_logger + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + + log_records = [] + + class TestHandler(logging.Handler): + def emit(self, record): + log_records.append(record) + + test_handler = TestHandler() + test_handler.setLevel(logging.WARNING) + codecarbon_logger.addHandler(test_handler) + + try: + rapl_file = RAPLFile( + name="package-0", + path=str(energy_file), + max_path=str(tmp_path / "does_not_exist"), + ) + rapl_file.start() + energy_file.write_text("10000") + rapl_file.delta(Time.from_seconds(10)) + + assert rapl_file.energy_delta.kWh == 0 + assert rapl_file.power.W == 0 + assert any( + "counter went backwards" in r.getMessage() for r in log_records + ), f"Expected warning, got: {[r.getMessage() for r in log_records]}" + finally: + codecarbon_logger.removeHandler(test_handler) + + +def test_rapl_wraparound_with_max_is_corrected(tmp_path): + """A wrap-around with a known max range is still corrected.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Energy, Time + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + max_file = tmp_path / "max_energy_range_uj" + max_file.write_text("4294967295") + + rapl_file = RAPLFile( + name="package-0", path=str(energy_file), max_path=str(max_file) + ) + rapl_file.start() + energy_file.write_text("10000") + rapl_file.delta(Time.from_seconds(10)) + + expected = Energy.from_ujoules(10000 + 4294967295 - 4000000000).kWh + assert rapl_file.energy_delta.kWh == pytest.approx(expected) + + +def test_rapl_read_error_does_not_inject_energy(tmp_path): + """A transient read error must not be mistaken for a wrap-around.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Energy, Time + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + max_file = tmp_path / "max_energy_range_uj" + max_file.write_text("4294967295") + + rapl_file = RAPLFile( + name="package-0", path=str(energy_file), max_path=str(max_file) + ) + rapl_file.start() + + # Unreadable content : previously this returned 0 and looked like a wrap. + energy_file.write_text("") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == 0 + assert rapl_file.power.W == 0 + + # The next sample re-baselines instead of counting the gap twice. + energy_file.write_text("4000010000") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == 0 + + energy_file.write_text("4000020000") + rapl_file.delta(Time.from_seconds(10)) + assert rapl_file.energy_delta.kWh == pytest.approx(Energy.from_ujoules(10000).kWh) + + +def test_rapl_uncorrectable_wrap_warns_once(tmp_path): + """The uncorrectable-wrap warning must not repeat on every cycle.""" + from codecarbon.core.rapl import RAPLFile + from codecarbon.core.units import Time + from codecarbon.external.logger import logger as codecarbon_logger + + energy_file = tmp_path / "energy_uj" + energy_file.write_text("4000000000") + + log_records = [] + + class TestHandler(logging.Handler): + def emit(self, record): + log_records.append(record) + + test_handler = TestHandler() + test_handler.setLevel(logging.WARNING) + codecarbon_logger.addHandler(test_handler) + try: + rapl_file = RAPLFile( + name="package-0", + path=str(energy_file), + max_path=str(tmp_path / "does_not_exist"), + ) + rapl_file.start() + for value in ("10000", "9000", "8000"): + energy_file.write_text(value) + rapl_file.delta(Time.from_seconds(10)) + + backwards = [ + r for r in log_records if "counter went backwards" in r.getMessage() + ] + assert len(backwards) == 1, [r.getMessage() for r in log_records] + finally: + codecarbon_logger.removeHandler(test_handler)