Skip to content

25.3 Stable Backport of #84317 - Report cgroup and system-wide metrics along with each other #2333

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
mkmkme merged 2 commits into from
Sep 11, 2026
+55 −26
Merged

25.3 Stable Backport of #84317 - Report cgroup and system-wide metrics along with each other #2333

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
ea6b24e
Merge pull request #84317 from ClickHouse/fix_metrics
nickitat Jul 30, 2025
188b160
Merge branch 'releases/25.3.14' into backports/25.3.14/84317
MyroTk Sep 10, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 38 additions & 15 deletions src/Common/AsynchronousMetrics.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -594,6 +594,19 @@ AsynchronousMetrics::NetworkInterfaceStatValues::operator-(const AsynchronousMet


#if defined(OS_LINUX)

void AsynchronousMetrics::applyCgroupCPUMetricsUpdate(
AsynchronousMetricValues & new_values, const ProcStatValuesCPU & delta_values, double multiplier)
{
new_values["CGroupUserTime"]
= {delta_values.user * multiplier,
"The ratio of time the CPU core was running userspace code."
" This includes also the time when the CPU was under-utilized due to the reasons internal to the CPU (memory loads, pipeline "
"stalls, branch mispredictions, running another SMT core)."};
new_values["CGroupSystemTime"]
= {delta_values.system * multiplier, "The ratio of time the CPU core was running OS kernel (system) code."};
}

void AsynchronousMetrics::applyCPUMetricsUpdate(
AsynchronousMetricValues & new_values, const std::string & cpu_suffix, const ProcStatValuesCPU & delta_values, double multiplier)
{
Expand Down Expand Up @@ -670,6 +683,25 @@ void AsynchronousMetrics::applyCPUMetricsUpdate(
"them [0..num cores]."};
}

void AsynchronousMetrics::applyCgroupNormalizedCPUMetricsUpdate(
AsynchronousMetricValues & new_values, double num_cpus_to_normalize, const ProcStatValuesCPU & delta_values_all_cpus, double multiplier)
{
chassert(num_cpus_to_normalize);

new_values["CGroupUserTimeNormalized"]
= {delta_values_all_cpus.user * multiplier / num_cpus_to_normalize,
"The value is similar to `CGroupUserTime` but divided by the number of available CPU cores to be measured in the [0..1] "
"interval regardless of the number of cores."
" This allows you to average the values of this metric across multiple servers in a cluster even if the number of cores is "
"non-uniform, and still get the average resource utilization metric."};
new_values["CGroupSystemTimeNormalized"]
= {delta_values_all_cpus.system * multiplier / num_cpus_to_normalize,
"The value is similar to `CGroupSystemTime` but divided by the number of available CPU cores to be measured in the [0..1] "
"interval regardless of the number of cores."
" This allows you to average the values of this metric across multiple servers in a cluster even if the number of cores is "
"non-uniform, and still get the average resource utilization metric."};
}

void AsynchronousMetrics::applyNormalizedCPUMetricsUpdate(
AsynchronousMetricValues & new_values, double num_cpus_to_normalize, const ProcStatValuesCPU & delta_values_all_cpus, double multiplier)
{
Expand Down Expand Up @@ -1003,8 +1035,7 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update)
new_values["CGroupMaxCPU"] = { max_cpu_cgroups, "The maximum number of CPU cores according to CGroups."};
}

const bool cgroup_cpu_metrics_present = cgroupcpu_stat || cgroupcpuacct_stat;
if (cgroup_cpu_metrics_present)
if (cgroupcpu_stat || cgroupcpuacct_stat)
{
try
{
Expand Down Expand Up @@ -1048,13 +1079,13 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update)
const double multiplier = 1.0 / cgroup_version_specific_divisor
/ (std::chrono::duration_cast<std::chrono::nanoseconds>(time_since_previous_update).count() / 1e9);

const ProcStatValuesCPU delta_values = current_values - proc_stat_values_all_cpus;
applyCPUMetricsUpdate(new_values, /*cpu_suffix=*/"", delta_values, multiplier);
const ProcStatValuesCPU delta_values = current_values - cgroup_values_all_cpus;
applyCgroupCPUMetricsUpdate(new_values, delta_values, multiplier);
if (max_cpu_cgroups > 0)
applyNormalizedCPUMetricsUpdate(new_values, max_cpu_cgroups, delta_values, multiplier);
applyCgroupNormalizedCPUMetricsUpdate(new_values, max_cpu_cgroups, delta_values, multiplier);
}

proc_stat_values_all_cpus = current_values;
cgroup_values_all_cpus = current_values;
}
catch (...)
{
Expand Down Expand Up @@ -1088,14 +1119,6 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update)

if (name.starts_with("cpu"))
{
if (cgroup_cpu_metrics_present)
{
/// Skip the CPU metrics if we already have them from cgroup
ProcStatValuesCPU current_values{};
current_values.read(*proc_stat);
continue;
}

String cpu_num_str = name.substr(strlen("cpu"));
UInt64 cpu_num = 0;
if (!cpu_num_str.empty())
Expand Down Expand Up @@ -1182,7 +1205,7 @@ void AsynchronousMetrics::update(TimePoint update_time, bool force_update)

Float64 num_cpus_to_normalize = max_cpu_cgroups > 0 ? max_cpu_cgroups : num_cpus;

if (num_cpus_to_normalize > 0 && !cgroup_cpu_metrics_present)
if (num_cpus_to_normalize > 0)
applyNormalizedCPUMetricsUpdate(new_values, num_cpus_to_normalize, delta_values_all_cpus, multiplier);
}

Expand Down
9 changes: 9 additions & 0 deletions src/Common/AsynchronousMetrics.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -190,6 +190,7 @@ class AsynchronousMetrics
ProcStatValuesOther operator-(const ProcStatValuesOther & other) const;
};

ProcStatValuesCPU cgroup_values_all_cpus TSA_GUARDED_BY(data_mutex){};
ProcStatValuesCPU proc_stat_values_all_cpus TSA_GUARDED_BY(data_mutex) {};
ProcStatValuesOther proc_stat_values_other TSA_GUARDED_BY(data_mutex) {};
std::vector<ProcStatValuesCPU> proc_stat_values_per_cpu TSA_GUARDED_BY(data_mutex);
Expand Down Expand Up @@ -242,6 +243,14 @@ class AsynchronousMetrics
void openSensorsChips();
void openEDAC();

void applyCgroupCPUMetricsUpdate(AsynchronousMetricValues & new_values, const ProcStatValuesCPU & delta_values, double multiplier);

void applyCgroupNormalizedCPUMetricsUpdate(
AsynchronousMetricValues & new_values,
double num_cpus_to_normalize,
const ProcStatValuesCPU & delta_values_all_cpus,
double multiplier);

void applyCPUMetricsUpdate(
AsynchronousMetricValues & new_values, const std::string & cpu_suffix, const ProcStatValuesCPU & delta_values, double multiplier);

Expand Down
19 changes: 8 additions & 11 deletions tests/integration/test_async_metrics_in_cgroup/test.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -46,17 +46,12 @@ def test_user_cpu_accounting(start_cluster):
# run query on the other node, its usage shouldn't be accounted by node1
run_cpu_intensive_task(node2)

node1_cpu_time = get_async_metric(node1, "OSUserTime")
assert float(node1_cpu_time) < 2
node1_cgroup_cpu_time = get_async_metric(node1, "CGroupUserTime")
assert float(node1_cgroup_cpu_time) < 2

# then let's test that we will account cpu time spent by the server itself
node2_cpu_time = get_async_metric(node2, "OSUserTime")
node1_os_cpu_time = get_async_metric(node1, "OSUserTime")
# this check is really weak, but CI is tough place and we cannot guarantee that test process will get many cpu time
assert float(node2_cpu_time) > 2

# OSIdleTime is not available for cgroups, so it shouldn't be present
for node in [node1, node2]:
assert get_async_metric(node, "OSIdleTime") == "0"
assert float(node1_os_cpu_time) > 2


def test_normalized_user_cpu(start_cluster):
Expand All @@ -66,10 +61,10 @@ def test_normalized_user_cpu(start_cluster):
# run query on the other node, its usage shouldn't be accounted by node1
run_cpu_intensive_task(node2)

node1_cpu_time = get_async_metric(node1, "OSUserTimeNormalized")
node1_cpu_time = get_async_metric(node1, "CGroupUserTimeNormalized")
assert float(node1_cpu_time) < 1.01

node2_cpu_time = get_async_metric(node2, "OSUserTimeNormalized")
node2_cpu_time = get_async_metric(node2, "CGroupUserTimeNormalized")
assert float(node2_cpu_time) < 1.01


Expand All @@ -89,6 +84,8 @@ def test_system_wide_metrics(start_cluster):
"OSProcessesRunning",
"OSInterrupts",
"OSMemoryTotal",
"OSUserTimeNormalized",
"OSSystemTimeNormalized",
]:
node2_value = get_async_metric(node2, metric)
assert float(node2_value) > 0
Loading