Skip to content
Draft
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 26 additions & 12 deletions cmd/metrics/metadata.go
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,7 @@ var baseMetadataScripts = []script.ScriptDefinition{
{
Name: scriptPerfSupportedEvents,
ScriptTemplate: `# Parse perf list JSON output to extract Hardware events and cstate/power events
perf list --json 2>/dev/null | awk '
timeout 30 perf list --json 2>/dev/null | awk '
BEGIN {
in_hardware_event = 0
event_name = ""
Expand Down Expand Up @@ -192,7 +192,7 @@ BEGIN {
{
Name: scriptPerfAllSupportedEvents,
ScriptTemplate: `# Parse perf list JSON output to extract Hardware events and cstate/power events
perf list --json 2>/dev/null | awk '
timeout 30 perf list --json 2>/dev/null | awk '
BEGIN {
event_name = ""
}
Expand Down Expand Up @@ -226,52 +226,52 @@ BEGIN {
},
{
Name: scriptPerfStatInstructions,
ScriptTemplate: "perf stat -a -e instructions sleep 1",
ScriptTemplate: "timeout 30 perf stat -a -e instructions sleep 1",
Depends: []string{"perf"},
},
{
Name: scriptPerfStatRefCycles,
ScriptTemplate: "perf stat -a -e ref-cycles sleep 1",
ScriptTemplate: "timeout 30 perf stat -a -e ref-cycles sleep 1",
Depends: []string{"perf"},
},
{
Name: scriptPerfStatPEBS,
ScriptTemplate: "perf stat -a -e INT_MISC.UNKNOWN_BRANCH_CYCLES sleep 1",
ScriptTemplate: "timeout 30 perf stat -a -e INT_MISC.UNKNOWN_BRANCH_CYCLES sleep 1",
Architectures: []string{cpus.X86Architecture},
Depends: []string{"perf"},
},
{
Name: scriptPerfStatOCR,
ScriptTemplate: "perf stat -a -e OCR.READS_TO_CORE.LOCAL_DRAM sleep 1",
ScriptTemplate: "timeout 30 perf stat -a -e OCR.READS_TO_CORE.LOCAL_DRAM sleep 1",
Architectures: []string{cpus.X86Architecture},
Depends: []string{"perf"},
},
{
Name: scriptPerfStatTMA,
ScriptTemplate: "perf stat -a -e '{topdown.slots, topdown-bad-spec}' sleep 1",
ScriptTemplate: "timeout 30 perf stat -a -e '{topdown.slots, topdown-bad-spec}' sleep 1",
Architectures: []string{cpus.X86Architecture},
Depends: []string{"perf"},
},
{
Name: scriptPerfStatAMDUncoreProbe,
ScriptTemplate: `perf stat -a -e "l3/event=0x4,umask=0xff,enallcores=0x1,enallslices=0x1,threadmask=0x3,name='l3_lookup_state.all_coherent_accesses_to_l3'/" sleep 1`,
ScriptTemplate: `timeout 30 perf stat -a -e "l3/event=0x4,umask=0xff,enallcores=0x1,enallslices=0x1,threadmask=0x3,name='l3_lookup_state.all_coherent_accesses_to_l3'/" sleep 1`,
Architectures: []string{cpus.X86Architecture},
Vendors: []string{cpus.AMDVendor},
Depends: []string{"perf"},
},
{
Name: scriptPerfStatFixedInstr,
ScriptTemplate: "perf stat -a -e '{{{.InstructionsList}}}' sleep 1",
ScriptTemplate: "timeout 30 perf stat -a -e '{{{.InstructionsList}}}' sleep 1",
Depends: []string{"perf"},
},
{
Name: scriptPerfStatFixedCycles,
ScriptTemplate: "perf stat -a -e '{{{.CpuCyclesList}}}' sleep 1",
ScriptTemplate: "timeout 30 perf stat -a -e '{{{.CpuCyclesList}}}' sleep 1",
Depends: []string{"perf"},
},
{
Name: scriptPerfStatFixedRefCycles,
ScriptTemplate: "perf stat -a -e '{{{.RefCyclesList}}}' sleep 1",
ScriptTemplate: "timeout 30 perf stat -a -e '{{{.RefCyclesList}}}' sleep 1",
Depends: []string{"perf"},
},
{
Expand Down Expand Up @@ -300,15 +300,25 @@ BEGIN {
},
}

// metadataScriptTimeout bounds how long any single metadata script may run.
// Metadata collection is a bounded probing phase -- every script here reads a
// sysfs/procfs value, runs a tool, or runs 'perf stat ... sleep 1' -- so none has
// a legitimate reason to run for long. Without a bound, a probe that wedges (e.g.
// a perf event that hangs the PMU on some virtualized instance types) stalls
// collection indefinitely, because the controller waits on it forever.
const metadataScriptTimeout = 60

// getMetadataScripts returns the list of scripts to run for metadata collection.
// It copies the base definitions and applies template replacements and privilege settings.
// It copies the base definitions and applies template replacements, privilege
// settings, and the metadata script timeout.
func getMetadataScripts(noRoot bool, noSystemSummary bool, numGPCounters int) ([]script.ScriptDefinition, error) {
metadataScripts := make([]script.ScriptDefinition, 0, len(baseMetadataScripts))

// Copy base scripts and apply settings
for _, baseDef := range baseMetadataScripts {
scriptDef := baseDef
scriptDef.Superuser = !noRoot
scriptDef.Timeout = metadataScriptTimeout

// Apply template replacements for fixed counter scripts
switch scriptDef.Name {
Expand Down Expand Up @@ -339,6 +349,10 @@ func getMetadataScripts(noRoot bool, noSystemSummary bool, numGPCounters int) ([
if !noSystemSummary {
for _, scriptName := range app.TableDefinitions[app.SystemSummaryTableName].ScriptNames {
scriptDef := script.GetScriptByName(scriptName)
// Only tighten the budget; never loosen one a script set for itself.
if scriptDef.Timeout == 0 || scriptDef.Timeout > metadataScriptTimeout {
scriptDef.Timeout = metadataScriptTimeout
}
metadataScripts = append(metadataScripts, scriptDef)
}
}
Expand Down
Loading