From be659b3cc9bde1e7c6a04231547d639711783c9c Mon Sep 17 00:00:00 2001 From: Gabriel Date: Fri, 18 Sep 2026 15:46:31 -0400 Subject: [PATCH] Initialize APIC debug namespace before DB reads (#446) --- aci-preupgrade-validation-script.py | 12 ++- .../test_apic_database_size_check.py | 86 ++++++++++++++++++- 2 files changed, 95 insertions(+), 3 deletions(-) diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 094f0e7..1b85b74 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6153,7 +6153,7 @@ def isis_database_byte_check(tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) -# Subprocess check - cat + acidiag +# Subprocess check - ls + cat + acidiag @check_wrapper(check_title='APIC Database Size') def apic_database_size_check(cversion, **kwargs): result = PASS @@ -6181,6 +6181,16 @@ def apic_database_size_check(cversion, **kwargs): apic_id_to_name = {"2": apic_id_to_name["2"]} if cversion.older_than("6.1(3a)"): + # Populate the dynamic /debug namespace before reading legacy counter files. + try: + run_cmd('/bin/ls /debug >/dev/null 2>&1', splitlines=False) + except subprocess.CalledProcessError as error: + # Keep the preflight best-effort; per-file reads remain authoritative. + log.warning( + 'Unable to initialize /debug before APIC database collection: %s', + error, + ) + for dme in dme_svc_list: for id in apic_id_to_name: apic_hostname = apic_id_to_name[id] diff --git a/tests/checks/apic_database_size_check/test_apic_database_size_check.py b/tests/checks/apic_database_size_check/test_apic_database_size_check.py index a4b058b..d7f9801 100644 --- a/tests/checks/apic_database_size_check/test_apic_database_size_check.py +++ b/tests/checks/apic_database_size_check/test_apic_database_size_check.py @@ -13,6 +13,7 @@ test_function = "apic_database_size_check" apic_node_api = 'infraWiNode.json' +debug_init_cmd = "/bin/ls /debug >/dev/null 2>&1" apic1_pm_cat = "cat /debug/apic1/policymgr/mitmocounters/mo 2>&1" apic1_pd_cat = "cat /debug/apic1/policydist/mitmocounters/mo 2>&1" @@ -177,6 +178,7 @@ ( {apic_node_api: read_data(dir, 'infraWiNode_3.json')}, { + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_pm_cat: {"splitlines": True, "output": mitcounters_policymgr_pos}, apic2_pd_cat: {"splitlines": True, "output": mitcounters_policydist_pos}, apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, @@ -189,6 +191,7 @@ ( {apic_node_api: read_data(dir, 'infraWiNode_3.json')}, { + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_pm_cat: {"splitlines": True, "output": mitcounters_neg}, apic2_pd_cat: {"splitlines": True, "output": mitcounters_neg}, apic2_vmm_cat: {"splitlines": True, "output": mitcounters_neg}, @@ -225,6 +228,7 @@ ( {apic_node_api: read_data(dir, 'infraWiNode_4.json')}, { + debug_init_cmd: {"splitlines": False, "output": ""}, apic1_pm_cat: {"splitlines": True, "output": mitcounters_policymgr_pos}, apic1_pd_cat: {"splitlines": True, "output": mitcounters_policydist_pos}, apic1_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, @@ -249,6 +253,7 @@ ( {apic_node_api: read_data(dir, 'infraWiNode_4.json')}, { + debug_init_cmd: {"splitlines": False, "output": ""}, apic1_pm_cat: {"splitlines": True, "output": mitcounters_neg}, apic1_pd_cat: {"splitlines": True, "output": mitcounters_neg}, apic1_vmm_cat: {"splitlines": True, "output": mitcounters_neg}, @@ -370,6 +375,7 @@ def test_missing_mitmocounters_returns_error( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_pm_cat: failure_details, apic2_pd_cat: failure_details, apic2_vmm_cat: failure_details, @@ -395,6 +401,7 @@ def test_collection_error_preserves_oversized_classes( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, apic2_pm_cat: { "CalledProcessError": True, @@ -427,6 +434,72 @@ def test_collection_error_preserves_oversized_classes( assert "high object counts" in result.recommended_action +def test_debug_namespace_initialized_before_counter_reads( + run_check, mock_icurl, icurl_outputs, monkeypatch +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + successful_outputs = { + apic2_vmm_cat: mitcounters_neg, + apic2_pm_cat: mitcounters_neg, + apic2_evm_cat: mitcounters_neg, + apic2_pd_cat: mitcounters_neg, + } + namespace_initialized = [False] + command_calls = [] + + def stateful_run_cmd(cmd, splitlines=False): + command_calls.append(cmd) + if cmd == debug_init_cmd: + namespace_initialized[0] = True + return "" + if not namespace_initialized[0]: + raise CalledProcessError( + 1, + cmd, + output=b"cat: file: No such file or directory\n", + ) + output = successful_outputs[cmd] + return output.splitlines() if splitlines else output + + monkeypatch.setattr(script, "run_cmd", stateful_run_cmd) + + result = run_check(cversion=script.AciVersion("5.2(2e)")) + + assert result.result == script.PASS + assert command_calls[0] == debug_init_cmd + assert command_calls.count(debug_init_cmd) == 1 + + +def test_debug_namespace_initialization_failure_is_best_effort( + run_check, mock_icurl, icurl_outputs, monkeypatch +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + successful_outputs = { + apic2_vmm_cat: mitcounters_neg, + apic2_pm_cat: mitcounters_neg, + apic2_evm_cat: mitcounters_neg, + apic2_pd_cat: mitcounters_neg, + } + + def init_failure_run_cmd(cmd, splitlines=False): + if cmd == debug_init_cmd: + raise CalledProcessError(1, cmd, output=b"ls: cannot open /debug\n") + output = successful_outputs[cmd] + return output.splitlines() if splitlines else output + + monkeypatch.setattr(script, "run_cmd", init_failure_run_cmd) + + result = run_check(cversion=script.AciVersion("5.2(2e)")) + + assert result.result == script.PASS + + def test_transient_counter_read_succeeds_on_retry( run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs, monkeypatch ): @@ -445,6 +518,8 @@ def test_transient_counter_read_succeeds_on_retry( def transient_run_cmd(cmd, splitlines=False): call_counts[cmd] = call_counts.get(cmd, 0) + 1 + if cmd == debug_init_cmd: + return "" if call_counts[cmd] == 1: raise CalledProcessError( 1, @@ -460,7 +535,8 @@ def transient_run_cmd(cmd, splitlines=False): result = run_check(cversion=script.AciVersion("6.0(8f)")) assert result.result == script.PASS - assert all(call_count == 2 for call_count in call_counts.values()) + assert call_counts[debug_init_cmd] == 1 + assert all(call_counts[cmd] == 2 for cmd in successful_outputs) assert sleep_calls.count(1) == 4 @@ -482,6 +558,8 @@ def test_empty_counter_read_succeeds_on_retry( def transient_run_cmd(cmd, splitlines=False): call_counts[cmd] = call_counts.get(cmd, 0) + 1 + if cmd == debug_init_cmd: + return "" if call_counts[cmd] == 1: return [] output = successful_outputs[cmd] @@ -493,7 +571,8 @@ def transient_run_cmd(cmd, splitlines=False): result = run_check(cversion=script.AciVersion("6.0(8f)")) assert result.result == script.PASS - assert all(call_count == 2 for call_count in call_counts.values()) + assert call_counts[debug_init_cmd] == 1 + assert all(call_counts[cmd] == 2 for cmd in successful_outputs) assert sleep_calls.count(1) == 4 @@ -513,6 +592,7 @@ def test_object_counters_are_sorted_before_top_four_and_thresholded( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_vmm_cat: {"splitlines": True, "output": unsorted_counters}, apic2_pm_cat: {"splitlines": True, "output": mitcounters_neg}, apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, @@ -538,6 +618,7 @@ def test_malformed_counter_preserves_oversized_classes( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, apic2_pm_cat: {"splitlines": True, "output": "brokenClass :"}, apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, @@ -564,6 +645,7 @@ def test_colonless_counter_data_returns_error( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_vmm_cat: { "splitlines": True, "output": "validClass : 10\ntruncatedClass",