diff --git a/aci-preupgrade-validation-script.py b/aci-preupgrade-validation-script.py index 094f0e7..13b678d 100644 --- a/aci-preupgrade-validation-script.py +++ b/aci-preupgrade-validation-script.py @@ -6153,7 +6153,7 @@ def isis_database_byte_check(tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) -# Subprocess check - cat + acidiag +# Subprocess check - ls + cat + acidiag @check_wrapper(check_title='APIC Database Size') def apic_database_size_check(cversion, **kwargs): result = PASS @@ -6181,6 +6181,16 @@ def apic_database_size_check(cversion, **kwargs): apic_id_to_name = {"2": apic_id_to_name["2"]} if cversion.older_than("6.1(3a)"): + # Populate the dynamic /debug namespace before reading legacy counter files. + try: + run_cmd('/bin/ls /debug >/dev/null 2>&1', splitlines=False) + except subprocess.CalledProcessError as error: + # Keep the preflight best-effort; per-file reads remain authoritative. + log.warning( + 'Unable to initialize /debug before APIC database collection: %s', + error, + ) + for dme in dme_svc_list: for id in apic_id_to_name: apic_hostname = apic_id_to_name[id] @@ -6342,6 +6352,55 @@ def configpush_shard_check(tversion, **kwargs): return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) +@check_wrapper(check_title='Port Tracking Minimal Uplink Zero') +def port_tracking_active_fabric_port_check(tversion, vpc_node_ids, **kwargs): + headers = ["Admin State", "Port Tracking Active Fabric Ports"] + data = [] + recommended_action = ( + 'Upgrade to a fixed release when possible. If upgrading to an affected release, either disable Port Tracking ' + 'before upgrading the leaf, or set Port Tracking Active Fabric Ports (minLink) to 1 only after verifying every ' + 'affected leaf has more than two operational fabric uplinks. If the issue has already occurred, disable Port ' + 'Tracking, reload the affected switch, and then re-enable Port Tracking.' + ) + doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#port-tracking-active-fabric-port-zero' + + if not tversion: + return Result(result=MANUAL, msg=TVER_MISSING, doc_url=doc_url) + + affected_versions = ("6.0(9d)", "6.1(3f)") + if not any(tversion.same_as(version) for version in affected_versions): + return Result(result=NA, msg=VER_NOT_AFFECTED, doc_url=doc_url) + + if not vpc_node_ids: + return Result(result=NA, msg="No vPC nodes found. Not susceptible.", doc_url=doc_url) + + port_tracking_api = 'uni/infra/trackEqptFabP-default.json' + port_tracking_mo = icurl('mo', port_tracking_api) + if not isinstance(port_tracking_mo, list) or len(port_tracking_mo) != 1: + object_count = len(port_tracking_mo) if isinstance(port_tracking_mo, list) else 0 + msg = "Expected exactly one infraPortTrackPol object, but found {}.".format(object_count) + return Result(result=ERROR, msg=msg, recommended_action=recommended_action, doc_url=doc_url) + + try: + attributes = port_tracking_mo[0]['infraPortTrackPol']['attributes'] + admin_st = attributes['adminSt'] + minimal_uplink = attributes['minlinks'] + except (KeyError, TypeError): + msg = "The infraPortTrackPol response is missing required adminSt or minlinks attributes." + return Result(result=ERROR, msg=msg, recommended_action=recommended_action, doc_url=doc_url) + + if admin_st not in ("on", "off") or not str(minimal_uplink).isdigit(): + msg = "The infraPortTrackPol response contains an invalid adminSt or minlinks value." + return Result(result=ERROR, msg=msg, recommended_action=recommended_action, doc_url=doc_url) + + result = PASS + if admin_st == "on" and str(minimal_uplink) == "0": + data.append([admin_st, minimal_uplink]) + result = FAIL_O + + return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url) + + @check_wrapper(check_title='APIC VMM inventory sync fault (F0132)') def apic_vmm_inventory_sync_faults_check(**kwargs): result = PASS @@ -7158,6 +7217,7 @@ class CheckManager: standby_sup_sync_check, isis_database_byte_check, configpush_shard_check, + port_tracking_active_fabric_port_check, auto_firmware_update_on_switch_check, rogue_ep_coop_exception_mac_check, n9k_c9408_model_lem_count_check, diff --git a/docs/docs/validations.md b/docs/docs/validations.md index 717c175..d20b6ea 100644 --- a/docs/docs/validations.md +++ b/docs/docs/validations.md @@ -199,6 +199,7 @@ Items | Defect | This Script [Stale pconsRA Object][d26] | CSCwp22212 | :warning:{title="Deprecated"} | :no_entry_sign: [ISIS DTEPs Byte Size][d27] | CSCwp15375 | :white_check_mark: | :no_entry_sign: [Policydist configpushShardCont Crash][d28] | CSCwp95515 | :white_check_mark: | :no_entry_sign: +[Port Tracking Active Fabric Port Zero][d39] | CSCwp91797 | :white_check_mark: | [Auto Firmware Update on Switch Discovery][d29] | CSCwe83941 | :white_check_mark: | :no_entry_sign: [Rogue EP Exception List missing on switches][d30] | CSCwp64296 | :white_check_mark: | :no_entry_sign: [N9K-C9408 with more than 5 N9K-X9400-16W LEMs][d31] | CSCws82819 | :white_check_mark: | :no_entry_sign: @@ -248,6 +249,7 @@ Items | Defect | This Script [d36]: #n9k-c93180yc-fx3-switch-memory-less-than-32gb [d37]: #stale-dbgacepgsummarytask-objects [d38]: #infravlan-overlap-access-policy-check +[d39]: #port-tracking-active-fabric-port-zero ## General Check Details @@ -2753,6 +2755,14 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade. +### Port Tracking Active Fabric Port Zero + +Due to [CSCwp91797][78], if port tracking is enabled and the number of active fabric ports that triggers port tracking (`minLink`) is zero, vPC port-channel member ports may remain down after a switch reload, upgrade, or boot. The affected physical links remain in the `initializing` state and MTS buffers may remain stuck on the leaf. + +The confirmed affected target releases checked by this validation are 6.0(9d) and 6.1(3f). Only fabrics containing vPC nodes are susceptible. + +Upgrade to a fixed release when possible. If an affected release must be used, either disable Port Tracking before upgrading each leaf, or change `minLink` from 0 to 1 only after verifying that every affected leaf has more than two operational fabric uplinks. If the issue has already occurred, disable Port Tracking, reload the affected switch, and then re-enable Port Tracking. + ### Auto Firmware Update on Switch Discovery @@ -2947,3 +2957,4 @@ To avoid this issue, modify the user VLAN pool ranges so that the InfraVLAN does [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698 [77]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt58626 +[78]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp91797 diff --git a/tests/checks/apic_database_size_check/test_apic_database_size_check.py b/tests/checks/apic_database_size_check/test_apic_database_size_check.py index a4b058b..d7f9801 100644 --- a/tests/checks/apic_database_size_check/test_apic_database_size_check.py +++ b/tests/checks/apic_database_size_check/test_apic_database_size_check.py @@ -13,6 +13,7 @@ test_function = "apic_database_size_check" apic_node_api = 'infraWiNode.json' +debug_init_cmd = "/bin/ls /debug >/dev/null 2>&1" apic1_pm_cat = "cat /debug/apic1/policymgr/mitmocounters/mo 2>&1" apic1_pd_cat = "cat /debug/apic1/policydist/mitmocounters/mo 2>&1" @@ -177,6 +178,7 @@ ( {apic_node_api: read_data(dir, 'infraWiNode_3.json')}, { + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_pm_cat: {"splitlines": True, "output": mitcounters_policymgr_pos}, apic2_pd_cat: {"splitlines": True, "output": mitcounters_policydist_pos}, apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, @@ -189,6 +191,7 @@ ( {apic_node_api: read_data(dir, 'infraWiNode_3.json')}, { + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_pm_cat: {"splitlines": True, "output": mitcounters_neg}, apic2_pd_cat: {"splitlines": True, "output": mitcounters_neg}, apic2_vmm_cat: {"splitlines": True, "output": mitcounters_neg}, @@ -225,6 +228,7 @@ ( {apic_node_api: read_data(dir, 'infraWiNode_4.json')}, { + debug_init_cmd: {"splitlines": False, "output": ""}, apic1_pm_cat: {"splitlines": True, "output": mitcounters_policymgr_pos}, apic1_pd_cat: {"splitlines": True, "output": mitcounters_policydist_pos}, apic1_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, @@ -249,6 +253,7 @@ ( {apic_node_api: read_data(dir, 'infraWiNode_4.json')}, { + debug_init_cmd: {"splitlines": False, "output": ""}, apic1_pm_cat: {"splitlines": True, "output": mitcounters_neg}, apic1_pd_cat: {"splitlines": True, "output": mitcounters_neg}, apic1_vmm_cat: {"splitlines": True, "output": mitcounters_neg}, @@ -370,6 +375,7 @@ def test_missing_mitmocounters_returns_error( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_pm_cat: failure_details, apic2_pd_cat: failure_details, apic2_vmm_cat: failure_details, @@ -395,6 +401,7 @@ def test_collection_error_preserves_oversized_classes( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, apic2_pm_cat: { "CalledProcessError": True, @@ -427,6 +434,72 @@ def test_collection_error_preserves_oversized_classes( assert "high object counts" in result.recommended_action +def test_debug_namespace_initialized_before_counter_reads( + run_check, mock_icurl, icurl_outputs, monkeypatch +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + successful_outputs = { + apic2_vmm_cat: mitcounters_neg, + apic2_pm_cat: mitcounters_neg, + apic2_evm_cat: mitcounters_neg, + apic2_pd_cat: mitcounters_neg, + } + namespace_initialized = [False] + command_calls = [] + + def stateful_run_cmd(cmd, splitlines=False): + command_calls.append(cmd) + if cmd == debug_init_cmd: + namespace_initialized[0] = True + return "" + if not namespace_initialized[0]: + raise CalledProcessError( + 1, + cmd, + output=b"cat: file: No such file or directory\n", + ) + output = successful_outputs[cmd] + return output.splitlines() if splitlines else output + + monkeypatch.setattr(script, "run_cmd", stateful_run_cmd) + + result = run_check(cversion=script.AciVersion("5.2(2e)")) + + assert result.result == script.PASS + assert command_calls[0] == debug_init_cmd + assert command_calls.count(debug_init_cmd) == 1 + + +def test_debug_namespace_initialization_failure_is_best_effort( + run_check, mock_icurl, icurl_outputs, monkeypatch +): + icurl_outputs.clear() + icurl_outputs.update({ + apic_node_api: read_data(dir, 'infraWiNode_3.json'), + }) + successful_outputs = { + apic2_vmm_cat: mitcounters_neg, + apic2_pm_cat: mitcounters_neg, + apic2_evm_cat: mitcounters_neg, + apic2_pd_cat: mitcounters_neg, + } + + def init_failure_run_cmd(cmd, splitlines=False): + if cmd == debug_init_cmd: + raise CalledProcessError(1, cmd, output=b"ls: cannot open /debug\n") + output = successful_outputs[cmd] + return output.splitlines() if splitlines else output + + monkeypatch.setattr(script, "run_cmd", init_failure_run_cmd) + + result = run_check(cversion=script.AciVersion("5.2(2e)")) + + assert result.result == script.PASS + + def test_transient_counter_read_succeeds_on_retry( run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs, monkeypatch ): @@ -445,6 +518,8 @@ def test_transient_counter_read_succeeds_on_retry( def transient_run_cmd(cmd, splitlines=False): call_counts[cmd] = call_counts.get(cmd, 0) + 1 + if cmd == debug_init_cmd: + return "" if call_counts[cmd] == 1: raise CalledProcessError( 1, @@ -460,7 +535,8 @@ def transient_run_cmd(cmd, splitlines=False): result = run_check(cversion=script.AciVersion("6.0(8f)")) assert result.result == script.PASS - assert all(call_count == 2 for call_count in call_counts.values()) + assert call_counts[debug_init_cmd] == 1 + assert all(call_counts[cmd] == 2 for cmd in successful_outputs) assert sleep_calls.count(1) == 4 @@ -482,6 +558,8 @@ def test_empty_counter_read_succeeds_on_retry( def transient_run_cmd(cmd, splitlines=False): call_counts[cmd] = call_counts.get(cmd, 0) + 1 + if cmd == debug_init_cmd: + return "" if call_counts[cmd] == 1: return [] output = successful_outputs[cmd] @@ -493,7 +571,8 @@ def transient_run_cmd(cmd, splitlines=False): result = run_check(cversion=script.AciVersion("6.0(8f)")) assert result.result == script.PASS - assert all(call_count == 2 for call_count in call_counts.values()) + assert call_counts[debug_init_cmd] == 1 + assert all(call_counts[cmd] == 2 for cmd in successful_outputs) assert sleep_calls.count(1) == 4 @@ -513,6 +592,7 @@ def test_object_counters_are_sorted_before_top_four_and_thresholded( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_vmm_cat: {"splitlines": True, "output": unsorted_counters}, apic2_pm_cat: {"splitlines": True, "output": mitcounters_neg}, apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, @@ -538,6 +618,7 @@ def test_malformed_counter_preserves_oversized_classes( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos}, apic2_pm_cat: {"splitlines": True, "output": "brokenClass :"}, apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg}, @@ -564,6 +645,7 @@ def test_colonless_counter_data_returns_error( }) cmd_outputs.clear() cmd_outputs.update({ + debug_init_cmd: {"splitlines": False, "output": ""}, apic2_vmm_cat: { "splitlines": True, "output": "validClass : 10\ntruncatedClass", diff --git a/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_neg.json b/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_neg.json new file mode 100644 index 0000000..ce366e3 --- /dev/null +++ b/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_neg.json @@ -0,0 +1,30 @@ +{ + "totalCount": "1", + "imdata": [ + { + "infraPortTrackPol": { + "attributes": { + "adminSt": "off", + "annotation": "", + "childAction": "", + "delay": "120", + "descr": "", + "dn": "uni/infra/trackEqptFabP-default", + "extMngdBy": "", + "includeApicPorts": "no", + "lcOwn": "local", + "minlinks": "0", + "modTs": "2025-10-31T17:13:46.220+11:00", + "name": "default", + "nameAlias": "", + "ownerKey": "", + "ownerTag": "", + "rn": "trackEqptFabP-default", + "status": "", + "uid": "0", + "userdom": "all" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_neg1.json b/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_neg1.json new file mode 100644 index 0000000..683f7a5 --- /dev/null +++ b/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_neg1.json @@ -0,0 +1,30 @@ +{ + "totalCount": "1", + "imdata": [ + { + "infraPortTrackPol": { + "attributes": { + "adminSt": "on", + "annotation": "", + "childAction": "", + "delay": "120", + "descr": "", + "dn": "uni/infra/trackEqptFabP-default", + "extMngdBy": "", + "includeApicPorts": "no", + "lcOwn": "local", + "minlinks": "1", + "modTs": "2025-10-31T17:13:46.220+11:00", + "name": "default", + "nameAlias": "", + "ownerKey": "", + "ownerTag": "", + "rn": "trackEqptFabP-default", + "status": "", + "uid": "0", + "userdom": "all" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_pos.json b/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_pos.json new file mode 100644 index 0000000..ed137ae --- /dev/null +++ b/tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_pos.json @@ -0,0 +1,30 @@ +{ + "totalCount": "1", + "imdata": [ + { + "infraPortTrackPol": { + "attributes": { + "adminSt": "on", + "annotation": "", + "childAction": "", + "delay": "120", + "descr": "", + "dn": "uni/infra/trackEqptFabP-default", + "extMngdBy": "", + "includeApicPorts": "no", + "lcOwn": "local", + "minlinks": "0", + "modTs": "2025-10-31T17:13:46.220+11:00", + "name": "default", + "nameAlias": "", + "ownerKey": "", + "ownerTag": "", + "rn": "trackEqptFabP-default", + "status": "", + "uid": "0", + "userdom": "all" + } + } + } + ] +} \ No newline at end of file diff --git a/tests/checks/port_tracking_active_fabric_port_check/test_port_tracking_active_fabric_port_check.py b/tests/checks/port_tracking_active_fabric_port_check/test_port_tracking_active_fabric_port_check.py new file mode 100644 index 0000000..7622b75 --- /dev/null +++ b/tests/checks/port_tracking_active_fabric_port_check/test_port_tracking_active_fabric_port_check.py @@ -0,0 +1,140 @@ +import importlib +import logging +import os + +import pytest + +from helpers.utils import read_data + +script = importlib.import_module("aci-preupgrade-validation-script") +log = logging.getLogger(__name__) +dir = os.path.dirname(os.path.abspath(__file__)) +test_function = "port_tracking_active_fabric_port_check" +infra_port_track_pol_api = "uni/infra/trackEqptFabP-default.json" +doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#port-tracking-active-fabric-port-zero" + + +@pytest.mark.parametrize( + "icurl_outputs, tversion, vpc_node_ids, expected_result, expected_data", + [ + ( + {}, + None, + ["101", "102"], + script.MANUAL, + [], + ), + ( + {}, + "6.0(9c)", + ["101", "102"], + script.NA, + [], + ), + ( + {}, + "6.0(9e)", + ["101", "102"], + script.NA, + [], + ), + ( + {infra_port_track_pol_api: read_data(dir, "infraPortTrackPol_neg.json")}, + "6.0(9d)", + ["101", "102"], + script.PASS, + [], + ), + ( + {infra_port_track_pol_api: read_data(dir, "infraPortTrackPol_neg1.json")}, + "6.0(9d)", + ["101", "102"], + script.PASS, + [], + ), + ( + {infra_port_track_pol_api: read_data(dir, "infraPortTrackPol_pos.json")}, + "6.0(9d)", + ["101", "102"], + script.FAIL_O, + [["on", "0"]], + ), + ( + {infra_port_track_pol_api: read_data(dir, "infraPortTrackPol_pos.json")}, + "6.1(3f)", + ["101", "102"], + script.FAIL_O, + [["on", "0"]], + ), + ( + {}, + "6.1(3f)", + [], + script.NA, + [], + ), + ( + {infra_port_track_pol_api: []}, + "6.1(3f)", + ["101", "102"], + script.ERROR, + [], + ), + ( + { + infra_port_track_pol_api: [ + {"infraPortTrackPol": {"attributes": {"adminSt": "on"}}} + ] + }, + "6.1(3f)", + ["101", "102"], + script.ERROR, + [], + ), + ( + { + infra_port_track_pol_api: [ + {"infraPortTrackPol": {"attributes": {"adminSt": "on", "minlinks": "0"}}}, + {"infraPortTrackPol": {"attributes": {"adminSt": "off", "minlinks": "0"}}}, + ] + }, + "6.1(3f)", + ["101", "102"], + script.ERROR, + [], + ), + ], +) + + +def test_logic(run_check, mock_icurl, tversion, vpc_node_ids, expected_result, expected_data): + result = run_check( + tversion=script.AciVersion(tversion) if tversion else None, + vpc_node_ids=vpc_node_ids, + ) + assert result.result == expected_result + assert result.data == expected_data + assert result.doc_url == doc_url + + +def test_failure_result_contract(run_check, mock_icurl, icurl_outputs): + icurl_outputs[infra_port_track_pol_api] = read_data(dir, "infraPortTrackPol_pos.json") + + result = run_check( + tversion=script.AciVersion("6.1(3f)"), + vpc_node_ids=["101", "102"], + ) + + assert "fixed release" in result.recommended_action + assert "disable Port Tracking" in result.recommended_action + assert "more than two operational fabric uplinks" in result.recommended_action + assert "reload the affected switch" in result.recommended_action + + aci_result = script.AciResult(test_function, "Port Tracking Minimal Uplink Zero", result) + assert aci_result.ruleStatus == script.AciResult.FAIL + assert aci_result.severity == "critical" + assert aci_result.docUrl == doc_url + assert aci_result.failureDetails["header"] == ["Admin State", "Port Tracking Active Fabric Ports"] + assert aci_result.failureDetails["data"] == [ + {"Admin State": "on", "Port Tracking Active Fabric Ports": "0"} + ]