NewValidation for CSCwp91797

aci-preupgrade-validation-script.py

@@ -6153,7 +6153,7 @@ def isis_database_byte_check(tversion, **kwargs):
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 
-# Subprocess check - cat + acidiag
+# Subprocess check - ls + cat + acidiag
 @check_wrapper(check_title='APIC Database Size')
 def apic_database_size_check(cversion, **kwargs):
     result = PASS
@@ -6181,6 +6181,16 @@ def apic_database_size_check(cversion, **kwargs):
         apic_id_to_name = {"2": apic_id_to_name["2"]}
 
     if cversion.older_than("6.1(3a)"):
+        # Populate the dynamic /debug namespace before reading legacy counter files.
+        try:
+            run_cmd('/bin/ls /debug >/dev/null 2>&1', splitlines=False)
+        except subprocess.CalledProcessError as error:
+            # Keep the preflight best-effort; per-file reads remain authoritative.
+            log.warning(
+                'Unable to initialize /debug before APIC database collection: %s',
+                error,
+            )
+
         for dme in dme_svc_list:
             for id in apic_id_to_name:
                 apic_hostname = apic_id_to_name[id]
@@ -6342,6 +6352,55 @@ def configpush_shard_check(tversion, **kwargs):
     return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 
+@check_wrapper(check_title='Port Tracking Minimal Uplink Zero')
+def port_tracking_active_fabric_port_check(tversion, vpc_node_ids, **kwargs):
+    headers = ["Admin State", "Port Tracking Active Fabric Ports"]
+    data = []
+    recommended_action = (
+        'Upgrade to a fixed release when possible. If upgrading to an affected release, either disable Port Tracking '
+        'before upgrading the leaf, or set Port Tracking Active Fabric Ports (minLink) to 1 only after verifying every '
+        'affected leaf has more than two operational fabric uplinks. If the issue has already occurred, disable Port '
+        'Tracking, reload the affected switch, and then re-enable Port Tracking.'
+    )
+    doc_url = 'https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#port-tracking-active-fabric-port-zero'
+
+    if not tversion:
+        return Result(result=MANUAL, msg=TVER_MISSING, doc_url=doc_url)
+
+    affected_versions = ("6.0(9d)", "6.1(3f)")
+    if not any(tversion.same_as(version) for version in affected_versions):
+        return Result(result=NA, msg=VER_NOT_AFFECTED, doc_url=doc_url)
+
+    if not vpc_node_ids:
+        return Result(result=NA, msg="No vPC nodes found. Not susceptible.", doc_url=doc_url)
+
+    port_tracking_api = 'uni/infra/trackEqptFabP-default.json'
+    port_tracking_mo = icurl('mo', port_tracking_api)
+    if not isinstance(port_tracking_mo, list) or len(port_tracking_mo) != 1:
+        object_count = len(port_tracking_mo) if isinstance(port_tracking_mo, list) else 0
+        msg = "Expected exactly one infraPortTrackPol object, but found {}.".format(object_count)
+        return Result(result=ERROR, msg=msg, recommended_action=recommended_action, doc_url=doc_url)
+
+    try:
+        attributes = port_tracking_mo[0]['infraPortTrackPol']['attributes']
+        admin_st = attributes['adminSt']
+        minimal_uplink = attributes['minlinks']
+    except (KeyError, TypeError):
+        msg = "The infraPortTrackPol response is missing required adminSt or minlinks attributes."
+        return Result(result=ERROR, msg=msg, recommended_action=recommended_action, doc_url=doc_url)
+
+    if admin_st not in ("on", "off") or not str(minimal_uplink).isdigit():
+        msg = "The infraPortTrackPol response contains an invalid adminSt or minlinks value."
+        return Result(result=ERROR, msg=msg, recommended_action=recommended_action, doc_url=doc_url)
+
+    result = PASS
+    if admin_st == "on" and str(minimal_uplink) == "0":
+        data.append([admin_st, minimal_uplink])
+        result = FAIL_O
+
+    return Result(result=result, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
+
+
 @check_wrapper(check_title='APIC VMM inventory sync fault (F0132)')
 def apic_vmm_inventory_sync_faults_check(**kwargs):
     result = PASS
@@ -7158,6 +7217,7 @@ class CheckManager:
         standby_sup_sync_check,
         isis_database_byte_check,
         configpush_shard_check,
+        port_tracking_active_fabric_port_check,
         auto_firmware_update_on_switch_check,
         rogue_ep_coop_exception_mac_check,
         n9k_c9408_model_lem_count_check,

docs/docs/validations.md

@@ -199,6 +199,7 @@ Items                                           | Defect       | This Script
 [Stale pconsRA Object][d26]                     | CSCwp22212   | :warning:{title="Deprecated"} | :no_entry_sign:
 [ISIS DTEPs Byte Size][d27]                     | CSCwp15375   | :white_check_mark: | :no_entry_sign:
 [Policydist configpushShardCont Crash][d28]     | CSCwp95515   | :white_check_mark: | :no_entry_sign:
+[Port Tracking Active Fabric Port Zero][d39]    | CSCwp91797   | :white_check_mark: |
 [Auto Firmware Update on Switch Discovery][d29] | CSCwe83941   | :white_check_mark: | :no_entry_sign:
 [Rogue EP Exception List missing on switches][d30] | CSCwp64296   | :white_check_mark: | :no_entry_sign:
 [N9K-C9408 with more than 5 N9K-X9400-16W LEMs][d31] | CSCws82819   | :white_check_mark: | :no_entry_sign:
@@ -248,6 +249,7 @@ Items                                           | Defect       | This Script
 [d36]: #n9k-c93180yc-fx3-switch-memory-less-than-32gb
 [d37]: #stale-dbgacepgsummarytask-objects
 [d38]: #infravlan-overlap-access-policy-check
+[d39]: #port-tracking-active-fabric-port-zero
 
 ## General Check Details
 
@@ -2753,6 +2755,14 @@ Due to [CSCwp95515][59], upgrading to an affected version while having any `conf
 
 If any instances of `configpushShardCont` are flagged by this script, Cisco TAC must be contacted to identify and resolve the underlying issue before performing the upgrade.
 
+### Port Tracking Active Fabric Port Zero
+
+Due to [CSCwp91797][78], if port tracking is enabled and the number of active fabric ports that triggers port tracking (`minLink`) is zero, vPC port-channel member ports may remain down after a switch reload, upgrade, or boot. The affected physical links remain in the `initializing` state and MTS buffers may remain stuck on the leaf.
+
+The confirmed affected target releases checked by this validation are 6.0(9d) and 6.1(3f). Only fabrics containing vPC nodes are susceptible.
+
+Upgrade to a fixed release when possible. If an affected release must be used, either disable Port Tracking before upgrading each leaf, or change `minLink` from 0 to 1 only after verifying that every affected leaf has more than two operational fabric uplinks. If the issue has already occurred, disable Port Tracking, reload the affected switch, and then re-enable Port Tracking.
+
 
 ### Auto Firmware Update on Switch Discovery
 
@@ -2947,3 +2957,4 @@ To avoid this issue, modify the user VLAN pool ranges so that the InfraVLAN does
 [75]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt69100
 [76]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt38698
 [77]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwt58626
+[78]: https://bst.cloudapps.cisco.com/bugsearch/bug/CSCwp91797

tests/checks/apic_database_size_check/test_apic_database_size_check.py

@@ -13,6 +13,7 @@
 test_function = "apic_database_size_check"
 
 apic_node_api = 'infraWiNode.json'
+debug_init_cmd = "/bin/ls /debug >/dev/null 2>&1"
 
 apic1_pm_cat = "cat /debug/apic1/policymgr/mitmocounters/mo 2>&1"
 apic1_pd_cat = "cat /debug/apic1/policydist/mitmocounters/mo 2>&1"
@@ -177,6 +178,7 @@
         (
             {apic_node_api: read_data(dir, 'infraWiNode_3.json')},
             {
+                debug_init_cmd: {"splitlines": False, "output": ""},
                 apic2_pm_cat: {"splitlines": True, "output": mitcounters_policymgr_pos},
                 apic2_pd_cat: {"splitlines": True, "output": mitcounters_policydist_pos},
                 apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos},
@@ -189,6 +191,7 @@
         (
             {apic_node_api: read_data(dir, 'infraWiNode_3.json')},
             {
+                debug_init_cmd: {"splitlines": False, "output": ""},
                 apic2_pm_cat: {"splitlines": True, "output": mitcounters_neg},
                 apic2_pd_cat: {"splitlines": True, "output": mitcounters_neg},
                 apic2_vmm_cat: {"splitlines": True, "output": mitcounters_neg},
@@ -225,6 +228,7 @@
         (
             {apic_node_api: read_data(dir, 'infraWiNode_4.json')},
             {
+                debug_init_cmd: {"splitlines": False, "output": ""},
                 apic1_pm_cat: {"splitlines": True, "output": mitcounters_policymgr_pos},
                 apic1_pd_cat: {"splitlines": True, "output": mitcounters_policydist_pos},
                 apic1_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos},
@@ -249,6 +253,7 @@
         (
             {apic_node_api: read_data(dir, 'infraWiNode_4.json')},
             {
+                debug_init_cmd: {"splitlines": False, "output": ""},
                 apic1_pm_cat: {"splitlines": True, "output": mitcounters_neg},
                 apic1_pd_cat: {"splitlines": True, "output": mitcounters_neg},
                 apic1_vmm_cat: {"splitlines": True, "output": mitcounters_neg},
@@ -370,6 +375,7 @@ def test_missing_mitmocounters_returns_error(
     })
     cmd_outputs.clear()
     cmd_outputs.update({
+        debug_init_cmd: {"splitlines": False, "output": ""},
         apic2_pm_cat: failure_details,
         apic2_pd_cat: failure_details,
         apic2_vmm_cat: failure_details,
@@ -395,6 +401,7 @@ def test_collection_error_preserves_oversized_classes(
     })
     cmd_outputs.clear()
     cmd_outputs.update({
+        debug_init_cmd: {"splitlines": False, "output": ""},
         apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos},
         apic2_pm_cat: {
             "CalledProcessError": True,
@@ -427,6 +434,72 @@ def test_collection_error_preserves_oversized_classes(
     assert "high object counts" in result.recommended_action
 
 
+def test_debug_namespace_initialized_before_counter_reads(
+    run_check, mock_icurl, icurl_outputs, monkeypatch
+):
+    icurl_outputs.clear()
+    icurl_outputs.update({
+        apic_node_api: read_data(dir, 'infraWiNode_3.json'),
+    })
+    successful_outputs = {
+        apic2_vmm_cat: mitcounters_neg,
+        apic2_pm_cat: mitcounters_neg,
+        apic2_evm_cat: mitcounters_neg,
+        apic2_pd_cat: mitcounters_neg,
+    }
+    namespace_initialized = [False]
+    command_calls = []
+
+    def stateful_run_cmd(cmd, splitlines=False):
+        command_calls.append(cmd)
+        if cmd == debug_init_cmd:
+            namespace_initialized[0] = True
+            return ""
+        if not namespace_initialized[0]:
+            raise CalledProcessError(
+                1,
+                cmd,
+                output=b"cat: file: No such file or directory\n",
+            )
+        output = successful_outputs[cmd]
+        return output.splitlines() if splitlines else output
+
+    monkeypatch.setattr(script, "run_cmd", stateful_run_cmd)
+
+    result = run_check(cversion=script.AciVersion("5.2(2e)"))
+
+    assert result.result == script.PASS
+    assert command_calls[0] == debug_init_cmd
+    assert command_calls.count(debug_init_cmd) == 1
+
+
+def test_debug_namespace_initialization_failure_is_best_effort(
+    run_check, mock_icurl, icurl_outputs, monkeypatch
+):
+    icurl_outputs.clear()
+    icurl_outputs.update({
+        apic_node_api: read_data(dir, 'infraWiNode_3.json'),
+    })
+    successful_outputs = {
+        apic2_vmm_cat: mitcounters_neg,
+        apic2_pm_cat: mitcounters_neg,
+        apic2_evm_cat: mitcounters_neg,
+        apic2_pd_cat: mitcounters_neg,
+    }
+
+    def init_failure_run_cmd(cmd, splitlines=False):
+        if cmd == debug_init_cmd:
+            raise CalledProcessError(1, cmd, output=b"ls: cannot open /debug\n")
+        output = successful_outputs[cmd]
+        return output.splitlines() if splitlines else output
+
+    monkeypatch.setattr(script, "run_cmd", init_failure_run_cmd)
+
+    result = run_check(cversion=script.AciVersion("5.2(2e)"))
+
+    assert result.result == script.PASS
+
+
 def test_transient_counter_read_succeeds_on_retry(
     run_check, mock_icurl, mock_run_cmd, icurl_outputs, cmd_outputs, monkeypatch
 ):
@@ -445,6 +518,8 @@ def test_transient_counter_read_succeeds_on_retry(
 
     def transient_run_cmd(cmd, splitlines=False):
         call_counts[cmd] = call_counts.get(cmd, 0) + 1
+        if cmd == debug_init_cmd:
+            return ""
         if call_counts[cmd] == 1:
             raise CalledProcessError(
                 1,
@@ -460,7 +535,8 @@ def transient_run_cmd(cmd, splitlines=False):
     result = run_check(cversion=script.AciVersion("6.0(8f)"))
 
     assert result.result == script.PASS
-    assert all(call_count == 2 for call_count in call_counts.values())
+    assert call_counts[debug_init_cmd] == 1
+    assert all(call_counts[cmd] == 2 for cmd in successful_outputs)
     assert sleep_calls.count(1) == 4
 
 
@@ -482,6 +558,8 @@ def test_empty_counter_read_succeeds_on_retry(
 
     def transient_run_cmd(cmd, splitlines=False):
         call_counts[cmd] = call_counts.get(cmd, 0) + 1
+        if cmd == debug_init_cmd:
+            return ""
         if call_counts[cmd] == 1:
             return []
         output = successful_outputs[cmd]
@@ -493,7 +571,8 @@ def transient_run_cmd(cmd, splitlines=False):
     result = run_check(cversion=script.AciVersion("6.0(8f)"))
 
     assert result.result == script.PASS
-    assert all(call_count == 2 for call_count in call_counts.values())
+    assert call_counts[debug_init_cmd] == 1
+    assert all(call_counts[cmd] == 2 for cmd in successful_outputs)
     assert sleep_calls.count(1) == 4
 
 
@@ -513,6 +592,7 @@ def test_object_counters_are_sorted_before_top_four_and_thresholded(
     })
     cmd_outputs.clear()
     cmd_outputs.update({
+        debug_init_cmd: {"splitlines": False, "output": ""},
         apic2_vmm_cat: {"splitlines": True, "output": unsorted_counters},
         apic2_pm_cat: {"splitlines": True, "output": mitcounters_neg},
         apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg},
@@ -538,6 +618,7 @@ def test_malformed_counter_preserves_oversized_classes(
     })
     cmd_outputs.clear()
     cmd_outputs.update({
+        debug_init_cmd: {"splitlines": False, "output": ""},
         apic2_vmm_cat: {"splitlines": True, "output": mitcounters_vmmmgr_pos},
         apic2_pm_cat: {"splitlines": True, "output": "brokenClass :"},
         apic2_evm_cat: {"splitlines": True, "output": mitcounters_neg},
@@ -564,6 +645,7 @@ def test_colonless_counter_data_returns_error(
     })
     cmd_outputs.clear()
     cmd_outputs.update({
+        debug_init_cmd: {"splitlines": False, "output": ""},
         apic2_vmm_cat: {
             "splitlines": True,
             "output": "validClass : 10\ntruncatedClass",

tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_neg.json

@@ -0,0 +1,30 @@
+{
+  "totalCount": "1",
+  "imdata": [
+    {
+      "infraPortTrackPol": {
+        "attributes": {
+          "adminSt": "off",
+          "annotation": "",
+          "childAction": "",
+          "delay": "120",
+          "descr": "",
+          "dn": "uni/infra/trackEqptFabP-default",
+          "extMngdBy": "",
+          "includeApicPorts": "no",
+          "lcOwn": "local",
+          "minlinks": "0",
+          "modTs": "2025-10-31T17:13:46.220+11:00",
+          "name": "default",
+          "nameAlias": "",
+          "ownerKey": "",
+          "ownerTag": "",
+          "rn": "trackEqptFabP-default",
+          "status": "",
+          "uid": "0",
+          "userdom": "all"
+        }
+      }
+    }
+  ]
+}

tests/checks/port_tracking_active_fabric_port_check/infraPortTrackPol_neg1.json

@@ -0,0 +1,30 @@
+{
+  "totalCount": "1",
+  "imdata": [
+    {
+      "infraPortTrackPol": {
+        "attributes": {
+          "adminSt": "on",
+          "annotation": "",
+          "childAction": "",
+          "delay": "120",
+          "descr": "",
+          "dn": "uni/infra/trackEqptFabP-default",
+          "extMngdBy": "",
+          "includeApicPorts": "no",
+          "lcOwn": "local",
+          "minlinks": "1",
+          "modTs": "2025-10-31T17:13:46.220+11:00",
+          "name": "default",
+          "nameAlias": "",
+          "ownerKey": "",
+          "ownerTag": "",
+          "rn": "trackEqptFabP-default",
+          "status": "",
+          "uid": "0",
+          "userdom": "all"
+        }
+      }
+    }
+  ]
+}