434-dynamically-calculate-the-required-free-space-in-switch-bootflash

aci-preupgrade-validation-script.py

@@ -2158,14 +2158,23 @@ def switch_group_guideline_check(fabric_nodes, **kwargs):
 
 
 @check_wrapper(check_title="Switch Node /bootflash usage")
-def switch_bootflash_usage_check(tversion, **kwargs):
+def switch_bootflash_usage_check(sw_cversion, tversion, **kwargs):
     result = FAIL_UF
     msg = ''
-    headers = ["Pod-ID", "Node-ID", "Utilization"]
+    headers = ["Pod-ID", "Node-ID", "Avail (MB)", "Required (MB)"]
     data = []
-    recommended_action = "Over 50% usage! Contact Cisco TAC for Support"
+    recommended_action = (
+        "Remove old, unused switch images to free bootflash space, then re-run this validation. "
+        "Contact Cisco TAC if sufficient space cannot be recovered."
+    )
     doc_url = "https://datacenter.github.io/ACI-Pre-Upgrade-Validation-Script/validations/#switch-node-bootflash-usage"
 
+    if not tversion:
+        return Result(result=MANUAL, msg=TVER_MISSING, doc_url=doc_url)
+
+    if not sw_cversion:
+        return Result(result=MANUAL, msg="Current switch version not found. Check switch health.", doc_url=doc_url)
+
     partitions_api = 'eqptcapacityFSPartition.json'
     partitions_api += '?query-target-filter=eq(eqptcapacityFSPartition.path,"/bootflash")'
 
@@ -2175,34 +2184,101 @@ def switch_bootflash_usage_check(tversion, **kwargs):
 
     partitions = icurl('class', partitions_api)
     if not partitions:
-        return Result(result=MANUAL, msg='bootflash objects not found. Check switch health.', doc_url=doc_url)
+        return Result(result=MANUAL, msg='/bootflash directory not found. Check switch health.', doc_url=doc_url)
 
     predownloaded_nodes = []
     try:
         download_sts = icurl('class', download_sts_api)
     except OldVerPropNotFound:
         # Older versions don't have 'dnldStatus' param
         download_sts = []
-
+   
     for maintUpgJob in download_sts:
         dn = re.search(node_regex, maintUpgJob['maintUpgJob']['attributes']['dn'])
-        node = dn.group("node")
-        predownloaded_nodes.append(node)
+        if dn:
+            predownloaded_nodes.append(dn.group("node"))
+
+    # Starting 6.0(2a), switch images are shipped as separate 32-bit and 64-bit
+    # isos (`-cs_64` suffix for 64-bit). Below that, only a single 32-bit iso exists.
+    boundary_version = "6.0(2a)"
+    switch_target_version = "aci-n9000-dk9.1{}.bin".format(tversion.dot_version)
+    switch_target_version_64 = "aci-n9000-dk9.1{}-cs_64.bin".format(tversion.dot_version)
+
+    firmware_api = 'firmwareFirmware.json?query-target-filter=eq(firmwareFirmware.type,"switch")'
+    firmwares = icurl('class', firmware_api)
+    fw_sizes = {}
+    for firmware in firmwares:
+        fw_attr = firmware['firmwareFirmware']['attributes']
+        fw_sizes[fw_attr['isoname']] = int(fw_attr['size'])
+
+    target_size_32 = fw_sizes.get(switch_target_version)
+    target_size_64 = fw_sizes.get(switch_target_version_64)
+
+    # sw_cversion (lowest switch version), not the APIC cversion, drives the image split
+    # decision: the upgrade guide has APICs reach 6.0(2a)+ before the switches, so the
+    # switches can still be pre-split while the APIC cluster is in the split-image era.
+    target_is_pre_split = tversion.older_than(boundary_version)
+    current_is_pre_split = sw_cversion.older_than(boundary_version)
+
+    if target_is_pre_split:
+        # Only the 32-bit image is ever used for a pre-6.0(2a) target.
+        if target_size_32 is None:
+            msg = 'Target switch image ({}) not found in Firmware Repository.'.format(switch_target_version)
+            return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+        required_space = 2 * target_size_32
+        downloaded_required_space = target_size_32
+
+    else:
+        # The larger image is used as a conservative estimate, so both sizes must be
+        # known; a missing one can't be assumed to be the smaller (or zero-byte) one.
+        if target_size_32 is None and target_size_64 is None:
+            msg = 'Target switch images ({}, {}) not found in Firmware Repository.'.format(switch_target_version, switch_target_version_64)
+            return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+        elif target_size_32 is None:
+            msg = '32-bit target switch image ({}) not found in Firmware Repository.'.format(switch_target_version)
+            return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+        elif target_size_64 is None:
+            msg = '64-bit target switch image ({}) not found in Firmware Repository.'.format(switch_target_version_64)
+            return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+
+        if current_is_pre_split:
+            # Crossing the 32/64-bit boundary: the pre-6.0(2a) switch only ever had a
+            # 32-bit image, so its size is freed once removed during the upgrade.
+            switch_current_version = "aci-n9000-dk9.1{}.bin".format(sw_cversion.dot_version)
+            current_size = fw_sizes.get(switch_current_version)
+            if current_size is None:
+                msg = 'Current switch image ({}) not found in Firmware Repository.'.format(switch_current_version)
+                return Result(result=MANUAL, msg=msg, doc_url=doc_url)
+            if target_size_32 > current_size:
+                required_space = 2 * (target_size_32 + target_size_64 - current_size)
+            else:
+                required_space = 2 * max(target_size_32, target_size_64)
+            # A downloaded 32-bit target is already reflected in the current `avail`.
+            downloaded_required_space = required_space - target_size_32
+        else:
+            required_space = 2 * max(target_size_32, target_size_64)
+            downloaded_required_space = max(target_size_32, target_size_64)
+
+    required_space_kb = required_space / 1024.0  # eqptcapacityFSPartition avail/used are in KB
+    downloaded_required_space_kb = downloaded_required_space / 1024.0
 
     for eqptcapacityFSPartition in partitions:
         dn = re.search(node_regex, eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['dn'])
         pod = dn.group("pod")
         node = dn.group("node")
         avail = int(eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['avail'])
-        used = int(eqptcapacityFSPartition['eqptcapacityFSPartition']['attributes']['used'])
 
-        usage = (used / (avail + used)) * 100
-        if (usage >= 50) and (node not in predownloaded_nodes):
-            data.append([pod, node, usage])
+        # dnldStatus == downloaded only proves the image was delivered, not that
+        # extraction (which still consumes bootflash) has completed, so a downloaded
+        # node is still checked, just against the smaller extraction-only requirement.
+        node_required_space_kb = downloaded_required_space_kb if node in predownloaded_nodes else required_space_kb
+
+        if avail < node_required_space_kb:
+            data.append([pod, node, round(avail / 1024.0, 2), round(node_required_space_kb / 1024.0, 2)])
 
     if not data:
         result = PASS
-        msg = 'All below 50% or pre-downloaded'
+        msg = 'All nodes have sufficient bootflash space'
     return Result(result=result, msg=msg, headers=headers, data=data, recommended_action=recommended_action, doc_url=doc_url)
 
 

docs/docs/index.md

@@ -105,7 +105,7 @@ You have chosen version "aci-apic-dk9.5.2.1d.bin"
 
 [Check 11/47] Switch Upgrade Group Guidelines... No upgrade groups found!                                            MANUAL CHECK REQUIRED
 [Check 12/47] APIC Disk Space Usage (F1527, F1528, F1529 equipment-full)...                                                           PASS
-[Check 13/47] Switch Node /bootflash usage... all below 50%                                                                           PASS
+[Check 13/47] Switch Node /bootflash usage... All nodes have sufficient bootflash space                                               PASS
 [Check 14/47] Standby APIC Disk Space Usage... No standby APIC found                                                                   N/A
 [Check 15/47] APIC SSD Health...                                                                                                      PASS
 [Check 16/47] Switch SSD Health (F3073, F3074 equipment-flash-warning)...                                                             PASS

docs/docs/validations.md

@@ -634,84 +634,26 @@ The script performs SSH into each standby Cisco APIC as `rescue-user`, then run
 
 ### Switch Node `/bootflash` usage
 
-ACI switches mainly have two different faults about the filesystem usage of each partition:
+ACI switches mainly have two different faults related to filesystem usage on each partition:
 
-* **F1820**: A minor level fault for switch partition usage. This is raised when the utilization of the partition exceeds the minor threshold.
+* **F1820**: A minor-level fault raised when partition utilization exceeds the minor threshold.
 
-* **F1821**: A major level fault for switch partition usage. This is raised when the utilization of the partition exceeds the major threshold.
+* **F1821**: A major-level fault raised when partition utilization exceeds the major threshold.
 
 !!! note
-    The threshold for minor and major depends on partitions. The critical one for upgrades is `/bootflash`. The threshold of bootflash is 80% for minor and 90% for major threshold.
+    Thresholds vary by partition. For `/bootflash`, the minor threshold is 80% utilization and the major threshold is 90%.
 
-On top of this, there is a built-in behavior added to every switch node where it will take action to ensure that the `/bootflash` directory maintains 50% capacity. This is specifically to ensure that switch upgrades are able to successfully transfer and extract the switch image over during an upgrade.
+ACI switches also include an internal cleanup process intended to maintain sufficient free `/bootflash` capacity for switch upgrades. When usage exceeds approximately 50%, the process can remove eligible files to make space for transferring and extracting switch images.
 
-To do this, there is an internal script that is monitoring `/bootflash` usage and, if over 50% usage, it will start removing files to free up the filesystem. Given its aggressiveness, there are some corner case scenarios where this cleanup script could potentially trigger against the switch image it is intending to use, which can result in a switch upgrade booting a switch into the loader prompt given that the boot image was removed from `/bootflash`.
+The fixed cleanup threshold does not cover every upgrade scenario. Larger target images, files that cannot be removed, and upgrades that cross the ACI 6.0(2) 32-bit/64-bit image boundary can require more free space than the cleanup process normally maintains. Insufficient space can prevent an image from being downloaded or extracted and may cause the switch upgrade to fail.
 
-To prevent this, check the `/bootflash` prior to an upgrade and take the necessary steps to understand what is written there and why. Once understood, take the necessary steps to clear up unnecessary `/bootflash` files to ensure there is enough space to prevent the auto-cleanup corner case scenario.
+The ACI Pre-Upgrade Validation script uses APIC API data to compare each switch's available `/bootflash` space with the space required for the target switch release. The requirement is calculated dynamically from the current and target switch images rather than from a fixed utilization percentage.
 
-The pre-upgrade validation built into Cisco APIC upgrade workflow monitors the fault F1821, which can capture the high utilization of any partition. When this fault is present, we recommend that you resolve it prior to the upgrade even if the fault is not for bootflash.
+The calculation generally reserves twice the applicable target image size. This accounts for space used by the downloaded image and additional space needed while the image is extracted. For an upgrade that crosses the 6.0(2) image boundary, the calculation accounts for both the 32-bit and 64-bit target images and the space recovered when the current image is removed.
 
-The ACI Pre-Upgrade Validation script (this script) focuses on the utilization of bootflash on each switch specifically to see if there are any issues with bootflash where the usage is more than 50%, which might trigger the internal cleanup script.
+Switches that have already downloaded the target image are still checked because image extraction and later upgrade stages can require additional space. Because the downloaded image is already reflected in the switch's available-space value, the check evaluates only the remaining space required to complete the upgrade.
 
-!!! example "Example of a query used by this script"
-    The script is calculating the bootflash usage using `avail` and `used` in the object `eqptcapacityFSPartition` for each switch.
-    ```
-    f2-apic1# moquery -c eqptcapacityFSPartition -f 'eqptcapacity.FSPartition.path=="/bootflash"'
-    Total Objects shown: 6
-
-    # eqptcapacity.FSPartition
-    name            : bootflash
-    avail           : 7214920
-    childAction     :
-    dn              : topology/pod-1/node-101/sys/eqptcapacity/fspartition-bootflash
-    memAlert        : normal
-    modTs           : never
-    monPolDn        : uni/fabric/monfab-default
-    path            : /bootflash
-    rn              : fspartition-bootflash
-    status          :
-    used            : 4320184
-    --- omit ---
-    ```
-
-!!! tip
-    Alternatively you can log into a leaf switch CLI, and check `/bootflash` usage `df -h`
-    ```
-    leaf1# df -h
-    Filesystem             Size    Used   Avail   Use%   Mounted on
-    rootfs                 2.5G    935M   1.6G    38%    /bin
-    /dev/sda4               12G    5.7G   4.9G    54%    /bootflash
-    /dev/sda2              4.7G    9.6M   4.4G     1%    /recovery
-    /dev/mapper/map-sda9    11G    5.7G   4.2G    58%    /isan/lib
-    none                   3.0G    602M   2.5G    20%    /dev/shm
-    none                    50M    3.4M    47M     7%    /etc
-    /dev/sda6               56M    1.3M    50M     3%    /mnt/cfg/1
-    /dev/sda5               56M    1.3M    50M     3%    /mnt/cfg/0
-    /dev/sda8               15G    140M    15G     1%    /mnt/ifc/log
-    /dev/sda3              115M     52M    54M    50%    /mnt/pss
-    none                   1.5G    2.3M   1.5G     1%    /tmp
-    none                    50M    240K    50M     1%    /var/log
-    /dev/sda7               12G    1.4G   9.3G    13%    /logflash
-    none                   350M     54M   297M    16%    /var/log/dme/log/dme_logs
-    none                   512M     24M   489M     5%    /var/sysmgr/mem_logs
-    none                    40M    4.0K    40M     1%    /var/sysmgr/startup-cfg
-    none                   500M     0     500M     0%    /volatile
-    ```
-
-!!! note
-    If you suspect that the auto cleanup removed some files within `/bootflash`, you can review a log to validate this:
-
-    ```
-    leaf1# egrep "higher|removed" /mnt/pss/core_control.log
-    [2020-07-22 16:52:08.928318] Bootflash Usage is higher than 50%!!
-    [2020-07-22 16:52:08.931990] File: MemoryLog.65%_usage removed !!
-    [2020-07-22 16:52:08.943914] File: mem_log.txt.old.gz removed !!
-    [2020-07-22 16:52:08.955376] File: libmon.logs removed !!
-    [2020-07-22 16:52:08.966686] File: urib_api_log.txt removed !!
-    [2020-07-22 16:52:08.977832] File: disk_log.txt removed !!
-    [2020-07-22 16:52:08.989102] File: mem_log.txt removed !!
-    [2020-07-22 16:52:09.414572] File: aci-n9000-dk9.13.2.1m.bin removed !!
-    ```
+If a switch does not have enough available space, the check reports an upgrade failure and displays the available and required space. Remove old, unused switch images to recover space, then run the validation again. Contact Cisco TAC if sufficient space cannot be recovered. If the current switch version, target firmware image, or `/bootflash` information is unavailable, the check reports that a manual review is required.
 
 
 ### APIC SSD Health

tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_602.json

@@ -0,0 +1,4 @@
+[
+  {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.0.2h.bin", "size": "2000000000"}}},
+  {"firmwareFirmware": {"attributes": {"isoname": "aci-n9000-dk9.16.0.2h-cs_64.bin", "size": "3000000000"}}}
+]

tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_insufficient.json

@@ -0,0 +1,35 @@
+[
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.16.1.5e.bin",
+        "fullVersion": "n9000-16.1(5e)",
+        "isoname": "aci-n9000-dk9.16.1.5e.bin",
+        "name": "aci-n9000-system.16.1.5e.bin",
+        "size": "3000000000"
+      }
+    }
+  },
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.16.1.5e-cs_64.bin",
+        "fullVersion": "n9000-16.1(5e)",
+        "isoname": "aci-n9000-dk9.16.1.5e-cs_64.bin",
+        "name": "aci-n9000-system.16.1.5e-cs_64.bin",
+        "size": "3000000000"
+      }
+    }
+  },
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.15.2.8h.bin",
+        "fullVersion": "n9000-15.2(8h)",
+        "isoname": "aci-n9000-dk9.15.2.8h.bin",
+        "name": "aci-n9000-system.15.2.8h.bin",
+        "size": "2000000000"
+      }
+    }
+  }
+]

tests/checks/switch_bootflash_usage_check/firmwareFirmware_dual_image_sufficient.json

@@ -0,0 +1,35 @@
+[
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.16.1.5e.bin",
+        "fullVersion": "n9000-16.1(5e)",
+        "isoname": "aci-n9000-dk9.16.1.5e.bin",
+        "name": "aci-n9000-system.16.1.5e.bin",
+        "size": "1500000000"
+      }
+    }
+  },
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.16.1.5e-cs_64.bin",
+        "fullVersion": "n9000-16.1(5e)",
+        "isoname": "aci-n9000-dk9.16.1.5e-cs_64.bin",
+        "name": "aci-n9000-system.16.1.5e-cs_64.bin",
+        "size": "1500000000"
+      }
+    }
+  },
+  {
+    "firmwareFirmware": {
+      "attributes": {
+        "dn": "fwrepo/fw-aci-n9000-system.15.2.8h.bin",
+        "fullVersion": "n9000-15.2(8h)",
+        "isoname": "aci-n9000-dk9.15.2.8h.bin",
+        "name": "aci-n9000-system.15.2.8h.bin",
+        "size": "2000000000"
+      }
+    }
+  }
+]

tests/checks/switch_bootflash_usage_check/maintUpgJob_all_downloaded.json

@@ -0,0 +1,13 @@
+[
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-102/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-103/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-205/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-206/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1002/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-1001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2002/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2003/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2001/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-2/node-2010/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}},
+  {"maintUpgJob": {"attributes": {"dn": "topology/pod-1/node-101/sys/maintupgjob", "dnldStatus": "downloaded", "dnldPercent": "100"}}}
+]