docs: move the Learn page onto the Astro site and generate its statistics

.github/scripts/update_learn_page.py

@@ -1,7 +1,7 @@
 #!/usr/bin/env python3
-"""Refresh the rule/playbook/severity statistics in the Learn page and README.
+"""Refresh repository-derived statistics in the deployed Learn page and README.
 
-docs/learn/index.html hardcodes rule, playbook, severity and category counts
+website/src/pages/learn.astro renders generated rule, playbook, severity and category counts
 in five places: the headline metric tiles, the hero terminal line, the
 pipeline step, the rules-section title/intro, the severity-box grid, and the
 "coverage by category" chart. README.md hardcodes the same rule and playbook
@@ -22,12 +22,12 @@
 import re
 import sys
 from pathlib import Path
-from typing import Dict, List, Tuple
+from typing import Dict, List, Optional, Tuple
 
 REPO_ROOT = Path(__file__).resolve().parents[2]
 RULES_DIR = REPO_ROOT / "scanner" / "rules"
 PLAYBOOKS_DIR = REPO_ROOT / "playbooks" / "cli"
-LEARN_PAGE = REPO_ROOT / "docs" / "learn" / "index.html"
+LEARN_PAGE = REPO_ROOT / "website" / "src" / "pages" / "learn.astro"
 README_PATH = REPO_ROOT / "README.md"
 
 # Rule modules are named az_<category>_<number>.py. Matching on that prefix is
@@ -152,6 +152,26 @@ def render_category_rows(categories: Dict[str, int]) -> str:
     return "\n".join(rows)
 
 
+def validate_statistics(rule_count: int, severities: Dict[str, int], categories: Dict[str, int]) -> Optional[str]:
+    """Return an error when generated Learn totals cannot support the headline.
+
+    The route displays CRITICAL/HIGH/MEDIUM/LOW rather than a partial severity
+    chart. Keep that display honest: every counted rule must occur exactly once
+    in both the displayed severity set and the category chart.
+    """
+    displayed_severities = ("CRITICAL", "HIGH", "MEDIUM", "LOW")
+    displayed_severity_total = sum(severities[severity] for severity in displayed_severities)
+    severity_total = sum(severities.values())
+    category_total = sum(categories.values())
+    if severity_total == rule_count and displayed_severity_total == rule_count and category_total == rule_count:
+        return None
+    return (
+        "generated Learn statistics do not reconcile with the rule total "
+        f"(rules: {rule_count}, all severities: {severity_total}, displayed "
+        f"CRITICAL/HIGH/MEDIUM/LOW: {displayed_severity_total}, categories: {category_total})."
+    )
+
+
 def _metric(label: str) -> str:
     """Build the pattern for one headline metric tile on the Learn page."""
     return rf'(<div class="metric"><strong>)\d+(</strong><span>{re.escape(label)}</span></div>)'
@@ -176,12 +196,13 @@ def render(
     content: str,
     rule_count: int,
     playbook_count: int,
+    critical_count: int,
     high_count: int,
     medium_count: int,
     low_count: int,
     category_rows: str,
 ) -> Tuple[str, List[str]]:
-    """Return (updated_content, failed_pattern_names) for docs/learn/index.html."""
+    """Return (updated_content, failed_pattern_names) for the Astro Learn route."""
     intro = (
         r'(<p class="section-intro">\s*OpenShield currently has )\d+'
         r"( dynamic rules\. The strongest contributor work improves rule "
@@ -193,6 +214,7 @@ def render(
     )
     section_title = r'(<h2 class="section-title">)\d+( Azure security rules</h2>)'
     hero_terminal = r'(<span class="dim">loading rules:</span> <span class="cyan">)\d+( dynamic checks</span></p>)'
+    severity_critical = r'(<div class="severity-box critical"><strong>)\d+(</strong><span>CRITICAL</span></div>)'
     severity_high = r'(<div class="severity-box high"><strong>)\d+(</strong><span>HIGH</span></div>)'
     severity_medium = r'(<div class="severity-box medium"><strong>)\d+(</strong><span>MEDIUM</span></div>)'
     severity_low = r'(<div class="severity-box low"><strong>)\d+(</strong><span>LOW</span></div>)'
@@ -207,14 +229,15 @@ def render(
         ("rules section title", section_title, rule_count),
         ("rules section intro paragraph", intro, rule_count),
         ("hero terminal: dynamic checks line", hero_terminal, rule_count),
+        ("severity box: CRITICAL", severity_critical, critical_count),
         ("severity box: HIGH", severity_high, high_count),
         ("severity box: MEDIUM", severity_medium, medium_count),
         ("severity box: LOW", severity_low, low_count),
     )
 
     content, failures = apply_replacements(content, replacements)
 
-    category_block = r'(<div class="rule-chart" aria-label="Rule count by category">\n)(.*?)(\n {10}</div>)'
+    category_block = r'(<div class="rule-chart" aria-label="Rule count by category">\n)(.*?)(\n\s*</div>)'
     content, count = re.subn(
         category_block,
         lambda m: m.group(1) + category_rows + m.group(3),
@@ -229,16 +252,14 @@ def render(
 
 def render_readme(content: str, rule_count: int, playbook_count: int) -> Tuple[str, List[str]]:
     """Return (updated_content, failed_pattern_names) for README.md."""
-    feature_row = (
-        r"(\| \*\*Misconfiguration Scanner\*\* \| Runs )\d+"
-        r"( Azure security rules across storage, network, identity, database, "
-        r"compute, Key Vault, AKS, post-quantum cryptography, backup, serverless, "
-        r"private endpoint, and supply chain posture \|)"
-    )
-    playbook_row = (
-        r"(\| \*\*Remediation Playbooks\*\* \| Every rule ships with a matching "
-        r"Azure CLI remediation script \()\d+( playbooks\) \|)"
-    )
+    # Anchored on the row label and the unit that follows the number, not on
+    # the full prose. The wording of these rows (the category list, how the
+    # scripts are described) is edited independently of the counts, and pinning
+    # the whole sentence made this script fail on every unrelated reword while
+    # the counts silently went stale. Losing the row itself, or the "Azure
+    # security rules"/"(N playbooks)" shape, still fails loudly.
+    feature_row = r"(\| \*\*Misconfiguration Scanner\*\* \| Runs )\d+( Azure security rules)"
+    playbook_row = r"(\| \*\*Remediation Playbooks\*\* \|[^|]*\()\d+( playbooks\) \|)"
     mermaid_scanner = r'(C\["Scanner Engine\\n)\d+( Python rules"\])'
     mermaid_playbooks = r'(G\["Azure CLI Playbooks\\n)\d+( remediation scripts"\])'
 
@@ -302,29 +323,21 @@ def main() -> int:
 
     if missing_severity:
         print(
-            f"Warning: {len(missing_severity)} rule file(s) have no parseable SEVERITY "
-            f"and are excluded from the severity counts: {', '.join(missing_severity)}",
+            f"Error: {len(missing_severity)} rule file(s) have no parseable SEVERITY: {', '.join(missing_severity)}",
             file=sys.stderr,
         )
+        return 1
     if missing_category:
         print(
-            f"Warning: {len(missing_category)} rule file(s) have no parseable CATEGORY "
-            f"and are excluded from the coverage-by-category chart: {', '.join(missing_category)}",
+            f"Error: {len(missing_category)} rule file(s) have no parseable CATEGORY: {', '.join(missing_category)}",
             file=sys.stderr,
         )
+        return 1
 
-    chart_severities = {"HIGH", "MEDIUM", "LOW"}
-    excluded_severities = {
-        severity: count for severity, count in severities.items() if severity not in chart_severities and count
-    }
-    if excluded_severities:
-        excluded_detail = ", ".join(f"{severity}: {count}" for severity, count in sorted(excluded_severities.items()))
-        excluded_total = sum(excluded_severities.values())
-        print(
-            f"Warning: {excluded_total} rule(s) with severities outside the "
-            f"HIGH/MEDIUM/LOW chart are excluded from the severity boxes: {excluded_detail}",
-            file=sys.stderr,
-        )
+    statistics_error = validate_statistics(rule_count, severities, categories)
+    if statistics_error:
+        print(f"Error: {statistics_error}", file=sys.stderr)
+        return 1
 
     category_rows = render_category_rows(categories)
 
@@ -333,6 +346,7 @@ def main() -> int:
         learn_original,
         rule_count,
         playbook_count,
+        severities["CRITICAL"],
         severities["HIGH"],
         severities["MEDIUM"],
         severities["LOW"],
@@ -342,7 +356,7 @@ def main() -> int:
     readme_original = README_PATH.read_text(encoding="utf-8")
     readme_updated, readme_failures = render_readme(readme_original, rule_count, playbook_count)
 
-    failures = [f"docs/learn/index.html -> {name}" for name in learn_failures]
+    failures = [f"website/src/pages/learn.astro -> {name}" for name in learn_failures]
     failures += [f"README.md -> {name}" for name in readme_failures]
 
     if failures:
@@ -369,7 +383,8 @@ def main() -> int:
 
     print(
         f"Updated {', '.join(changed)} - rules: {rule_count}, playbooks: {playbook_count}, "
-        f"severity HIGH: {severities['HIGH']}, MEDIUM: {severities['MEDIUM']}, LOW: {severities['LOW']}"
+        f"severity CRITICAL: {severities['CRITICAL']}, HIGH: {severities['HIGH']}, "
+        f"MEDIUM: {severities['MEDIUM']}, LOW: {severities['LOW']}"
     )
     return 0
 

.github/workflows/update-learn-page.yml

@@ -31,7 +31,7 @@ jobs:
       - name: Detect changes
         id: diff
         run: |
-          if git diff --quiet -- docs/learn/index.html README.md; then
+          if git diff --quiet -- website/src/pages/learn.astro README.md; then
             echo "changed=false" >> "$GITHUB_OUTPUT"
           else
             echo "changed=true" >> "$GITHUB_OUTPUT"
@@ -44,6 +44,6 @@ jobs:
         run: |
           git config user.name "github-actions[bot]"
           git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
-          git add docs/learn/index.html README.md
+          git add website/src/pages/learn.astro README.md
           git commit -s -m "docs: refresh learn page and README statistics [skip ci]"
           git push origin "HEAD:$TARGET_REF"

README.md

@@ -74,10 +74,10 @@ Findings map to NIST FIPS 203 (ML-KEM), FIPS 204 (ML-DSA), and FIPS 205 (SLH-DSA
 
 | Feature | Description |
 |---|---|
-| **Misconfiguration Scanner** | Runs 95 Azure security rules across storage, network, identity, database, compute, Key Vault, AKS, Kubernetes workloads, post-quantum cryptography, backup, serverless, private endpoint, and supply chain posture |
+| **Misconfiguration Scanner** | Runs 143 Azure security rules across storage, network, identity, database, compute, Key Vault, AKS, Kubernetes workloads, post-quantum cryptography, backup, serverless, private endpoint, and supply chain posture |
 | **Compliance Mapper** | Maps findings to CIS Benchmarks, NIST CSF, ISO 27001, and SOC 2 framework JSON files |
 | **Scan History API** | Stores scans and findings in PostgreSQL and exposes findings, score, scan history, compliance posture, drift, and resource inventory over REST |
-| **Remediation Playbooks** | Every documented rule ships with a matching review-gated remediation script (95 playbooks) |
+| **Remediation Playbooks** | Every documented rule ships with a matching review-gated remediation script (143 playbooks) |
 | **Security Dashboard** | Full React dashboard deployed on Vercel - live monitoring, findings, compliance, drift, prioritization, and AI-layer views |
 | **Project Website** | Documentation and reference site at [owasp.github.io/openshield](https://owasp.github.io/openshield/) - blog, rules gallery, architecture, evidence guides, roadmap, and releases |
 | **Sentinel Integration** | Normalises findings and pushes them into Microsoft Sentinel via a Log Analytics custom table and KQL analytics rules |
@@ -119,11 +119,11 @@ Project policies and assurance evidence:
 flowchart TD
     A["React Dashboard\nVercel · Live"]
     B["Flask REST API\nJWT · CORS · Blueprints"]
-    C["Scanner Engine\n95 Python rules"]
+    C["Scanner Engine\n143 Python rules"]
     D["Azure Subscription\nScanned via Azure SDK + Graph"]
     E["Compliance Framework JSON\nCIS · NIST · ISO 27001 · SOC 2"]
     F["PostgreSQL Database\nFindings · Scans"]
-    G["Azure CLI Playbooks\n95 remediation scripts"]
+    G["Azure CLI Playbooks\n143 remediation scripts"]
     H["sentinel/ingest.py\nNormalise + HMAC upload"]
     I["Microsoft Sentinel\nOpenShieldFindings_CL · KQL rules"]
 
@@ -330,7 +330,7 @@ Learn OpenShield covers:
 - Contributor onboarding
 - Documentation navigation
 
-Live Learning Portal: https://openshieldlearn.netlify.app/learn/
+Live Learning Portal: https://owasp.github.io/openshield/learn/
 Full documentation, the security rules gallery, architecture guide, evidence guide, and blog are available at the project website:
 
 **[owasp.github.io/openshield](https://owasp.github.io/openshield/)**