#!/usr/bin/env python3
"""
generate_sitemap.py — RFC-compliant XML sitemap generator and validator.

Site convention:
  - Every page is <dir>/index.html -> URL = domain/<dir>/
  - Root index.html -> domain/
  - Loose .html files such as 404.html, sitemap.html, etc. are skipped.
  - Excluded directories are not scanned.

Generated XML structure:

<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
  <url>
    <loc>https://example.com/</loc>
    <lastmod>2026-09-05</lastmod>
    <changefreq>always</changefreq>
    <priority>1.0</priority>
  </url>
</urlset>

URL requirements:
  - RFC 3986 URI syntax
  - RFC 3987-compatible Unicode IRI handling
  - XML-safe escaping
  - HTTPS/HTTP scheme only
  - Valid hostname
  - No fragments (#)
  - No whitespace
  - HTTPS
  - non-www hostname
  - root URL ends with /
  - every non-root URL has NO trailing slash
  - no query string
  - no fragment

Usage:

  Dry run:
    python3 generate_sitemap.py \
      --domain https://schooloffreelancing.com

  Write + validate:
    python3 generate_sitemap.py \
      --domain https://schooloffreelancing.com \
      --apply

  Custom root:
    python3 generate_sitemap.py \
      --root /var/www/html \
      --domain https://schooloffreelancing.com \
      --apply

  Additional exclusions:
    python3 generate_sitemap.py \
      --domain https://schooloffreelancing.com \
      --exclude private \
      --exclude private/* \
      --apply
"""

import argparse
import datetime
import fnmatch
import ipaddress
import os
import re
import shutil
import sys
import unicodedata
import urllib.parse
import xml.etree.ElementTree as ET


# ---------------------------------------------------------------------------
# Sitemap XML namespace
# ---------------------------------------------------------------------------

NS = "http://www.sitemaps.org/schemas/sitemap/0.9"
ET.register_namespace("", NS)

INDEX_FILE = "index.html"


# ---------------------------------------------------------------------------
# Sitemap defaults
# ---------------------------------------------------------------------------

DEFAULT_CHANGEFREQ = "always"

# Priority is assigned according to URL depth:
#
#   /                         -> 1.0
#   /page                     -> 0.8
#   /category/page            -> 0.6
#   /a/b/c/page               -> 0.4
#
# This is configurable with --priority if required.
DEFAULT_ROOT_PRIORITY = 1.0
DEFAULT_PAGE_PRIORITY = 0.8
DEFAULT_DEEP_PRIORITY = 0.6


# ---------------------------------------------------------------------------
# Directories that should not become sitemap URLs
# ---------------------------------------------------------------------------

DEFAULT_EXCLUDES = [
    "assets",
    "assets/*",

    "api",
    "api/*",

    "mcp",
    "mcp/*",

    "gsc-repair-reports",
    "gsc-repair-reports/*",

    ".git",
    ".git/*",

    ".well-known",
    ".well-known/*",

    "node_modules",
    "node_modules/*",
]


# ---------------------------------------------------------------------------
# XML sitemap allowed changefreq values
# ---------------------------------------------------------------------------

VALID_CHANGEFREQ = {
    "always",
    "hourly",
    "daily",
    "weekly",
    "monthly",
    "yearly",
    "never",
}


# ---------------------------------------------------------------------------
# Command-line arguments
# ---------------------------------------------------------------------------

def parse_args():
    parser = argparse.ArgumentParser(
        description=__doc__,
        formatter_class=argparse.RawDescriptionHelpFormatter,
    )

    parser.add_argument(
        "--root",
        default="/var/www/html",
        help="Web root to scan (default: /var/www/html)",
    )

    parser.add_argument(
        "--out",
        default=None,
        help="Output sitemap path (default: <root>/sitemap.xml)",
    )

    parser.add_argument(
        "--domain",
        required=True,
        help="Base domain, e.g. https://example.com",
    )

    parser.add_argument(
        "--exclude",
        action="append",
        default=[],
        help="Additional glob pattern to exclude. Can be repeated.",
    )

    parser.add_argument(
        "--changefreq",
        default=DEFAULT_CHANGEFREQ,
        choices=sorted(VALID_CHANGEFREQ),
        help="Sitemap changefreq value (default: always)",
    )

    parser.add_argument(
        "--root-priority",
        type=float,
        default=DEFAULT_ROOT_PRIORITY,
        help="Priority for root URL (default: 1.0)",
    )

    parser.add_argument(
        "--page-priority",
        type=float,
        default=DEFAULT_PAGE_PRIORITY,
        help="Priority for normal pages (default: 0.8)",
    )

    parser.add_argument(
        "--deep-priority",
        type=float,
        default=DEFAULT_DEEP_PRIORITY,
        help="Priority for deeply nested pages (default: 0.6)",
    )

    parser.add_argument(
        "--apply",
        action="store_true",
        help="Write sitemap.xml. Without this option the script performs a dry run.",
    )

    args = parser.parse_args()

    args.domain = normalize_domain(args.domain)

    args.out = args.out or os.path.join(args.root, "sitemap.xml")

    args.excludes = DEFAULT_EXCLUDES + args.exclude

    validate_priority(args.root_priority, "root-priority")
    validate_priority(args.page_priority, "page-priority")
    validate_priority(args.deep_priority, "deep-priority")

    return args


# ---------------------------------------------------------------------------
# Priority validation
# ---------------------------------------------------------------------------

def validate_priority(value, name):
    if not 0.0 <= value <= 1.0:
        raise SystemExit(
            f"ERROR: --{name} must be between 0.0 and 1.0."
        )


# ---------------------------------------------------------------------------
# Domain normalization
# ---------------------------------------------------------------------------

def normalize_domain(domain):
    """
    Normalize and validate the sitemap base domain.

    Requirements:
      - HTTP or HTTPS only
      - No username/password
      - No query
      - No fragment
      - Valid hostname
      - No whitespace
      - Non-www hostname (bare apex/subdomain, no leading "www.")
    """

    domain = domain.strip()

    if not domain:
        raise SystemExit("ERROR: --domain cannot be empty.")

    if any(ch.isspace() for ch in domain):
        raise SystemExit("ERROR: domain contains whitespace.")

    parsed = urllib.parse.urlsplit(domain)

    if parsed.scheme.lower() not in ("http", "https"):
        raise SystemExit(
            "ERROR: --domain must use http:// or https://."
        )

    if not parsed.netloc:
        raise SystemExit("ERROR: domain is missing a hostname.")

    if parsed.username or parsed.password:
        raise SystemExit(
            "ERROR: domain must not contain username/password credentials."
        )

    if parsed.query:
        raise SystemExit(
            "ERROR: domain must not contain a query string."
        )

    if parsed.fragment:
        raise SystemExit(
            "ERROR: domain must not contain a fragment."
        )

    hostname = parsed.hostname

    if not hostname:
        raise SystemExit("ERROR: domain has no valid hostname.")

    validate_hostname(hostname)

    if hostname.lower().startswith("www."):
        raise SystemExit(
            "ERROR: --domain must be the non-www hostname "
            f"(got {hostname!r}). Use the bare domain; the live "
            "site redirects www -> non-www, and sitemap URLs must "
            "point at the final canonical host to avoid redirect "
            "chains."
        )

    # Normalize scheme and hostname.
    scheme = parsed.scheme.lower()
    hostname = hostname.lower()

    # Preserve explicit port if present.
    netloc = hostname

    if parsed.port is not None:
        default_port = (
            443 if scheme == "https"
            else 80
        )

        if parsed.port != default_port:
            netloc = f"{hostname}:{parsed.port}"

    return f"{scheme}://{netloc}"


# ---------------------------------------------------------------------------
# Hostname validation
# ---------------------------------------------------------------------------

def validate_hostname(hostname):
    """
    Validate normal DNS hostname or IP address.

    Supports:
      - DNS hostnames
      - IPv4
      - IPv6
    """

    if any(ch.isspace() for ch in hostname):
        raise ValueError("hostname contains whitespace")

    # IPv4 / IPv6
    try:
        ipaddress.ip_address(hostname)
        return
    except ValueError:
        pass

    # Internationalized domain names are allowed by RFC 3987.
    # Validate each label through IDNA.
    try:
        ascii_hostname = hostname.encode("idna").decode("ascii")
    except UnicodeError as exc:
        raise ValueError(
            f"invalid internationalized hostname: {hostname}"
        ) from exc

    if len(ascii_hostname) > 253:
        raise ValueError("hostname exceeds 253 characters")

    labels = ascii_hostname.split(".")

    if any(not label for label in labels):
        raise ValueError("hostname contains an empty label")

    label_pattern = re.compile(
        r"^(?!-)[A-Za-z0-9-]{1,63}(?<!-)$"
    )

    for label in labels:
        if not label_pattern.fullmatch(label):
            raise ValueError(
                f"invalid hostname label: {label}"
            )


# ---------------------------------------------------------------------------
# Exclusion handling
# ---------------------------------------------------------------------------

def is_excluded(reldir, excludes):
    if reldir == "":
        return False

    candidates = {
        reldir,
        reldir + "/",
    }

    return any(
        fnmatch.fnmatch(candidate, pattern)
        for candidate in candidates
        for pattern in excludes
    )


# ---------------------------------------------------------------------------
# URL path normalization
# ---------------------------------------------------------------------------

def normalize_url_path(path):
    """
    Convert a filesystem-relative directory into a valid RFC 3986/3987 URL path.

    Convention (matches the live site's actual redirect behaviour):
      - root ("")        -> "/"
      - any other dir     -> "/segment/segment"  (NO trailing slash)

    The live server 301-redirects "/dir/" -> "/dir" (no trailing slash).
    Emitting trailing-slash URLs in the sitemap therefore sends Googlebot
    through an extra redirect hop on every single page, which is exactly
    what produced the "Not found (404)" / stuck-pending validation batch
    in Search Console. Non-root URLs must be written WITHOUT a trailing
    slash so the sitemap points straight at the final 200 response.

    This function:
      - normalizes Unicode
      - converts Windows separators
      - removes unsafe whitespace
      - percent-encodes unsafe characters
      - preserves existing percent-encoded sequences
    """

    path = path.replace(os.sep, "/")

    if path:
        path = unicodedata.normalize("NFC", path)

    # Never permit a filesystem path to escape the URL root.
    path = path.lstrip("/")

    # Encode each path segment separately.
    segments = []

    for segment in path.split("/"):
        if not segment:
            continue

        # RFC 3986 unreserved characters plus selected sub-delimiters.
        encoded = urllib.parse.quote(
            segment,
            safe="-._~!$&'()*+,;=:@%"
        )

        segments.append(encoded)

    if not segments:
        # Root page.
        return "/"

    # Non-root page: NO trailing slash.
    return "/" + "/".join(segments)


# ---------------------------------------------------------------------------
# URL construction
# ---------------------------------------------------------------------------

def build_page_url(domain, rel_dir):
    """
    Convert a relative directory into the canonical URL:
    trailing slash for root only, no trailing slash otherwise.
    """

    path = normalize_url_path(rel_dir)

    return domain + path


# ---------------------------------------------------------------------------
# RFC 3986 / RFC 3987 URL validation
# ---------------------------------------------------------------------------

def validate_url(url, expected_domain=None):
    """
    Validate sitemap URL.

    Checks:
      - RFC 3986 URI structure
      - RFC 3987-compatible Unicode IRI handling
      - HTTP/HTTPS only
      - valid host
      - no fragment
      - no query
      - no whitespace
      - absolute URL
      - root URL ends with "/"; every non-root URL has NO trailing slash
      - no control characters
      - valid percent encoding
      - non-www hostname
      - optional domain restriction
    """

    errors = []

    if not isinstance(url, str) or not url:
        return ["URL is empty or not a string."]

    # XML 1.0 disallows control characters such as NUL.
    for char in url:
        code = ord(char)

        if (
            code == 0x00
            or 0x01 <= code <= 0x08
            or 0x0B <= code <= 0x0C
            or 0x0E <= code <= 0x1F
            or code == 0x7F
        ):
            errors.append(
                f"URL contains illegal XML/control character U+{code:04X}"
            )
            break

    if any(char.isspace() for char in url):
        errors.append("URL contains whitespace.")

    parsed = urllib.parse.urlsplit(url)

    if not parsed.scheme:
        errors.append("URL has no scheme.")

    if parsed.scheme.lower() not in ("http", "https"):
        errors.append(
            f"URL scheme must be http or https, got {parsed.scheme!r}."
        )

    if not parsed.netloc:
        errors.append("URL has no hostname.")

    if parsed.username or parsed.password:
        errors.append("URL contains credentials.")

    if parsed.fragment:
        errors.append("URL contains a fragment (#...).")

    if parsed.query:
        errors.append(
            "URL contains a query string; sitemap URLs should be canonical."
        )

    # Trailing-slash convention: root "/" must keep it, every other
    # path must NOT have one. This matches the live server's actual
    # redirect target and avoids extra 301 hops for Googlebot.
    if not parsed.path:
        errors.append("URL has no path.")
    elif parsed.path == "/":
        pass
    elif parsed.path.endswith("/"):
        errors.append(
            "Non-root URL must NOT end with a trailing slash "
            "(the live site redirects trailing-slash URLs to the "
            "no-slash form)."
        )

    # Validate hostname.
    try:
        hostname = parsed.hostname

        if not hostname:
            errors.append("URL hostname is empty.")
        else:
            validate_hostname(hostname)

            if hostname.lower().startswith("www."):
                errors.append(
                    "URL hostname must not start with 'www.' "
                    "(the live site redirects www -> non-www)."
                )

    except ValueError as exc:
        errors.append(f"Invalid hostname: {exc}")

    # Validate port parsing.
    try:
        _ = parsed.port
    except ValueError as exc:
        errors.append(f"Invalid port: {exc}")

    # Check percent encoding.
    bad_percent = re.search(
        r"%(?![0-9A-Fa-f]{2})",
        url
    )

    if bad_percent:
        errors.append(
            "URL contains invalid percent-encoding."
        )

    # Expected domain restriction.
    if expected_domain:
        expected = urllib.parse.urlsplit(expected_domain)

        if parsed.scheme.lower() != expected.scheme.lower():
            errors.append(
                "URL scheme does not match configured domain."
            )

        if parsed.hostname and expected.hostname:
            if parsed.hostname.lower() != expected.hostname.lower():
                errors.append(
                    "URL hostname does not match configured domain."
                )

    return errors


# ---------------------------------------------------------------------------
# Sitemap priority calculation
# ---------------------------------------------------------------------------

def calculate_priority(
    rel_dir,
    root_priority,
    page_priority,
    deep_priority,
):
    """
    Calculate priority based on URL depth.
    """

    if not rel_dir:
        return root_priority

    depth = len(
        [
            part
            for part in rel_dir.split("/")
            if part
        ]
    )

    if depth <= 1:
        return page_priority

    return deep_priority


# ---------------------------------------------------------------------------
# Scan website
# ---------------------------------------------------------------------------

def scan_pages(
    root,
    excludes,
    domain,
    root_priority,
    page_priority,
    deep_priority,
):
    """
    Return:

        {
            canonical_url: {
                "filepath": ...,
                "lastmod": ...,
                "priority": ...
            }
        }
    """

    pages = {}

    root = os.path.abspath(root)

    if not os.path.isdir(root):
        raise SystemExit(
            f"ERROR: web root does not exist or is not a directory: {root}"
        )

    for dirpath, dirnames, filenames in os.walk(root):

        # Do not descend into hidden directories.
        dirnames[:] = sorted(
            d for d in dirnames
            if not d.startswith(".")
        )

        rel_dir = os.path.relpath(
            dirpath,
            root
        ).replace(os.sep, "/")

        if rel_dir == ".":
            rel_dir = ""

        if is_excluded(rel_dir, excludes):
            dirnames[:] = []
            continue

        if INDEX_FILE not in filenames:
            continue

        full_path = os.path.join(
            dirpath,
            INDEX_FILE
        )

        url = build_page_url(
            domain,
            rel_dir
        )

        validation_errors = validate_url(
            url,
            expected_domain=domain
        )

        if validation_errors:
            raise SystemExit(
                "ERROR: generated invalid URL:\n"
                f"  {url}\n"
                + "\n".join(
                    f"  - {error}"
                    for error in validation_errors
                )
            )

        try:
            timestamp = os.path.getmtime(full_path)
            lastmod = datetime.datetime.fromtimestamp(
                timestamp,
                datetime.timezone.utc,
            ).date().isoformat()

        except OSError as exc:
            raise SystemExit(
                f"ERROR: unable to read modification time for {full_path}: {exc}"
            )

        priority = calculate_priority(
            rel_dir,
            root_priority,
            page_priority,
            deep_priority,
        )

        pages[url] = {
            "filepath": full_path,
            "lastmod": lastmod,
            "priority": f"{priority:.1f}",
        }

    return pages


# ---------------------------------------------------------------------------
# Build XML sitemap
# ---------------------------------------------------------------------------

def build_sitemap_xml(
    pages,
    changefreq=DEFAULT_CHANGEFREQ,
):
    """
    Generate UTF-8 XML sitemap.
    """

    root = ET.Element(
        f"{{{NS}}}urlset"
    )

    for url in sorted(pages):

        data = pages[url]

        url_element = ET.SubElement(
            root,
            f"{{{NS}}}url"
        )

        loc = ET.SubElement(
            url_element,
            f"{{{NS}}}loc"
        )

        # ElementTree automatically XML-escapes:
        #   &
        #   <
        #   >
        #   "
        #   '
        #
        # URL itself has already been RFC encoded.
        loc.text = url

        lastmod = ET.SubElement(
            url_element,
            f"{{{NS}}}lastmod"
        )

        lastmod.text = data["lastmod"]

        change = ET.SubElement(
            url_element,
            f"{{{NS}}}changefreq"
        )

        change.text = changefreq

        priority = ET.SubElement(
            url_element,
            f"{{{NS}}}priority"
        )

        priority.text = data["priority"]

    ET.indent(
        root,
        space="  "
    )

    xml = ET.tostring(
        root,
        encoding="utf-8",
        xml_declaration=False,
    )

    return (
        b'<?xml version="1.0" encoding="UTF-8"?>\n'
        + xml
        + b"\n"
    )


# ---------------------------------------------------------------------------
# Sitemap validation
# ---------------------------------------------------------------------------

def validate_sitemap(
    path,
    expected_pages,
    domain,
    expected_changefreq,
):
    """
    Validate:
      - XML well-formedness
      - correct sitemap namespace
      - correct URL set
      - no duplicates
      - RFC 3986/3987-compatible URLs
      - root ends with "/"; non-root has NO trailing slash
      - lastmod format
      - changefreq
      - priority
    """

    errors = []

    try:
        tree = ET.parse(path)

    except ET.ParseError as exc:
        return [
            f"XML parse error: {exc}"
        ]

    root = tree.getroot()

    expected_root = f"{{{NS}}}urlset"

    if root.tag != expected_root:
        errors.append(
            "Root element is not the Sitemap <urlset> "
            f"with namespace {NS!r}."
        )

    url_elements = root.findall(
        f"./{{{NS}}}url"
    )

    loc_texts = []

    for index, url_element in enumerate(
        url_elements,
        start=1,
    ):

        loc = url_element.find(
            f"{{{NS}}}loc"
        )

        lastmod = url_element.find(
            f"{{{NS}}}lastmod"
        )

        changefreq = url_element.find(
            f"{{{NS}}}changefreq"
        )

        priority = url_element.find(
            f"{{{NS}}}priority"
        )

        # ---------------------------------------------------------------
        # <loc>
        # ---------------------------------------------------------------

        if loc is None or not loc.text:
            errors.append(
                f"<url> #{index} is missing <loc>."
            )
            continue

        url = loc.text.strip()

        loc_texts.append(url)

        url_errors = validate_url(
            url,
            expected_domain=domain,
        )

        for error in url_errors:
            errors.append(
                f"<loc> #{index}: {error}"
            )

        # ---------------------------------------------------------------
        # <lastmod>
        # ---------------------------------------------------------------

        if lastmod is None or not lastmod.text:
            errors.append(
                f"<url> #{index} is missing <lastmod>."
            )

        else:
            value = lastmod.text.strip()

            try:
                datetime.date.fromisoformat(value)

            except ValueError:
                errors.append(
                    f"<url> #{index}: invalid lastmod date: {value!r}"
                )

        # ---------------------------------------------------------------
        # <changefreq>
        # ---------------------------------------------------------------

        if changefreq is None or not changefreq.text:
            errors.append(
                f"<url> #{index} is missing <changefreq>."
            )

        else:
            value = changefreq.text.strip().lower()

            if value not in VALID_CHANGEFREQ:
                errors.append(
                    f"<url> #{index}: invalid changefreq: {value!r}"
                )

            elif value != expected_changefreq:
                errors.append(
                    f"<url> #{index}: expected changefreq "
                    f"{expected_changefreq!r}, got {value!r}"
                )

        # ---------------------------------------------------------------
        # <priority>
        # ---------------------------------------------------------------

        if priority is None or not priority.text:
            errors.append(
                f"<url> #{index} is missing <priority>."
            )

        else:
            value = priority.text.strip()

            try:
                number = float(value)

                if not 0.0 <= number <= 1.0:
                    errors.append(
                        f"<url> #{index}: priority must be "
                        f"between 0.0 and 1.0."
                    )

            except ValueError:
                errors.append(
                    f"<url> #{index}: invalid priority: {value!r}"
                )

    # -------------------------------------------------------------------
    # Duplicate detection
    # -------------------------------------------------------------------

    seen = set()
    duplicates = set()

    for url in loc_texts:

        if url in seen:
            duplicates.add(url)

        seen.add(url)

    if duplicates:
        errors.append(
            f"{len(duplicates)} duplicate URL(s): "
            f"{sorted(duplicates)[:10]}"
        )

    # -------------------------------------------------------------------
    # Expected URL set
    # -------------------------------------------------------------------

    actual_urls = set(loc_texts)

    expected_urls = set(expected_pages)

    missing = expected_urls - actual_urls
    extra = actual_urls - expected_urls

    if missing:
        errors.append(
            f"{len(missing)} expected URL(s) missing: "
            f"{sorted(missing)[:10]}"
        )

    if extra:
        errors.append(
            f"{len(extra)} unexpected URL(s): "
            f"{sorted(extra)[:10]}"
        )

    # -------------------------------------------------------------------
    # URL count
    # -------------------------------------------------------------------

    if len(loc_texts) != len(expected_urls):
        errors.append(
            f"URL count mismatch: sitemap={len(loc_texts)}, "
            f"expected={len(expected_urls)}."
        )

    return errors


# ---------------------------------------------------------------------------
# Atomic file writing
# ---------------------------------------------------------------------------

def atomic_write(path, data):
    """
    Write data to a temporary file and atomically replace destination.
    """

    directory = os.path.dirname(
        os.path.abspath(path)
    )

    os.makedirs(
        directory,
        exist_ok=True,
    )

    temp_path = path + ".tmp"

    try:
        with open(
            temp_path,
            "wb",
        ) as handle:

            handle.write(data)
            handle.flush()
            os.fsync(handle.fileno())

        os.replace(
            temp_path,
            path,
        )

    except Exception:

        if os.path.exists(temp_path):
            try:
                os.remove(temp_path)
            except OSError:
                pass

        raise


# ---------------------------------------------------------------------------
# Main
# ---------------------------------------------------------------------------

def main():

    try:
        args = parse_args()

    except ValueError as exc:
        print(
            f"ERROR: {exc}",
            file=sys.stderr,
        )
        sys.exit(1)

    print("=" * 72)
    print(
        "SITEMAP GENERATOR — RFC 3986 / RFC 3987 / XML VALIDATION"
    )
    print("=" * 72)

    print(
        f"Mode:         "
        f"{'APPLY' if args.apply else 'DRY RUN'}"
    )

    print(f"Root:         {args.root}")
    print(f"Output:       {args.out}")
    print(f"Domain:       {args.domain}")
    print(f"Changefreq:   {args.changefreq}")
    print(
        f"Priorities:   root={args.root_priority:.1f}, "
        f"page={args.page_priority:.1f}, "
        f"deep={args.deep_priority:.1f}"
    )
    print(
        "URL format:   root='/' , non-root='/path' (NO trailing slash)"
    )

    print()

    # -------------------------------------------------------------------
    # Validate domain before scanning
    # -------------------------------------------------------------------

    domain_errors = validate_url(
        args.domain + "/",
        expected_domain=args.domain,
    )

    if domain_errors:
        print("ERROR: invalid domain:")

        for error in domain_errors:
            print(f"  ! {error}")

        sys.exit(1)

    # -------------------------------------------------------------------
    # Scan
    # -------------------------------------------------------------------

    try:
        pages = scan_pages(
            root=args.root,
            excludes=args.excludes,
            domain=args.domain,
            root_priority=args.root_priority,
            page_priority=args.page_priority,
            deep_priority=args.deep_priority,
        )

    except SystemExit:
        raise

    except Exception as exc:
        print(
            f"ERROR: sitemap scan failed: {exc}",
            file=sys.stderr,
        )
        sys.exit(1)

    urls = sorted(pages)

    print(
        f"Pages found (index.html): {len(urls)}"
    )

    if not urls:
        print(
            "WARNING: no index.html pages were found."
        )

    for url in urls:

        data = pages[url]

        print(
            f"  {url}"
            f"  lastmod={data['lastmod']}"
            f"  changefreq={args.changefreq}"
            f"  priority={data['priority']}"
        )

    # -------------------------------------------------------------------
    # Validate every URL before generating XML
    # -------------------------------------------------------------------

    print()
    print("-" * 72)
    print("URL VALIDATION")
    print("-" * 72)

    url_errors = []

    for url in urls:

        errors = validate_url(
            url,
            expected_domain=args.domain,
        )

        if errors:

            for error in errors:
                url_errors.append(
                    f"{url}: {error}"
                )

    if url_errors:

        print("FAILED:")

        for error in url_errors:
            print(f"  ! {error}")

        sys.exit(1)

    print(
        "PASSED: all URLs satisfy the configured "
        "RFC 3986 / RFC 3987 / XML safety checks."
    )

    # -------------------------------------------------------------------
    # Dry run
    # -------------------------------------------------------------------

    if not args.apply:

        print()
        print("=" * 72)
        print("DRY RUN COMPLETE")
        print("=" * 72)

        print(
            "No sitemap file was written."
        )

        print(
            "Run again with --apply to generate and validate sitemap.xml."
        )

        return

    # -------------------------------------------------------------------
    # Existing sitemap backup
    # -------------------------------------------------------------------

    backup_path = None

    if os.path.exists(args.out):

        backup_path = args.out + ".bak"

        shutil.copy2(
            args.out,
            backup_path,
        )

        print()
        print(
            f"Backed up existing sitemap:"
            f" {backup_path}"
        )

    # -------------------------------------------------------------------
    # Generate XML
    # -------------------------------------------------------------------

    xml_bytes = build_sitemap_xml(
        pages,
        changefreq=args.changefreq,
    )

    # -------------------------------------------------------------------
    # Write atomically
    # -------------------------------------------------------------------

    try:

        atomic_write(
            args.out,
            xml_bytes,
        )

        print(
            f"Wrote sitemap: {args.out}"
        )

    except Exception as exc:

        print(
            f"ERROR: unable to write sitemap: {exc}",
            file=sys.stderr,
        )

        if backup_path and os.path.exists(
            backup_path
        ):

            shutil.copy2(
                backup_path,
                args.out,
            )

            print(
                "Restored previous sitemap from backup."
            )

        sys.exit(1)

    # -------------------------------------------------------------------
    # Validate generated sitemap
    # -------------------------------------------------------------------

    print()
    print("=" * 72)
    print("SITEMAP VALIDATION")
    print("=" * 72)

    errors = validate_sitemap(
        path=args.out,
        expected_pages=pages,
        domain=args.domain,
        expected_changefreq=args.changefreq,
    )

    if errors:

        print("FAILED:")

        for error in errors:
            print(f"  ! {error}")

        if backup_path and os.path.exists(
            backup_path
        ):

            shutil.copy2(
                backup_path,
                args.out,
            )

            print(
                "Restored previous sitemap from backup."
            )

        sys.exit(1)

    print(
        "PASSED: XML well-formed."
    )

    print(
        "PASSED: sitemap namespace is correct."
    )

    print(
        "PASSED: all URLs are valid HTTP/HTTPS URLs."
    )

    print(
        "PASSED: URLs use root='/' / non-root-no-trailing-slash convention."
    )

    print(
        "PASSED: no duplicate URLs."
    )

    print(
        "PASSED: URL set matches index.html files on disk."
    )

    print(
        "PASSED: all <lastmod> values are valid ISO dates."
    )

    print(
        "PASSED: all <changefreq> values are 'always'."
    )

    print(
        "PASSED: all <priority> values are between 0.0 and 1.0."
    )

    # -------------------------------------------------------------------
    # Remove backup only after successful validation
    # -------------------------------------------------------------------

    if backup_path and os.path.exists(
        backup_path
    ):

        os.remove(
            backup_path
        )

        print(
            f"Removed backup: {backup_path}"
        )

    # -------------------------------------------------------------------
    # Summary
    # -------------------------------------------------------------------

    print()
    print("=" * 72)
    print("SUMMARY")
    print("=" * 72)

    print(
        f"Total URLs written: {len(urls)}"
    )

    print(
        f"Changefreq: always"
    )

    print(
        "URL format: root ends with '/', all other URLs have NO trailing slash"
    )

    print(
        "RFC 3986 validation: PASSED"
    )

    print(
        "RFC 3987 compatibility: PASSED"
    )

    print(
        "XML validation: PASSED"
    )

    print(
        "Sitemap validation: PASSED"
    )

    print(
        f"Output: {args.out}"
    )

    print("=" * 72)


if __name__ == "__main__":
    main()