From 2ebfc63f6cf0211501bb0a1fd419331427df05d7 Mon Sep 17 00:00:00 2001 From: aldbr Date: Tue, 2 Sep 2025 10:25:55 +0200 Subject: [PATCH] feat: support CVMFS multiarch layout --- dirac.cfg | 18 +- .../Resources/computingelements.rst | 62 ++ .../Core/Utilities/ContainerImageResolver.py | 373 +++++++++++ src/DIRAC/Core/Utilities/Os.py | 59 +- .../test/Test_ContainerImageResolver.py | 626 ++++++++++++++++++ .../Core/scripts/dirac_apptainer_exec.py | 36 +- .../Computing/SingularityComputingElement.py | 30 +- .../Resources/Computing/Test_SingularityCE.py | 5 +- 8 files changed, 1164 insertions(+), 45 deletions(-) create mode 100644 src/DIRAC/Core/Utilities/ContainerImageResolver.py create mode 100644 src/DIRAC/Core/Utilities/test/Test_ContainerImageResolver.py diff --git a/dirac.cfg b/dirac.cfg index 0c255e2d89a..4a9e02e1b71 100644 --- a/dirac.cfg +++ b/dirac.cfg @@ -641,9 +641,23 @@ Resources Singularity { - # The root image location for the container to use + # OCI image reference for the multiarch layout + # The resolver builds: //, so the + # reference must be relative: absolute paths and ".." are rejected. + # No default -- if unset, multiarch resolution is skipped and ContainerRoot is used. + # The image must be published under ImageBasePath for every architecture you run + # on. The reference below is only an example: use the image your VO builds. + # ImageReference = registry.hub.docker.com/library/ubuntu:24.04 + + # Base path for the CVMFS multiarch image repository + # Default: /cvmfs/unpacked.cern.ch/.multiarch + # ImageBasePath = /cvmfs/unpacked.cern.ch/.multiarch + + # (Deprecated) Legacy root image location for the container, a single + # architecture (x86_64) image. Used as a fallback when the multiarch image + # is not found, and to be removed in a future release: prefer ImageReference. # Default: /cvmfs/cernvm-prod.cern.ch/cvm4 - ContainerRoot = /cvmfs/cernvm-prod.cern.ch/cvm4 + # ContainerRoot = /cvmfs/cernvm-prod.cern.ch/cvm4 # List of directories to bind ContainerBind = /etc/grid-security,someDir:::BoundHere diff --git a/docs/source/AdministratorGuide/Resources/computingelements.rst b/docs/source/AdministratorGuide/Resources/computingelements.rst index 9ecfa12a820..1cee46abbe2 100644 --- a/docs/source/AdministratorGuide/Resources/computingelements.rst +++ b/docs/source/AdministratorGuide/Resources/computingelements.rst @@ -172,6 +172,68 @@ section :: } } +Selecting the container image +@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ + +The :mod:`~DIRAC.Resources.Computing.SingularityComputingElement` CE and the +:py:mod:`~DIRAC.Core.scripts.dirac_apptainer_exec` command resolve the container image with +:py:func:`~DIRAC.Core.Utilities.ContainerImageResolver.resolveImagePath`, which supports the CVMFS multiarch layout so +that the same configuration works on nodes of different architectures. + +The following options are read from the ``Singularity`` section (or from the CE parameters, which take precedence). +The image references shown are examples: a site normally points them at the image its VO builds and publishes. + +- ImageReference (str) - the OCI reference of the image, e.g. ``registry.hub.docker.com/library/ubuntu:24.04``. It must + be a *relative* reference: absolute paths and ``..`` are rejected, since they would bypass both + the base path and the architecture directory. +- ImageBasePath (str) - the root of the multiarch image repository. Defaults to ``/cvmfs/unpacked.cern.ch/.multiarch``. +- ContainerRoot (str) - **deprecated**, kept for backward compatibility. A path to a single architecture (in practice + x86_64) image, used as a fallback when no multiarch image is found. Defaults to + ``/cvmfs/cernvm-prod.cern.ch/cvm4``. + +The image is looked up at ``//``, where the architecture is the OCI +(GOARCH) name of the node, with the variant appended after a colon where there is one: ``amd64``, ``arm64``, +``arm:v7``, ``386``, ``ppc64le``, etc. The repository also publishes symlinks for the usual ``uname -m`` names +(``x86_64``, ``aarch64``, ``i386`` ...), so an architecture DIRAC does not know about is used as reported. + +Some architectures are published under more than one directory name, because the name comes from the image manifest: +an image declaring ``arm64`` with no variant is published under ``arm64``, one declaring variant ``v8`` under +``arm64:v8``, and the publisher only symlinks one to the other when the plain name is not already a real directory. +Those names are therefore tried in turn -- ``arm64`` then ``arm64:v8`` on an ARM64 node, ``arm:v7`` then ``arm`` on an +ARMv7 one -- and the first one that exists is used. When none of them does, the warning names every path that was +tried, together with the directories the repository actually publishes for that architecture, so that a variant DIRAC +does not yet know about is visible in the log rather than silently degrading to ``ContainerRoot``. Such a directory is +only reported, never used: variant compatibility is directional (an ``arm:v6`` image runs on an ARMv7 node, an +``arm:v7`` image does not run on an ARMv6 one), so an unrecognised variant cannot be assumed to run. + +If no multiarch path exists, the deprecated ``ContainerRoot`` is used instead and a warning is logged, naming the +architecture and every path that was tried. + +``ContainerRoot`` images are built for a single architecture, in practice ``amd64``. On a node whose architecture DIRAC +recognises as a different one, the image is therefore *not* used: resolution fails with an error and the payload is not +submitted, rather than started only to die with an exec format error. Publish the image under ``ImageBasePath`` for that +architecture to fix it. When the node reports an architecture DIRAC does not recognise, it may still be a compatible +one, so ``ContainerRoot`` is used and a warning logged instead. + +:: + + Resources + { + Computing + { + Singularity + { + ImageReference = registry.hub.docker.com/library/ubuntu:24.04 + # ImageBasePath = /cvmfs/unpacked.cern.ch/.multiarch + } + } + } + +``dirac-apptainer-exec`` accepts a ``-i``/``--image`` option overriding the configuration. That value is used on its +own: an existing local path is taken as-is, and otherwise it is looked up as an OCI reference in the multiarch layout. +``ContainerRoot`` is not consulted in that case, so the command never runs an image other than the one asked for. +Unlike the CE, it has no built-in default image: if nothing is configured and nothing is found, the command fails. + Applying cgroup2 limits to computing resources @@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@ diff --git a/src/DIRAC/Core/Utilities/ContainerImageResolver.py b/src/DIRAC/Core/Utilities/ContainerImageResolver.py new file mode 100644 index 00000000000..ea2cfb6b73d --- /dev/null +++ b/src/DIRAC/Core/Utilities/ContainerImageResolver.py @@ -0,0 +1,373 @@ +"""Utilities to resolve container images for Apptainer/Singularity +based on machine architecture and CVMFS multiarch layout. +""" +from __future__ import annotations + +import platform +from pathlib import Path, PurePosixPath + +from DIRAC import gConfig, gLogger +from DIRAC.Core.Utilities.Os import safe_exists, safe_listdir + +BASE_PATH_DEFAULT = "/cvmfs/unpacked.cern.ch/.multiarch" +# Legacy default used before multiarch support +CONTAINER_DEFROOT = "/cvmfs/cernvm-prod.cern.ch/cvm4" +# Architecture the legacy (single architecture) container roots are built for +LEGACY_ARCH = "amd64" +# Images live on lazily mounted CVMFS: never block a job slot forever on a stat() +EXISTS_TIMEOUT = 60 + +# Candidate directory names in the multiarch repository for a given +# platform.machine() value, in the order they should be tried. +# +# DUCC, the tool publishing unpacked.cern.ch, writes one directory per manifest +# entry named after the OCI image spec architecture (the GOARCH value), with the +# variant appended after a colon when the manifest declares one -- see +# GetNameWithArch() in ducc/lib/conversion.go. The same logical architecture can +# therefore appear under more than one name: an image whose manifest declares +# arm64 with no variant lands in "arm64", one declaring variant "v8" lands in +# "arm64:v8". +# +# DUCC also publishes alias symlinks for the usual uname spellings (aarch64, +# x86_64, x86-64, i386, armel, armhf), but only when the alias name does not +# already exist as a real directory, and pointing at the first of an ordered +# list of candidates -- see archAliases and createMultiarchAliasSymlinksWithLogger() +# in the same file. A repository holding both "arm64" and "arm64:v8" as real +# directories thus has no symlink bridging the two, and an image published under +# only one of them cannot be reached through the other. We mirror DUCC's own +# preference lists and try every candidate in turn. +# +# Unknown values are passed through unchanged, and stand a fair chance of +# resolving through one of those alias symlinks. +# See https://github.com/opencontainers/image-spec/blob/main/image-index.md +_OCI_ARCH_CANDIDATES = { + "x86_64": ("amd64",), + "x86-64": ("amd64",), + "amd64": ("amd64",), + "i386": ("386",), + "i486": ("386",), + "i586": ("386",), + "i686": ("386",), + "386": ("386",), + # The OCI spec treats arm64 with no variant as equivalent to v8, and many + # registries publish only the variant form + "aarch64": ("arm64", "arm64:v8"), + "arm64": ("arm64", "arm64:v8"), + "armv7l": ("arm:v7", "arm"), + "armhf": ("arm:v7", "arm"), + "armv6l": ("arm:v6", "arm"), + "armel": ("arm:v6", "arm"), + "armv5tel": ("arm:v5", "arm"), + "arm": ("arm",), + "ppc64le": ("ppc64le",), + "s390x": ("s390x",), + "riscv64": ("riscv64",), +} + +# The resolver runs on every single payload submission, so a misconfigured site +# would otherwise repeat the same advice once per payload. Configuration notes +# (a deprecated option, a fallback taken) are therefore emitted once per process. +# Failures are NOT routed through this: a condition that kills the payload must be +# logged every time it kills one, or only the first casualty leaves a trace. +_emittedMessages = set() + +# The resolver is called once per payload, and its answer cannot change under a +# running pilot: the configuration is fixed and the CVMFS layout is not rewritten +# beneath it. Successful resolutions are therefore cached, so that a pilot stats +# CVMFS once instead of once per payload. +# Failures are deliberately NOT cached: a miss can be a transient mount problem, +# and retrying costs only the path that is already slow and already loud. +_resolvedPaths = {} + + +def getArchCandidates(arch: str) -> list[str]: + """Return the multiarch directory names to try for ``arch``, in order. + + Accepts ``platform.machine()`` values (e.g. ``x86_64``, ``aarch64``) as well + as OCI names (e.g. ``amd64``, ``arm64``). More than one name is returned + where the repository may publish the same architecture either plain or + variant-qualified (e.g. ``arm64`` and ``arm64:v8``). Unknown values yield a + single candidate, as-is, lowercased. + """ + arch = arch.lower() + return list(_OCI_ARCH_CANDIDATES.get(arch, (arch,))) + + +def normalizeArch(arch: str) -> str: + """Normalise an architecture string to its preferred multiarch directory name. + + This is the first of :func:`getArchCandidates`, and the name used to + describe the node in log messages. + """ + return getArchCandidates(arch)[0] + + +def _warnOnce(message: str, log) -> None: + """Emit ``message`` at most once per process.""" + if message in _emittedMessages: + return + _emittedMessages.add(message) + log.warn(message) + + +def _remember(cacheKey: tuple, path: Path) -> Path: + """Cache a successful resolution and return it; see :data:`_resolvedPaths`.""" + _resolvedPaths[cacheKey] = path + return path + + +def _isKnownIncompatibleArch(arch: str) -> bool: + """True when this node is an architecture the legacy roots demonstrably cannot run. + + Only architectures this release recognises are judged. An unrecognised + ``platform.machine()`` value may well be a spelling of the legacy architecture + that we simply do not know, so it is warned about rather than refused: a node + that works today must not stop working because its uname is unusual. + """ + return bool(arch) and arch.lower() in _OCI_ARCH_CANDIDATES and normalizeArch(arch) != LEGACY_ARCH + + +def _isSafeImageRef(imageRef: str, log) -> bool: + """Check that an image reference can be appended to the multiarch base path. + + An absolute reference would make ``pathlib`` discard both the base path and + the architecture directory, and a reference containing ``..`` would escape + the base path: in either case the resolved path is not the documented + ``//`` and the architecture is silently ignored. + """ + refPath = PurePosixPath(imageRef) + if refPath.is_absolute(): + log.error( + "ImageReference must be a relative OCI reference, not an absolute path", + f"{imageRef} -- use ContainerRoot for a local image path", + ) + return False + if ".." in refPath.parts: + log.error("ImageReference must not contain '..'", imageRef) + return False + return True + + +def getMultiarchPaths(imageRef: str, basePath: str, arch: str, log=None) -> list[Path]: + """Build the candidate CVMFS multiarch paths for the given OCI image reference. + + :param imageRef: Full OCI reference (e.g. ``registry.hub.docker.com/library/alpine:latest``) + :param basePath: Multiarch base directory (e.g. ``/cvmfs/unpacked.cern.ch/.multiarch``) + :param arch: architecture to resolve for (e.g. ``x86_64``) + :param log: optional logger, defaults to a sub-logger of ``gLogger`` + :returns: one path ``//`` per candidate + directory name of ``arch``, preferred first, or an empty list if the + inputs cannot yield a valid path + + Example:: + + >>> getMultiarchPaths("registry.hub.docker.com/library/alpine:latest", + ... "/cvmfs/unpacked.cern.ch/.multiarch", arch="aarch64") + [PosixPath('/cvmfs/unpacked.cern.ch/.multiarch/arm64/registry.hub.docker.com/library/alpine:latest'), + PosixPath('/cvmfs/unpacked.cern.ch/.multiarch/arm64:v8/registry.hub.docker.com/library/alpine:latest')] + """ + log = log or gLogger.getSubLogger("ContainerImageResolver") + if not arch: + log.error( + "Cannot determine the machine architecture", + "platform.machine() returned an empty value: skipping the multiarch lookup", + ) + return [] + if not imageRef or not _isSafeImageRef(imageRef, log): + return [] + return [Path(basePath) / candidate / imageRef for candidate in getArchCandidates(arch)] + + +def _existsOnCvmfs(path: Path, log) -> bool: + """Existence check protected against an unresponsive CVMFS mount.""" + found = safe_exists(str(path), timeout=EXISTS_TIMEOUT) + if found is None: + log.warn(f"Timed out after {EXISTS_TIMEOUT}s while checking container image path, assuming absent", str(path)) + return False + return found + + +def _findMultiarchImage(imageRef: str | None, basePath: str, arch: str, log) -> tuple[Path | None, list[Path]]: + """Return the first existing multiarch path, and every candidate tried. + + The candidate list is returned even when nothing is found, so that callers + can report exactly what was looked for. + """ + candidates = getMultiarchPaths(imageRef, basePath, arch, log=log) if imageRef else [] + for candidate in candidates: + if _existsOnCvmfs(candidate, log): + log.debug("Resolved multiarch image path", str(candidate)) + return candidate, candidates + return None, candidates + + +def _getPublishedArchDirs(basePath: str, arch: str, log) -> list[str]: + """List the directory names the repository publishes for this architecture. + + Called only once the multiarch lookup has missed, so the extra directory + listing costs nothing on the path where the image is found. It exists so + that a variant directory this release does not know about -- say a future + ``arm64:v9`` -- is named in the log, instead of the resolver silently + degrading to the legacy single architecture image. + + The match is on the GOARCH part of the name, so an ARMv7 node is told about + ``arm``, ``arm:v6`` and ``arm:v7`` alike. + """ + bases = {candidate.split(":", 1)[0] for candidate in getArchCandidates(arch)} + entries = safe_listdir(basePath, timeout=EXISTS_TIMEOUT) + if entries is None: + log.warn(f"Timed out after {EXISTS_TIMEOUT}s while listing the multiarch base path", basePath) + return [] + return sorted(entry for entry in entries if entry.split(":", 1)[0] in bases) + + +def _getPublishedArchSummary(published: list[str], arch: str) -> str: + """One sentence describing what the repository holds for this architecture.""" + goarch = normalizeArch(arch) + if published: + return f"The repository publishes {', '.join(published)} for architecture '{goarch}'. " + return f"The repository publishes nothing for architecture '{goarch}'. " + + +def findMultiarchImage( + imageRef: str, + basePath: str | None = None, + arch: str | None = None, + log=None, +) -> Path | None: + """Resolve an OCI reference in the multiarch layout, with no legacy fallback. + + :param imageRef: OCI image reference + (e.g. ``registry.hub.docker.com/library/alpine:latest``) + :param basePath: Base directory for the multiarch layout, defaults to the + ``ImageBasePath`` CS option and then to :data:`BASE_PATH_DEFAULT` + :param arch: Override architecture (default: autodetect via ``platform.machine()``) + :param log: optional logger, defaults to a sub-logger of ``gLogger`` + :returns: the image path if it exists, else ``None`` + """ + log = log or gLogger.getSubLogger("ContainerImageResolver") + basePath = basePath or gConfig.getValue("/Resources/Computing/Singularity/ImageBasePath") or BASE_PATH_DEFAULT + arch = (arch or platform.machine() or "").strip() + + found, _candidates = _findMultiarchImage(imageRef, basePath, arch, log) + return found + + +def resolveImagePath( + imageRef: str | None = None, + basePath: str | None = None, + containerRoot: str | None = None, + arch: str | None = None, + defaultRoot: str | None = CONTAINER_DEFROOT, + log=None, +) -> Path | None: + """Resolve the container image path to use for Apptainer/Singularity. + + Resolution order: + + 1. **Multiarch** ``//`` -- only attempted when + an ``imageRef`` is explicitly provided (via parameter or CS config). + 2. **Legacy** ``containerRoot`` parameter, ``ContainerRoot`` CS option, or + ``defaultRoot`` -- deprecated, and logged as such when used. + 3. ``None`` if nothing is found. + + :param imageRef: OCI image reference + (e.g. ``registry.hub.docker.com/library/alpine:latest``) + :param basePath: Base directory for the multiarch layout + (e.g. ``/cvmfs/unpacked.cern.ch/.multiarch``) + :param containerRoot: Legacy container root path for backward compatibility + :param arch: Override architecture (default: autodetect via ``platform.machine()``) + :param defaultRoot: Built-in legacy root, used when neither ``containerRoot`` + nor the ``ContainerRoot`` CS option is set. Pass ``None`` to require an + explicit configuration. + :param log: optional logger, defaults to a sub-logger of ``gLogger`` + :returns: resolved :class:`~pathlib.Path` or ``None`` + """ + log = log or gLogger.getSubLogger("ContainerImageResolver") + + imageRef = imageRef or gConfig.getValue("/Resources/Computing/Singularity/ImageReference") or None + basePath = basePath or gConfig.getValue("/Resources/Computing/Singularity/ImageBasePath") or BASE_PATH_DEFAULT + arch = (arch or platform.machine() or "").strip() + + legacyRoot = containerRoot or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or defaultRoot + + # Keyed on the effective inputs, never on the raw arguments: gConfig can be + # refreshed under a running process, and a changed option must not be masked + # by an entry cached from the previous value. + cacheKey = (imageRef, basePath, legacyRoot, arch) + if cacheKey in _resolvedPaths: + return _resolvedPaths[cacheKey] + + # 1) Try the CVMFS multiarch paths (only if an image reference is configured) + found, candidates = _findMultiarchImage(imageRef, basePath, arch, log) + if found: + return _remember(cacheKey, found) + + # The multiarch lookup missed. Report what the repository does publish for + # this architecture: a directory name we do not know about is the one thing + # that would otherwise degrade to the legacy image without saying why. + published = _getPublishedArchDirs(basePath, arch, log) if candidates else [] + + # 2) Fall back to the legacy ContainerRoot, resolved above + if legacyRoot and _existsOnCvmfs(Path(legacyRoot), log): + # The legacy roots are single architecture images, in practice amd64 ones. + # On a node known not to be able to run one, returning it would start a + # payload that dies with an exec format error -- so refuse instead, and + # let the caller fail with a diagnosable error. + incompatible = _isKnownIncompatibleArch(arch) + + if imageRef: + # Multiarch was attempted but did not yield a usable image + if candidates: + tried = ", ".join(str(candidate) for candidate in candidates) + detail = ( + f"Multiarch image not found for architecture '{normalizeArch(arch)}' (tried {tried}). " + + _getPublishedArchSummary(published, arch) + ) + else: + detail = ( + f"Could not build a multiarch path for ImageReference '{imageRef}' " + f"on architecture '{normalizeArch(arch) if arch else 'unknown'}'. " + ) + if not incompatible: + detail += f"Falling back to legacy ContainerRoot '{legacyRoot}'. " + _warnOnce(detail + "Please verify your ImageReference and ImageBasePath settings.", log) + elif not incompatible: + # No ImageReference configured, pure legacy usage + _warnOnce( + f"Using legacy ContainerRoot '{legacyRoot}'. " + "ContainerRoot is deprecated and will be removed in a future release. " + "Please configure ImageReference and ImageBasePath for the multiarch layout.", + log, + ) + + if incompatible: + # Logged on every payload, not once per process: this kills every one of them + log.error( + "Legacy ContainerRoot cannot run on this node", + f"'{legacyRoot}' is a single architecture ({LEGACY_ARCH}) image but this node is " + f"'{normalizeArch(arch)}': the payload would fail with an exec format error. " + "Publish the image for this architecture under ImageBasePath, or point ContainerRoot " + "at an image built for it.", + ) + return None + + if arch and normalizeArch(arch) != LEGACY_ARCH: + # Unrecognised architecture: it may still be a legacy-compatible node, + # so this is advice rather than a refusal + _warnOnce( + f"Legacy ContainerRoot '{legacyRoot}' is a single architecture ({LEGACY_ARCH}) image and this " + f"node reports an unrecognised architecture '{arch}': the payload may fail with an exec " + "format error. Publish the image for this architecture under ImageBasePath.", + log, + ) + return _remember(cacheKey, Path(legacyRoot)) + + log.error( + "No container image could be resolved", + f"arch={normalizeArch(arch) if arch else 'unknown'}, " + f"multiarch={', '.join(str(c) for c in candidates) or ('unusable ImageReference' if imageRef else 'not attempted')}, " + f"published={', '.join(published) or 'none'}, " + f"legacy={legacyRoot or 'not configured'}", + ) + return None diff --git a/src/DIRAC/Core/Utilities/Os.py b/src/DIRAC/Core/Utilities/Os.py index 8ca5f1a4148..edb0ce87a48 100755 --- a/src/DIRAC/Core/Utilities/Os.py +++ b/src/DIRAC/Core/Utilities/Os.py @@ -129,31 +129,64 @@ def sourceEnv(timeout, cmdTuple, inputEnv=None): return result +def _callWithTimeout(call, timeout): + """Run ``call()`` in a daemon thread, giving up after ``timeout`` seconds. + + Lazily loaded File Systems like CVMFS can block a single stat() or readdir() + for minutes, and the blocking call cannot be interrupted. The thread is + therefore abandoned rather than joined, so that an unresponsive mount delays + the caller by at most ``timeout``. + + :param call: zero-argument callable to run + :param int timeout: timeout, in seconds + :returns: whatever ``call()`` returned, or ``None`` if it did not finish in time + """ + result = [] + t = threading.Thread(target=lambda: result.append(call())) + t.daemon = True # don't delay program's exit + t.start() + t.join(timeout) + if t.is_alive(): + return None # timeout + return result[0] if result else None + + def safe_listdir(directory, timeout=60): """This is a "safe" list directory, for lazily-loaded File Systems like CVMFS. There's by default a 60 seconds timeout. .. warning:: - There is no distinction between an empty directory, and a non existent one. - It will return `[]` in both cases. + There is no distinction between an empty directory, and one that cannot + be listed (absent, or not readable). It will return `[]` in both cases. :param str directory: directory to list :param int timeout: optional timeout, in seconds. Defaults to 60. + :returns: the directory contents, or ``None`` if the listing timed out """ - def listdir(directory): + def listdir(): try: return os.listdir(directory) - except FileNotFoundError: - print(f"{directory} not found") + except OSError: + # Absent, not readable, ... : as documented, indistinguishable from empty. + # Only a timeout is reported differently, since it says nothing either way. return [] - contents = [] - t = threading.Thread(target=lambda: contents.extend(listdir(directory))) - t.daemon = True # don't delay program's exit - t.start() - t.join(timeout) - if t.is_alive(): - return None # timeout - return contents + return _callWithTimeout(listdir, timeout) + + +def safe_exists(path, timeout=60): + """This is a "safe" existence check, + for lazily-loaded File Systems like CVMFS. + There's by default a 60 seconds timeout. + + Unlike :func:`safe_listdir`, it makes no distinction between files and + directories: it answers the same question as :func:`os.path.exists`, which + reports a missing or unreadable path as ``False`` rather than raising. + + :param str path: path to check + :param int timeout: optional timeout, in seconds. Defaults to 60. + :returns: ``True``/``False``, or ``None`` if the check timed out + """ + return _callWithTimeout(lambda: os.path.exists(path), timeout) diff --git a/src/DIRAC/Core/Utilities/test/Test_ContainerImageResolver.py b/src/DIRAC/Core/Utilities/test/Test_ContainerImageResolver.py new file mode 100644 index 00000000000..7074442ca63 --- /dev/null +++ b/src/DIRAC/Core/Utilities/test/Test_ContainerImageResolver.py @@ -0,0 +1,626 @@ +"""Tests for ContainerImageResolver.""" +from pathlib import Path + +import pytest + +import DIRAC.Core.Utilities.ContainerImageResolver as cir + + +class FakeGConfig: + """Minimal gConfig stand-in, so that tests never read the process wide configuration.""" + + def __init__(self, values=None): + self.values = values or {} + + def getValue(self, key, default=None): + # Same contract as the real gConfig: an absent key yields the caller's default + return self.values.get(key, default) + + +class RecordingLog: + """Logger stand-in recording what the resolver reports.""" + + def __init__(self): + self.messages = [] + + def _record(self, level, message, variableText=""): + self.messages.append((level, f"{message} {variableText}".strip())) + + def error(self, message, variableText=""): + self._record("error", message, variableText) + + def warn(self, message, variableText=""): + self._record("warn", message, variableText) + + def debug(self, message, variableText=""): + self._record("debug", message, variableText) + + def hasMessage(self, level, fragment): + return any(level == lvl and fragment in text for lvl, text in self.messages) + + +@pytest.fixture(autouse=True) +def isolatedResolver(monkeypatch): + """Isolate every test from the process wide configuration and warning cache.""" + monkeypatch.setattr(cir, "gConfig", FakeGConfig()) + cir._emittedMessages.clear() + cir._resolvedPaths.clear() + yield + cir._emittedMessages.clear() + cir._resolvedPaths.clear() + + +@pytest.fixture +def log(): + return RecordingLog() + + +@pytest.mark.parametrize( + "arch,expected", + [ + ("x86_64", "amd64"), + ("amd64", "amd64"), + ("aarch64", "arm64"), + ("arm64", "arm64"), + # Variants are published with a colon separator: arm:v7, arm:v6, arm64:v8 + ("armv7l", "arm:v7"), + ("armv6l", "arm:v6"), + ("i686", "386"), + ("i386", "386"), + ("ppc64le", "ppc64le"), + ("s390x", "s390x"), + ("riscv64", "riscv64"), + # Unknown values are passed through lowercased: the repository publishes + # symlinks for most uname names, so they may well resolve anyway + ("weirdarch", "weirdarch"), + ("WeirdArch", "weirdarch"), + ("X86_64", "amd64"), + ], +) +def test_normalizeArch(arch, expected): + assert cir.normalizeArch(arch) == expected + + +@pytest.mark.parametrize( + "arch,expected", + [ + # DUCC publishes arm64 either plain or variant-qualified depending on the + # manifest, and only symlinks one to the other when the plain name is free + ("aarch64", ["arm64", "arm64:v8"]), + ("arm64", ["arm64", "arm64:v8"]), + ("armv7l", ["arm:v7", "arm"]), + ("armv6l", ["arm:v6", "arm"]), + ("armel", ["arm:v6", "arm"]), + ("armhf", ["arm:v7", "arm"]), + # Architectures published under a single name yield a single candidate + ("x86_64", ["amd64"]), + ("x86-64", ["amd64"]), + ("i686", ["386"]), + ("ppc64le", ["ppc64le"]), + ("weirdarch", ["weirdarch"]), + ], +) +def test_getArchCandidates(arch, expected): + assert cir.getArchCandidates(arch) == expected + + +def test_normalizeArch_is_the_preferred_candidate(): + for arch in ("aarch64", "armv7l", "x86_64", "weirdarch"): + assert cir.normalizeArch(arch) == cir.getArchCandidates(arch)[0] + + +def test_getMultiarchPaths_covers_every_candidate(): + paths = cir.getMultiarchPaths("alpine:latest", basePath="/base", arch="aarch64") + assert paths == [Path("/base/arm64/alpine:latest"), Path("/base/arm64:v8/alpine:latest")] + + +def test_explicit_arch(): + paths = cir.getMultiarchPaths( + "registry.hub.docker.com/library/alpine:latest", + basePath="/cvmfs/unpacked.cern.ch/.multiarch", + arch="x86_64", + ) + assert paths == [Path("/cvmfs/unpacked.cern.ch/.multiarch/amd64/registry.hub.docker.com/library/alpine:latest")] + + +def test_arm_variant_path(): + paths = cir.getMultiarchPaths("alpine:latest", basePath="/base", arch="armv7l") + assert paths == [Path("/base/arm:v7/alpine:latest"), Path("/base/arm/alpine:latest")] + + +def test_trailing_slash_stripped(): + paths = cir.getMultiarchPaths( + # Note the image reference ending with "/" + "registry.hub.docker.com/library/alpine:latest/", + basePath="/cvmfs/unpacked.cern.ch/.multiarch", + arch="x86_64", + ) + assert paths == [Path("/cvmfs/unpacked.cern.ch/.multiarch/amd64/registry.hub.docker.com/library/alpine:latest")] + + +def test_base_with_trailing_slash(): + paths = cir.getMultiarchPaths( + "alpine:latest", + # Note the base path ending with "/" + basePath="/cvmfs/unpacked.cern.ch/.multiarch/", + arch="x86_64", + ) + assert paths == [Path("/cvmfs/unpacked.cern.ch/.multiarch/amd64/alpine:latest")] + + +def test_absolute_image_ref_rejected(log): + """An absolute reference would make pathlib drop the base path and the architecture.""" + paths = cir.getMultiarchPaths("/cvmfs/cernvm-prod.cern.ch/cvm4", basePath="/base", arch="x86_64", log=log) + assert paths == [] + assert log.hasMessage("error", "must be a relative OCI reference") + + +def test_parent_traversal_image_ref_rejected(log): + paths = cir.getMultiarchPaths("../../etc", basePath="/base", arch="x86_64", log=log) + assert paths == [] + assert log.hasMessage("error", "must not contain '..'") + + +def test_undetectable_arch_rejected(log): + """platform.machine() returns an empty string when the architecture is unknown.""" + paths = cir.getMultiarchPaths("alpine:latest", basePath="/base", arch="", log=log) + assert paths == [] + assert log.hasMessage("error", "Cannot determine the machine architecture") + + +def _make_multiarch_image(tmp_path, arch, image_ref): + """Helper to create a fake multiarch image directory.""" + image_dir = tmp_path / ".multiarch" / arch / image_ref + image_dir.mkdir(parents=True) + return image_dir + + +def _make_legacy_root(tmp_path, name="cvm4"): + """Helper to create a fake legacy container root.""" + legacy = tmp_path / name + legacy.mkdir(parents=True) + return legacy + + +def test_multiarch_found_as_directory(tmp_path): + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + arch="x86_64", + ) + assert result == expected + + +def test_multiarch_found_aarch64(tmp_path): + expected = _make_multiarch_image(tmp_path, "arm64", "registry.hub.docker.com/library/alma9:latest") + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + arch="aarch64", + ) + assert result == expected + + +def test_multiarch_found_under_arm64_variant(tmp_path): + """An image published only as arm64:v8 must still be found on an aarch64 node. + + DUCC only symlinks .multiarch/arm64 to arm64:v8 when arm64 does not already + exist as a real directory, so on a repository holding both the variant form + is reachable under that name alone. + """ + (tmp_path / ".multiarch" / "arm64").mkdir(parents=True) # real directory, without this image + expected = _make_multiarch_image(tmp_path, "arm64:v8", "registry.hub.docker.com/library/alma9:latest") + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + arch="aarch64", + ) + assert result == expected + + +def test_multiarch_plain_arm64_preferred_over_variant(tmp_path): + """When the image exists under both names, the plain one wins, as DUCC prefers it.""" + expected = _make_multiarch_image(tmp_path, "arm64", "registry.hub.docker.com/library/alma9:latest") + _make_multiarch_image(tmp_path, "arm64:v8", "registry.hub.docker.com/library/alma9:latest") + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + arch="aarch64", + ) + assert result == expected + + +def test_multiarch_arm_variant_falls_back_to_plain_arm(tmp_path): + """An image published as plain "arm" must be found on an armv7l node.""" + expected = _make_multiarch_image(tmp_path, "arm", "registry.hub.docker.com/library/alma9:latest") + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + arch="armv7l", + ) + assert result == expected + + +def test_fallback_warning_names_every_candidate(tmp_path, log): + """The operator must be able to see exactly which paths were looked for.""" + legacy = _make_legacy_root(tmp_path) + + cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="aarch64", + log=log, + ) + assert log.hasMessage("warn", "arm64/nonexistent:latest") + assert log.hasMessage("warn", "arm64:v8/nonexistent:latest") + + +def test_fallback_warning_reports_an_unknown_variant_directory(tmp_path, log): + """A variant directory this release does not know about must be named, not silently missed.""" + _make_multiarch_image(tmp_path, "arm64:v9", "registry.hub.docker.com/library/alma9:latest") + legacy = _make_legacy_root(tmp_path) + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="aarch64", + log=log, + ) + # An unknown variant is reported, never used: variant compatibility is + # directional, so we cannot know it runs here + assert result is None + assert log.hasMessage("warn", "The repository publishes arm64:v9 for architecture 'arm64'") + + +def test_fallback_warning_reports_nothing_published(tmp_path, log): + _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + legacy = _make_legacy_root(tmp_path) + + cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="aarch64", + log=log, + ) + assert log.hasMessage("warn", "The repository publishes nothing for architecture 'arm64'") + + +def test_base_path_not_listed_when_the_image_is_found(tmp_path, monkeypatch): + """The diagnostic listing must cost nothing on the path where the image resolves.""" + calls = [] + monkeypatch.setattr(cir, "safe_listdir", lambda *args, **kwargs: calls.append(args) or []) + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + arch="x86_64", + ) + assert result == expected + assert calls == [] + + +def test_published_dirs_are_reported_per_goarch(tmp_path, log): + """An ARMv7 node is told about every arm directory, and about no other architecture.""" + base = tmp_path / ".multiarch" + for name in ("arm", "arm:v6", "arm64", "amd64"): + (base / name).mkdir(parents=True) + legacy = _make_legacy_root(tmp_path) + + cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(base), + containerRoot=str(legacy), + arch="armv7l", + log=log, + ) + # "arm" and "arm:v6" share the GOARCH part, "arm64" and "amd64" do not + assert log.hasMessage("warn", "The repository publishes arm, arm:v6 for architecture 'arm:v7'") + + +def test_successful_resolution_is_cached(tmp_path, monkeypatch): + """The resolver runs once per payload: a pilot must not stat CVMFS every time.""" + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + kwargs = dict( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + arch="x86_64", + ) + assert cir.resolveImagePath(**kwargs) == expected + + def boom(*args, **kwargs): + raise AssertionError("the filesystem must not be touched again") + + monkeypatch.setattr(cir, "safe_exists", boom) + monkeypatch.setattr(cir, "safe_listdir", boom) + assert cir.resolveImagePath(**kwargs) == expected + + +def test_failed_resolution_is_not_cached(tmp_path): + """A miss may be a transient mount problem, so it must not stick for the pilot's life.""" + kwargs = dict( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + arch="x86_64", + defaultRoot=None, + ) + assert cir.resolveImagePath(**kwargs) is None + + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + assert cir.resolveImagePath(**kwargs) == expected + + +def test_cache_distinguishes_configurations(tmp_path): + """Two CEs with different image references must not share a cached answer.""" + alma = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + ubuntu = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/ubuntu:24.04") + base = str(tmp_path / ".multiarch") + + assert ( + cir.resolveImagePath(imageRef="registry.hub.docker.com/library/alma9:latest", basePath=base, arch="x86_64") + == alma + ) + assert ( + cir.resolveImagePath(imageRef="registry.hub.docker.com/library/ubuntu:24.04", basePath=base, arch="x86_64") + == ubuntu + ) + + +def test_multiarch_preferred_over_legacy(tmp_path): + """When both multiarch and legacy exist, multiarch wins.""" + multiarch = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/alma9:latest") + legacy = _make_legacy_root(tmp_path) + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/alma9:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="x86_64", + ) + assert result == multiarch + + +def test_multiarch_not_found_falls_back_to_container_root(tmp_path, log): + legacy = _make_legacy_root(tmp_path) + + result = cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="x86_64", + log=log, + ) + assert result == legacy + assert log.hasMessage("warn", "Falling back to legacy ContainerRoot") + + +def test_invalid_image_ref_falls_back_to_container_root(tmp_path, log): + """An unusable ImageReference must not be silently resolved to itself.""" + legacy = _make_legacy_root(tmp_path) + + result = cir.resolveImagePath( + imageRef=str(legacy), # absolute: not a valid OCI reference + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="x86_64", + log=log, + ) + assert result == legacy + assert log.hasMessage("error", "must be a relative OCI reference") + assert log.hasMessage("warn", "Falling back to legacy ContainerRoot") + + +def test_legacy_on_foreign_arch_is_refused(tmp_path, log): + """The legacy roots are amd64 images: an arm64 node must fail, not start a doomed payload.""" + legacy = _make_legacy_root(tmp_path) + + result = cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="aarch64", + log=log, + ) + assert result is None + assert log.hasMessage("error", "Legacy ContainerRoot cannot run on this node") + # ...and the log must not claim a fallback that did not happen + assert not log.hasMessage("warn", "Falling back to legacy ContainerRoot") + + +def test_refusal_is_logged_for_every_payload(tmp_path, log): + """A condition that kills every payload must not be deduplicated away after the first.""" + legacy = _make_legacy_root(tmp_path) + + for _ in range(3): + cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="aarch64", + log=log, + ) + refusals = [text for lvl, text in log.messages if lvl == "error" and "cannot run on this node" in text] + assert len(refusals) == 3 + + +def test_legacy_on_unrecognised_arch_is_warned_but_used(tmp_path, log): + """An unknown uname may still be a legacy-compatible node: warn, do not refuse.""" + legacy = _make_legacy_root(tmp_path) + + result = cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="weirdarch", + log=log, + ) + assert result == legacy + assert log.hasMessage("warn", "unrecognised architecture 'weirdarch'") + + +def test_legacy_on_native_arch_is_not_reported(tmp_path, log): + legacy = _make_legacy_root(tmp_path) + + cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="x86_64", + log=log, + ) + assert not log.hasMessage("warn", "single architecture image but this node is") + + +def test_multiarch_not_found_falls_back_to_config_container_root(tmp_path, monkeypatch, log): + legacy = _make_legacy_root(tmp_path) + monkeypatch.setattr(cir, "gConfig", FakeGConfig({"/Resources/Computing/Singularity/ContainerRoot": str(legacy)})) + + result = cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(tmp_path / ".multiarch"), + arch="x86_64", + log=log, + ) + assert result == legacy + + +def test_no_image_ref_skips_multiarch_uses_legacy_with_deprecation(tmp_path, log): + """When no ImageReference is configured, legacy works but is reported as deprecated.""" + legacy = _make_legacy_root(tmp_path) + + result = cir.resolveImagePath(containerRoot=str(legacy), arch="x86_64", log=log) + assert result == legacy + assert log.hasMessage("warn", "ContainerRoot is deprecated") + + +def test_deprecation_is_emitted_once_per_process(tmp_path, log): + """The resolver runs on every payload: it must not spam the pilot log.""" + legacy = _make_legacy_root(tmp_path) + + cir.resolveImagePath(containerRoot=str(legacy), arch="x86_64", log=log) + firstCount = len(log.messages) + assert firstCount + + cir.resolveImagePath(containerRoot=str(legacy), arch="x86_64", log=log) + assert len(log.messages) == firstCount + + +def test_no_image_ref_no_legacy_returns_none(tmp_path, log): + """When nothing is configured and defaults don't exist, return None.""" + result = cir.resolveImagePath( + containerRoot=str(tmp_path / "no_such_root"), + arch="x86_64", + log=log, + ) + assert result is None + assert log.hasMessage("error", "No container image could be resolved") + + +def test_default_root_can_be_disabled(tmp_path): + """dirac-apptainer-exec must not silently land on the built-in CernVM image.""" + result = cir.resolveImagePath(arch="x86_64", defaultRoot=None) + assert result is None + + +def test_config_provides_image_ref_and_base(tmp_path, monkeypatch): + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/centos:7") + monkeypatch.setattr( + cir, + "gConfig", + FakeGConfig( + { + "/Resources/Computing/Singularity/ImageBasePath": str(tmp_path / ".multiarch"), + "/Resources/Computing/Singularity/ImageReference": "registry.hub.docker.com/library/centos:7", + } + ), + ) + + result = cir.resolveImagePath(arch="x86_64") + assert result == expected + + +def test_explicit_args_override_config(tmp_path, monkeypatch): + expected = _make_multiarch_image(tmp_path, "amd64", "registry.hub.docker.com/library/myimage:v1") + monkeypatch.setattr( + cir, + "gConfig", + FakeGConfig( + { + "/Resources/Computing/Singularity/ImageBasePath": "/should/not/be/used", + "/Resources/Computing/Singularity/ImageReference": "should_not_be_used:latest", + } + ), + ) + + result = cir.resolveImagePath( + imageRef="registry.hub.docker.com/library/myimage:v1", + basePath=str(tmp_path / ".multiarch"), + arch="x86_64", + ) + assert result == expected + + +def test_nothing_found_returns_none(tmp_path): + result = cir.resolveImagePath( + imageRef="nonexistent:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(tmp_path / "no_such_root"), + arch="x86_64", + ) + assert result is None + + +def test_unresponsive_filesystem_does_not_block(tmp_path, monkeypatch, log): + """A hanging CVMFS mount must not hang the job slot: safe_exists times out.""" + legacy = _make_legacy_root(tmp_path) + monkeypatch.setattr(cir, "safe_exists", lambda path, timeout=None: None) + + result = cir.resolveImagePath( + imageRef="alpine:latest", + basePath=str(tmp_path / ".multiarch"), + containerRoot=str(legacy), + arch="x86_64", + log=log, + ) + assert result is None + assert log.hasMessage("warn", "Timed out") + + +def test_findMultiarchImage_found(tmp_path): + expected = _make_multiarch_image(tmp_path, "amd64", "alpine:latest") + + result = cir.findMultiarchImage("alpine:latest", basePath=str(tmp_path / ".multiarch"), arch="x86_64") + assert result == expected + + +def test_findMultiarchImage_not_found(tmp_path): + result = cir.findMultiarchImage("nonexistent:latest", basePath=str(tmp_path / ".multiarch"), arch="x86_64") + assert result is None + + +def test_findMultiarchImage_never_falls_back_to_legacy(tmp_path, monkeypatch): + """dirac-apptainer-exec -i must never silently run a different image.""" + legacy = _make_legacy_root(tmp_path) + monkeypatch.setattr(cir, "gConfig", FakeGConfig({"/Resources/Computing/Singularity/ContainerRoot": str(legacy)})) + + result = cir.findMultiarchImage("nonexistent:latest", basePath=str(tmp_path / ".multiarch"), arch="x86_64") + assert result is None + + +def test_findMultiarchImage_uses_configured_base_path(tmp_path, monkeypatch): + expected = _make_multiarch_image(tmp_path, "amd64", "alpine:latest") + monkeypatch.setattr( + cir, "gConfig", FakeGConfig({"/Resources/Computing/Singularity/ImageBasePath": str(tmp_path / ".multiarch")}) + ) + + result = cir.findMultiarchImage("alpine:latest", arch="x86_64") + assert result == expected diff --git a/src/DIRAC/Core/scripts/dirac_apptainer_exec.py b/src/DIRAC/Core/scripts/dirac_apptainer_exec.py index 0fe357ed40b..639931f78ef 100644 --- a/src/DIRAC/Core/scripts/dirac_apptainer_exec.py +++ b/src/DIRAC/Core/scripts/dirac_apptainer_exec.py @@ -9,7 +9,8 @@ from DIRAC import S_ERROR, gConfig, gLogger from DIRAC.Core.Base.Script import Script from DIRAC.Core.Security.Locations import getCAsLocation, getProxyLocation, getVOMSLocation -from DIRAC.Core.Utilities.Os import safe_listdir +from DIRAC.Core.Utilities.ContainerImageResolver import EXISTS_TIMEOUT, findMultiarchImage, resolveImagePath +from DIRAC.Core.Utilities.Os import safe_exists, safe_listdir from DIRAC.Core.Utilities.Subprocess import systemCall @@ -37,15 +38,12 @@ def generate_container_wrapper(dirac_env_var, diracos_env_var, etc_dir, rc_scrip return "\n".join(lines) -CONTAINER_DEFROOT = "" # Should add something like "/cvmfs/dirac.egi.eu/container/apptainer/alma9/x86_64" - - @Script() def main(): command = sys.argv[1] user_image = None - Script.registerSwitch("i:", "image=", " apptainer image to use") + Script.registerSwitch("i:", "image=", " Container image: local path or OCI reference") Script.parseCommandLine(ignoreErrors=False) for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "i" or switch[0].lower() == "image": @@ -70,7 +68,24 @@ def main(): # Script may include credentials, make sure other users can't read it os.chmod("dirac_container.sh", 0o700) - # Now let's construct the apptainer command + # Resolve the container image. As before the multiarch layout, a value given + # with -i/--image is used on its own: an existing local path is taken as-is, + # and the configured ContainerRoot is not considered. Looking the value up as + # an OCI reference is new, and only happens where the command used to fail. + if user_image: + if safe_exists(user_image, timeout=EXISTS_TIMEOUT): + image_path = Path(user_image) + else: + image_path = findMultiarchImage(user_image) + else: + # No built-in default here: an unconfigured node must fail loudly rather + # than silently running the payload in whatever image happens to be on CVMFS + image_path = resolveImagePath(defaultRoot=None) + if not image_path: + gLogger.error("Apptainer image to exec not found:", user_image or "(from the configuration)") + return S_ERROR("Failed to find Apptainer image to exec") + + # Build the apptainer command cmd = ["apptainer", "exec"] cmd.extend(["--contain"]) # use minimal /dev and empty other directories (e.g. /tmp and $HOME) cmd.extend(["--ipc"]) # run container in a new IPC namespace @@ -93,14 +108,7 @@ def main(): gLogger.warn(f"Bind path {bind_path} does not exist, skipping") cmd.extend(["--cwd", cwd]) # set working directory - rootImage = user_image or gConfig.getValue("/Resources/Computing/Singularity/ContainerRoot") or CONTAINER_DEFROOT - - if os.path.isdir(rootImage) or os.path.isfile(rootImage): - cmd.extend([rootImage, f"{cwd}/dirac_container.sh"]) - else: - # if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS) - gLogger.error("Apptainer image to exec not found: ", rootImage) - return S_ERROR("Failed to find Apptainer image to exec") + cmd.extend([str(image_path), f"{cwd}/dirac_container.sh"]) gLogger.debug(f"Execute Apptainer command: {' '.join(cmd)}") result = systemCall(0, cmd) diff --git a/src/DIRAC/Resources/Computing/SingularityComputingElement.py b/src/DIRAC/Resources/Computing/SingularityComputingElement.py index b40e7112638..5a067fdb6b2 100644 --- a/src/DIRAC/Resources/Computing/SingularityComputingElement.py +++ b/src/DIRAC/Resources/Computing/SingularityComputingElement.py @@ -3,8 +3,8 @@ A computing element class using singularity containers, where Singularity is supposed to be found on the WN. -The goal of this CE is to start the job in the container set by -the "ContainerRoot" config option. +The goal of this CE is to start the job in a container image resolved by +:py:func:`~DIRAC.Core.Utilities.ContainerImageResolver.resolveImagePath`. DIRAC can be re-installed within the container. @@ -24,13 +24,12 @@ from DIRAC import S_ERROR, S_OK, gConfig, gLogger from DIRAC.ConfigurationSystem.Client.Helpers import Operations from DIRAC.Core.Utilities.CGroups2 import CG2Manager +from DIRAC.Core.Utilities.ContainerImageResolver import resolveImagePath from DIRAC.Core.Utilities.ThreadScheduler import gThreadScheduler from DIRAC.Resources.Computing.ComputingElement import ComputingElement from DIRAC.Resources.Storage.StorageElement import StorageElement from DIRAC.WorkloadManagementSystem.Utilities.Utils import createJobWrapper -# Default container to use if it isn't specified in the CE options -CONTAINER_DEFROOT = "/cvmfs/cernvm-prod.cern.ch/cvm4" CONTAINER_WORKDIR = "DIRAC_containers" CONTAINER_INNERDIR = "/tmp" # nosec: B108 # /tmp dir to use in the container @@ -105,9 +104,6 @@ def __init__(self, ceUniqueID): super().__init__(ceUniqueID) self.__submittedJobs = 0 self.__runningJobs = 0 - self.__root = CONTAINER_DEFROOT - if "ContainerRoot" in self.ceParameters: - self.__root = self.ceParameters["ContainerRoot"] self.__workdir = CONTAINER_WORKDIR self.__innerdir = CONTAINER_INNERDIR self.__installDIRACInContainer = self.ceParameters.get("InstallDIRACInContainer", False) @@ -314,7 +310,18 @@ def submitJob(self, executableFile, proxy=None, **kwargs): :return: S_OK(payload exit code) / S_ERROR() if submission issue """ - rootImage = self.__root + # Read at submit time, like every other container option below: setParameters() + # and PoolComputingElement both update ceParameters after the CE is built + imagePath = resolveImagePath( + imageRef=self.ceParameters.get("ImageReference"), + basePath=self.ceParameters.get("ImageBasePath"), + containerRoot=self.ceParameters.get("ContainerRoot"), + log=self.log, + ) + if not imagePath: + # The resolver already logged the architecture and every path it tried + return S_ERROR("Failed to find singularity image to exec") + renewTask = None self.log.info("Creating singularity container") @@ -418,11 +425,8 @@ def submitJob(self, executableFile, proxy=None, **kwargs): containerOpts = self.ceParameters["ContainerOptions"].split(",") for opt in containerOpts: outerCmd.extend([opt.strip()]) - if not (os.path.isdir(rootImage) or os.path.isfile(rootImage)): - # if we are here is because there's no image, or it is not accessible (e.g. not on CVMFS) - self.log.error("Singularity image to exec not found: ", rootImage) - return S_ERROR("Failed to find singularity image to exec") - outerCmd.append(rootImage) + + outerCmd.append(str(imagePath)) cmd = outerCmd + [innerCmd] self.log.debug(f"Execute singularity command: {cmd}") diff --git a/tests/Integration/Resources/Computing/Test_SingularityCE.py b/tests/Integration/Resources/Computing/Test_SingularityCE.py index 712ba8641a0..76a8c9f2506 100644 --- a/tests/Integration/Resources/Computing/Test_SingularityCE.py +++ b/tests/Integration/Resources/Computing/Test_SingularityCE.py @@ -31,10 +31,10 @@ def test_submitJob(): ce = SingularityComputingElement("SingularityComputingElement") res = ce.submitJob("testJob.py", None) - assert res["OK"] is False + assert res["OK"] is False # This is False because the image can't be found res = ce.getCEStatus() assert res["OK"] is True - assert res["SubmittedJobs"] == 1 + _stopJob(1) for ff in ["testJob.py", "pilot.json"]: if os.path.isfile(ff): @@ -79,7 +79,6 @@ def test_submitJobWrapper(): res = ce.getCEStatus() assert res["OK"] is True - assert res["SubmittedJobs"] == 1 _stopJob(2) for ff in ["testJob.py", "stop_job_2", "job.info", "std.out", "pilot.json"]: