diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 9952af1..22984c0 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -17,7 +17,7 @@ include: .images_matrix: - DOCKERFILE_RELPATH: "geolab-base" - DOCKER_EXTRA_OPTIONS: "--build-arg IMAGE_AUTHORS=geolab@earthscope.org --build-arg PYTHON_VERSION=3.12" + DOCKER_EXTRA_OPTIONS: "--build-arg IMAGE_AUTHORS=geolab@earthscope.org --build-arg PYTHON_VERSION=3.12 --build-arg GEOLAB_VERSION=${IMAGE_VERSION}" #- DOCKERFILE_RELPATH: "geolab-gpu" @@ -25,4 +25,4 @@ variables: CONTAINER_REGISTRY_PLATFORM: "AWS-PUB" DOCKERFILE_RELPATH_IS_IMAGE_NAME: "true" GITLAB_HOSTED_RUNNER_SIZE: "saas-linux-medium-amd64" - + GEOLAB_VERSION: "" \ No newline at end of file diff --git a/geolab-base/CHANGELOG.md b/geolab-base/CHANGELOG.md new file mode 100644 index 0000000..98486de --- /dev/null +++ b/geolab-base/CHANGELOG.md @@ -0,0 +1,22 @@ +# Changelog + +All notable changes to this project will be documented in this file. + +## [0.2.0] + +### Added + +- pin base image to pangeo-base:04bb14b +- add geolab-base version as ENV GEOLAB_VERSION (set during CI build) +- add graphviz package +- add pygraphviz package +- add ipycytoscape +- added tests for new packages +- add build_process.png for README + +### Changed + +- removed test_packages.py +- moved test_notebook functions to test_helpers.py module to make it easier for users to import when writing tests +- updated test_notebook.ipynb to use test_helpers functions +- updated README to match Building Custom Images in docs diff --git a/geolab-base/Dockerfile b/geolab-base/Dockerfile index 14d9039..0394f79 100644 --- a/geolab-base/Dockerfile +++ b/geolab-base/Dockerfile @@ -11,21 +11,28 @@ # docker build --platform linux/amd64 \ # --build-arg IMAGE_TITLE=my-geolab-image \ # --build-arg IMAGE_AUTHORS=you@university.edu \ +# --build-arg GEOLAB_VERSION=1.0.0 \ # -t my-geolab-image . # ────────────────────────────────────────────────────────────── -FROM pangeo/base-image:latest +# Changing this tag is a major (breaking) change under semantic versioning. +FROM pangeo/base-image:04bb14b ARG IMAGE_TITLE="custom-geolab-image" # default if user passes nothing ARG IMAGE_AUTHORS="NoSpecifiedAuthors" # default if user passes nothing +ARG GEOLAB_VERSION # no default; must be passed with --build-arg + +RUN test -n "$GEOLAB_VERSION" || (echo "GEOLAB_VERSION must not be empty" >&2 && exit 1) LABEL org.opencontainers.image.title="${IMAGE_TITLE}" \ - org.opencontainers.image.authors="${IMAGE_AUTHORS}" + org.opencontainers.image.authors="${IMAGE_AUTHORS}" \ + org.opencontainers.image.version="${GEOLAB_VERSION}" # Set locations of PROJ/GDAL resource directories ENV PROJ_DATA=/srv/conda/envs/notebook/share/proj \ PROJ_LIB=/srv/conda/envs/notebook/share/proj \ - GDAL_DATA=/srv/conda/envs/notebook/share/gdal + GDAL_DATA=/srv/conda/envs/notebook/share/gdal \ + GEOLAB_VERSION=${GEOLAB_VERSION} # Default command for standalone use. JupyterHub spawning passes # its own command (jupyterhub-singleuser), which start respects. diff --git a/geolab-base/README.md b/geolab-base/README.md index 7a6d5b1..a55cbf2 100644 --- a/geolab-base/README.md +++ b/geolab-base/README.md @@ -1,168 +1,154 @@ # Building Custom GeoLab Images -GeoLab environments run as **containers** based on **images** — self-contained packages that include an operating system, software libraries, Python packages, and more. This guide documents the steps using Docker and git to create such an image. +A GeoLab image is a complete, prepackaged computing environment that runs in JupyterLab, bundling common geophysics Python and scientific packages together. Starting a GeoLab session launches an image. This guide walks through building an image with a customized environment, using Docker and git. Steps: -- [Installing and logging into Docker](#installing-and-logging-into-docker) -- [Installing git](#installing-git) -- [Copy the base template](#copy-the-base-template) -- [Configuring a custom image](#configuring-a-custom-image) - - [Installing system software with apt](#installing-system-software-with-apt) - - [Installing Conda packages](#installing-conda-packages) - - [Installing pip packages](#installing-pip-packages) - - [Creating a postBuild script](#creating-a-postbuild-script) -- [Building and testing the image locally](#building-and-testing-the-image-locally) - - [Building the local testing image](#building-the-local-testing-image) - - [Running the local testing image](#running-the-local-testing-image) +- [How It Works](#how-it-works) +- [Before Starting](#before-starting) +- [Step 1: Get the Template](#step-1-get-the-template) +- [Step 2: Add Your Packages](#step-2-add-your-packages) + - [environment.yml: Your Main Package List](#environmentyml-your-main-package-list) + - [requirements.txt: Packages Only on PyPI](#requirementstxt-packages-only-on-pypi) + - [apt.txt: System Software (Rarely Needed)](#apttxt-system-software-rarely-needed) + - [postBuild: One-Time Setup Commands](#postbuild-one-time-setup-commands) +- [Step 3: Build and Test Locally](#step-3-build-and-test-locally) - [Verifying the installed packages](#verifying-the-installed-packages) -- [Building and publishing the image](#building-and-publishing-the-image) - - [Building the platform image](#building-the-platform-image) - - [Publishing the platform image](#publishing-the-platform-image) -- [Running your published image in GeoLab](#running-your-published-image-in-geolab) +- [Step 4: Publish Your Image](#step-4-publish-your-image) + - [Rebuild for GeoLab's platform](#rebuild-for-geolabs-platform) + - [Push the image](#push-the-image) + - [Publishing to GitHub or AWS Image Repositories](#publishing-to-github-or-aws-image-repositories) +- [Step 5: Launch It in GeoLab](#step-5-launch-it-in-geolab) +- [Making Changes Later](#making-changes-later) +- [Troubleshooting Package Installation](#troubleshooting-package-installation) +- [Quick Reference](#quick-reference) +- [Getting a Personal Access Token (for GHCR)](#getting-a-personal-access-token-for-ghcr) > [!NOTE] -> These instructions are written for macOS, Linux and similar systems. While the same steps can be executed in Windows the details will vary. +> These instructions are written for macOS, Linux and similar systems. While the same steps can be executed on Windows the details will vary. --- -## Installing and logging into Docker +## How It Works -Docker Desktop is the recommended option to build and run images on your computer and to publish them for access by others (and GeoLab itself). Follow the [instructions](https://docs.docker.com/get-started/introduction/) to install it. If you are new to Docker, images, and containers, we recommend working through the [getting-started modules](https://docs.docker.com/get-started/introduction/#modules) to learn how to build, run, and publish images. +Think of an **image** as a recipe, with each Python package as an ingredient. Building the image is cooking the meal from the recipe. A **container** is that meal served on a plate. The recipe doesn't change and you can make the same meal over and over, GeoLab does the same thing, launching a fresh container from your image every time. -> [!TIP] -> Alternatively, Docker Engine (with build plugins) can be used; this is what is often installed on Linux systems. +The `geolab-base` template (used in the steps below) is based on a Pangeo image (`pangeo/base-image`) as its starting point; a custom image is created by modifying the build on top of it. -After starting Docker Desktop, log into Docker, creating an account if needed. This enables pushing (aka publishing) an image to Docker Hub, Docker’s image repository where it is available to others. +Install Python packages in an image by editing plaintext files that list the required software. `Docker` reads those files and builds the image. The image must then be published in an image repository so GeoLab can access it: -> [!TIP] -> Docker Hub is just one of many repositories for images, many others exist and can be used, but Docker Hub is easiest because it is the default repository for components in the ecosystem, including in GeoLab. +![Edit config files, Docker builds, Image, Push image to repository, GeoLab runs it](./build_process.png) --- -## Installing git - -Git is needed to make a copy of the GeoLab repository containing the template for the base image for building custom images. The git program is often already installed, or easily installed on macOS and Linux systems. Follow the [instructions](https://github.com/git-guides/install-git) to check for and install git if needed. +## Before Starting ---- - -## Copy the base template +Two pieces of software must be installed on **your computer**: -The `geolab-base` directory in the GeoLab repository contains all the files needed to build a custom GeoLab image. Open a terminal and execute the following commands: +1. **Docker Desktop.** Download it at [docker.com](https://www.docker.com/products/docker-desktop/), install it, and leave it running in the background. -```shell -cd ~ -git clone --depth 1 https://github.com/EarthScope/GeoLab.git -cp -R GeoLab/geolab-base my-geolab-image -cd my-geolab-image -``` + > [!TIP] + > If you're new to Docker, images, and containers, work through the [getting-started modules](https://docs.docker.com/get-started/introduction/#modules) to learn how to build, run, and publish images. Alternatively, Docker Engine (with build plugins) can be used instead of Docker Desktop; this is what is often installed on Linux systems. -This set of commands does the following: + After starting Docker Desktop, log into Docker, creating an account if needed. This is what enables pushing (publishing) an image to Docker Hub, Docker's image repository, so it's available to others (and to GeoLab itself). -1. Change from the current directory to your home directory. -2. Use git to copy the GeoLab repository, only getting the current state (`--depth 1`) -3. Copy the geolab-base directory to a new directory in your home directory. -4. Change into the newly created directory with a copy of the image template files. +2. **Git client**, to download the GeoLab Dockerfile template. Use the operating system's package manager to install one, or follow the [instructions](https://github.com/git-guides/install-git) to check for and install git if needed. -Your `my-geolab-image` directory should have the following files: +Verify Docker and git are installed and working by opening a terminal and running: -```shell -./my-geolab-image -├── apt.txt -├── README.md -├── Dockerfile -├── environment.yml -├── requirements.txt -├── start -├── test_notebook.ipynb -├── test_packages.py -└── ... +``` +docker --version +git --version ``` -> [!TIP] -> For image development it is recommended to keep your files in a git repository to track changes, share with others, etc. The sooner this is started the better. This is the right stage to commit these starting files to a new repo. - ---- - -## Configuring a custom image - -A container image is a snapshot of a complete computing environment. When GeoLab launches, it starts a container from such an image. The `geolab-base` image is based on a Pangeo image (`pangeo/base-image`) as its starting point. A custom image can be created by modifying the build of this image. - -The `Dockerfile` specifies how the image is built. To create a custom image, edit the configuration files before building: +If they print a version number, they are installed and working. +In addition to the required software, a **GitHub** account at [github.com](https://github.com), a Docker account, or an AWS account is needed for publishing the image and making it available to GeoLab. -| File | What it controls | -| ------------------ | ------------------------------------------------------------ | -| `apt.txt` | System-level software (installed via `apt`) | -| `environment.yml` | Python from Conda packages and channels | -| `requirements.txt` | Python packages from PyPI (installed via `pip`) | -| `postBuild` | Commands to run after the build completes (create if needed) | -| `start` | Entrypoint script; normally leave unchanged | +--- +## Step 1: Get the Template -### Installing system software with apt +EarthScope provides a starter template. Download it using git to set up a working folder: -`apt` is the Ubuntu package manager — it installs system-level tools like compilers, runtime libraries, and command-line utilities. Edit `apt.txt` to add any packages you need, one per line. Best practice is to list packages in alphabetical order, which makes it easier to find a specific package. +```shell +git clone --depth 1 https://github.com/EarthScope/GeoLab.git +cp -R GeoLab/geolab-base my-geolab-image +cd my-geolab-image +``` -**Example:** Adding Node.js (nodejs) and npm to `apt.txt`: +The `my-geolab-image` folder contains these files: -```shell -build-essential -gfortran -git -gmt-dcw -gmt-gshhg -make -nodejs <-- NEW -npm <-- NEW +``` +my-geolab-image/ +├── Dockerfile ← do not edit this +├── environment.yml ← add your conda packages here +├── requirements.txt ← add PyPI-only packages here +├── apt.txt ← add system software here (rarely needed) +├── start ← do not edit this +├── test_helpers.py ← Python module with testing functions for packages +└── test_notebook.ipynb ← interactive version of the smoke test ``` +> [!NOTE] +> The only files to edit are `environment.yml`, `requirements.txt`, and `apt.txt` (plus an optional `postBuild` script, covered below). Everything else is set up for you. + > [!TIP] -> Only add packages here that aren't available through conda. Most scientific Python libraries are better managed in environment.yml. +> For image development it is recommended to keep your files in a git repository to track changes, share with others, etc. The sooner this is started the better, this is the right stage to commit these starting files to a new repo. + +--- -### Installing Conda packages +## Step 2: Add Your Packages -Conda manages Python (and non-Python) packages within isolated environments. Edit `environment.yml` to add packages by name under the appropriate section (the sections are comments, not important for conda). Always use the `conda-forge` channel for the broadest package availability unless otherwise specified in the package’s installation instructions. +### environment.yml: Your Main Package List -**Example:** Adding SimPEG (`simpeg`) to the Geophysics section of `environment.yml`: +Add conda Python packages from `conda-forge` here. Open the file and add packages under the `dependencies` section: ```yaml -... +channels: + - conda-forge + - nodefaults dependencies: - ... - # ── Geophysics ────────────────────────────────────── - - dascore + - python=3.12 + # --- Geophysics --- - obspy - - obsplus - - gmt - pygmt - - simpeg <-- NEW - ... + # --- Geospatial --- + - cartopy + - geopandas + # add your packages below: + - my-package-name ``` -> [!TIP] -> Prefer conda packages over pip when a package is available in both. Conda resolves environment-wide dependencies more reliably. - -### Installing pip packages +Conda packages are preferred, because conda checks that everything works together before installing and reduces the possibility of dependency conflicts among packages. -Some packages are only available on PyPI (Python's package index) and can be installed with `pip`. Add them to `requirements.txt`, one per line. You can pin a specific version with `==` to ensure reproducibility. +### requirements.txt: Packages Only on PyPI -**Example:** Adding `gnss-lib-py`: +Some packages aren't available through conda-forge and must be installed from PyPI. Add them here, one per line. You can pin a specific version with `==` to ensure reproducibility: -```shell -# --- EarthScope --- +``` earthscope-sdk==1.4.1 -earthscope-cli==1.2.0 -earthscopestraintools -gnss-lib-py <-- NEW +seisbench +``` + +> [!TIP] +> Pin versions for packages critical to your workflow (e.g., `earthscope-sdk==1.4.1`). This prevents silent breakage from upstream releases if you rebuild the image later. + +### apt.txt: System Software (Rarely Needed) + +Most scientific packages go in `environment.yml`. Only use `apt.txt` for low-level system tools that can't be installed any other way: + +``` +build-essential +git ``` > [!TIP] -> Pin versions for packages critical to your workflow (e.g., `earthscope-sdk==1.4.1`). This prevents silent breakage when upstream packages release updates if you rebuild the image. +> Only add packages here that aren't available through conda. Most scientific Python libraries are better managed in `environment.yml`. -### Creating a postBuild script +### postBuild: One-Time Setup Commands -A postBuild script runs automatically after all packages are installed. Use it for one-time setup steps that can't be expressed as package installs — for example, configuring tools, downloading data files, or logging build metadata. +A `postBuild` script (create the file if you need one) runs automatically after all packages are installed. Use it for one-time setup steps that can't be expressed as package installs, for example, configuring tools, downloading data files, or logging build metadata. **Example:** Create a `postBuild` file to record the build timestamp: @@ -180,64 +166,78 @@ echo "Build stage completed successfully." --- -## Building and testing the image locally +## Step 3: Build and Test Locally -Building and testing the image locally is the fastest way to iterate on changes and fix issues. Services that are only available in GeoLab such as direct access to repositories in S3 storage cannot be tested locally. +Building and testing the image locally is the fastest way to iterate on changes and fix issues. Services that are only available in GeoLab, such as direct access to repositories in S3 storage, cannot be tested locally. -### Building the local testing image +**Build the image:** ```shell docker build -f Dockerfile --tag my-geolab-image:0.1.0 . ``` -You may omit the `:0.1.0` part of the tag if you wish. +This reads the config files and assembles the image; it can take several minutes the first time. You may omit the `:0.1.0` part of the tag if you wish. > [!TIP] -> Do not publish and try to run this image in GeoLab, it may not be the correct platform. See the next section for instructions to build the platform image. +> Do not publish and try to run this image in GeoLab, it may not be the correct platform. See [Step 4](#step-4-publish-your-image) for building the platform image. -### Running the local testing image - -Use Docker to run the image with this command: +**Run it locally:** ```shell docker run --rm -p 8888:8888 my-geolab-image:0.1.0 ``` -Copy the URL from the log that looks like: `http://127.0.0.1:8888/lab?token...` (with the token value) and connect to the container with a web browser. +The `--rm` flag deletes the container created from the image after the run completes, keeping repeated commands from piling up containers. The `-p 8888:8888` option maps the container's network port so your computer can reach it. -What is the docker run command doing? The `--rm` flag will delete the container created from the image after the run completes, keeping repeated commands from creating a new container on each run. The `-p 8888:8888` option maps the network port for the service in the container so the local computer can reach it. +Look in the output for a line like `http://127.0.0.1:8888/lab?token=...` and copy that URL into a browser, a JupyterLab session will open. > [!TIP] > This image is not running in the GeoLab platform, so any features only available in GeoLab will not work from this local environment. ### Verifying the installed packages -The image includes two test options that ensure installed packages import and run. Both are copied into the container at build time, so they are available in the running container. Use them after a build to confirm nothing is broken (a missing system library or version conflict often installs cleanly but fails at import). Adjust as needed for the packages that you added or removed from the build. +The image includes `test_notebook.ipynb`, which ensures installed packages import and run. It is copied into the container at build time, so it is available in the running container. Use it after a build to confirm nothing is broken (a missing system library or version conflict often installs cleanly but fails at import). Adjust it as needed for the packages that you added or removed from the build. -In the Jupyter interface at `http://127.0.0.1:8888/lab...`: +The notebook's checks are built on top of `test_helpers.py`, a small module of test helpers (also copied into the container) that the notebook imports rather than duplicating this logic in every cell: -**Option 1 — `test_packages.py` (pytest, fastest).** This runs a minimal API call for each package and prints a pass/fail line per package. +| Function | Use for | What it does | +| --- | --- | --- | +| `py(modname, alias=None, smoke=None)` | Python packages | Imports `modname` and, if given, calls `smoke(mod)` as a minimal sanity check (e.g. constructing an object or calling a function). Records a pass with the package's `__version__`, or a fail with the exception. | +| `cli(cmd, version_flag='--version')` | Command-line tools | Confirms `cmd` is on `$PATH` and responds to `version_flag`. Records a pass with the version string, or a fail if it's missing. | -Create a new Terminal inside a running JupyterLab session (File → New → Terminal): +Each call appends a `(name, status, version, error)` row to the shared `RESULTS` list, which the notebook's final cell renders as a summary table. -```shell -pytest test_packages.py -v +In the JupyterLab session at `http://127.0.0.1:8888/lab...`, open `test_notebook.ipynb` and run all cells (Run → Run All Cells). The notebook is organized into one section per category in `environment.yml`/`requirements.txt` (Cloud & storage, Geospatial, Core scientific stack, etc.), each running `py()`/`cli()` checks for the packages in that category, and ends with a summary table listing the status (and version) of each package, with failures highlighted in red. + +**Adding a test for a new package.** If you add a package to `environment.yml` or `requirements.txt`, add a matching check to `test_notebook.ipynb` so it's covered by the summary table. Pick the section that matches where you added the package (or add a new section), and add a `py()` or `cli()` call. + +For example, adding `seisfetch` (a Python package) to the Geo / geoscience section: + +```python +py('dascore') +cli('gmt', version_flag='--version') +py('obspy', + smoke=lambda m: m.UTCDateTime('2020-01-01').timestamp) +py('obsplus') +py('pygmt') +py('seisfetch', + smoke=lambda m: m.Client()) # replace with a minimal, side-effect-free call ``` -**Option 2 — `test_notebook.ipynb` (interactive).** Open `test_notebook.ipynb`, and run all cells (Run → Run All Cells). The notebook performs the same import-and-exercise checks and ends with a summary table listing the status (and version) of each package, with failures highlighted in red. +The `smoke` argument is optional but recommended, a bare import can succeed even when the package is broken in ways that only show up on first use (e.g. a missing compiled extension). Pick a call that exercises the package without hitting the network or requiring credentials, since the notebook may run without EarthScope services available locally. > [!TIP] -> A failure here points at the package, not your notebook code — usually a missing system dependency (add it to `apt.txt`) or a version conflict between conda and pip packages. If you add or remove a package in `environment.yml` or `requirements.txt`, update the tests to match. +> A failure here points at the package, not your notebook code, usually a missing system dependency (add it to `apt.txt`) or a version conflict between conda and pip packages. If something fails, it usually means a package name is misspelled or a version is unavailable, so go back to `environment.yml` or `requirements.txt`, fix it, and rebuild. --- -## Building and publishing the image +## Step 4: Publish Your Image -Once your configuration files are ready, you build the image locally *for the GeoLab platform* and push it to a container registry so GeoLab can access it. +Once your configuration files are ready and the local test passes, rebuild the image *for the GeoLab platform* and push it to a container registry so GeoLab can access it. -### Building the platform image +### Rebuild for GeoLab's platform -The `--platform linux/amd64` flag ensures the image runs on the same platform as GeoLab regardless of your own computer architecture. Name the image using your repository username, a descriptive name and tag to track versions, such as `username/my-geolab-image:0.1.0`. +GeoLab runs on Linux (`linux/amd64`). Depending on your computer's architecture (e.g. Apple Silicon), you may need to rebuild the image for that platform. Name the image using your repository username, a descriptive name, and a tag to track versions, such as `username/my-geolab-image:0.1.0`. ```shell docker build --no-cache -f Dockerfile \ @@ -247,41 +247,116 @@ docker build --no-cache -f Dockerfile \ --tag username/my-geolab-image:0.1.0 . ``` -Replace `username` with your Docker Hub username (or your registry path), `my-geolab-image` with your image name, and `0.1.0` with your version tag. The `--build-arg` values for `IMAGE_TITLE` and `IMAGE_AUTHORS` are optional but recommended for image metadata. +> [!TIP] +> Setting a version in the image tag is a best practice, it lets you track changes and reproduce a specific build later. Record what changed for each version in a `CHANGELOG.md` file. Without an explicit tag, Docker defaults to tagging the image `latest`, which makes it hard to tell which build is actually running. -What does `--no-cache` do? It forces Docker to rerun build steps from scratch, ensuring a clean build when publishing. +Replace `username` with your Docker Hub username (or your registry path), `my-geolab-image` with your image name, and `0.1.0` with your version tag. The `--build-arg` values for `IMAGE_TITLE` and `IMAGE_AUTHORS` are optional but recommended for image metadata. > [!NOTE] -> Images will be cached by different systems, including the image repository and GeoLab. If you are using a version (e.g. `0.1.0`) you should increment it for each build to avoid inadvertently using cached copies. +> **Why `--platform linux/amd64`?** GeoLab runs on Linux. If you're on a Mac with Apple Silicon, your local machine uses a different architecture, this flag ensures the image works on GeoLab regardless of what you built it on. -### Publishing the platform image +`--no-cache` forces Docker to rerun build steps from scratch, ensuring a clean build when publishing. -Push the image to Docker Hub, AWS ECR, or another registry so GeoLab can access it. If you have logged into your Docker account, you can push the image to Docker Hub with this command: +### Push the image + +If you created an account using Docker Desktop, pushing to Docker Hub does not require additional authentication: ```shell docker push username/my-geolab-image:0.1.0 ``` -Replace the details to match the --tag value in the build command. +Replace the details to match the `--tag` value in the build command. By default, images published to Docker Hub are public and available for use with GeoLab. > [!TIP] -> If the push results in an error, make sure you are logged into Docker Hub using `docker login` if needed. +> If the push results in an error, make sure you are logged into Docker Hub using `docker login`. + +Many other image repositories exist. If you use AWS ECR, follow these [instructions](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html). + +### Publishing to GitHub or AWS Image Repositories + +Alternatives to Docker Hub include GitHub Container Registry (ghcr) or AWS Elastic Container Registry (ECR). Choosing an image repository depends on your requirements. GitHub features tight integration with CI (Continuous Integration) through GitHub Actions that can trigger an image build and push to ghcr, automating the process through a `pull request`. AWS ECR offers cloud-scale uploads and downloads to support multiple instances of GeoLab requested by hundreds of users or more. + +Both ghcr and ECR have more stringent authorization practices and controls over publicly available images. For a step-by-step walkthrough for pushing images to either repository, go to [Pushing Images to GitHub or AWS ECR](https://docs.earthscope.org/geolab/advanced-topics/environments/pushing-to-ghcr-ecr) for detailed instructions. + +--- + +## Step 5: Launch It in GeoLab + +1. Go to [earthscope.org/data/geolab](https://www.earthscope.org/data/geolab/) and click **Launch GeoLab**. +2. Log in with your [EarthScope account](https://www.earthscope.org/user/login). +3. If a **Stop My Server** button appears, click it first. +4. Click **Start My Server**. +5. Under **Environment**, choose **Other**. +6. In the **Custom image** field, enter the image name from your registry, e.g. `ghcr.io/your-github-username/my-geolab-image:0.1.0` or `username/my-geolab-image:0.1.0` (for Docker Hub). If the image is not in Docker Hub, use the full image reference. +7. Click **Start**. + +GeoLab will pull your image and launch a session from it. The first launch takes a bit longer while it downloads; after that it's cached and starts quickly. + +--- + +## Making Changes Later + +Edit your config files, then rebuild and push with a new version number: + +```shell +docker build --no-cache -f Dockerfile \ + --platform linux/amd64 \ + --tag ghcr.io/your-github-username/my-geolab-image:0.1.1 . + +docker push ghcr.io/your-github-username/my-geolab-image:0.1.1 +``` -Many other image repositories exist. If you use AWS ECR, follow these [instructions](https://docs.aws.amazon.com/AmazonECR/latest/userguide/docker-push-ecr-image.html). +> [!TIP] +> Always use a new version number (`0.1.1`, `0.1.2`, etc.) when you rebuild. Images are cached by different systems, including the image repository and GeoLab, if you reuse the same tag, GeoLab may load the old cached version instead of your new one. --- -## Running your published image in GeoLab +## Troubleshooting Package Installation + +In general, it's best practice to install packages using the conda package manager for the GeoLab image. Conda checks packages for dependencies, which helps ensure that conflicts are resolved in the environment. Conda has a search function to discover packages. -1. Open [GeoLab's Hub Control Panel](https://geolab.earthscope.cloud/hub/home/) - * Login with your [EarthScope account](https://www.earthscope.org/user/login) - * If "Stop My Server" button is visible, select it to stop your current server - * Select "Start My Server" button -2. Choose **Environment → Other** -3. In **Custom image** enter the image name from your registry, e.g.: `username/my-geolab-image:0.1.0` (for Docker Hub) -4. Select **Start** +Some packages are only available on PyPI and are installed with the pip package manager, which also has a search function. > [!TIP] -> If the image is not in Docker Hub, the **Custom image** value should be the full image reference. +> Keep in mind that installation name and import name can be different (for example, `scikit-learn` vs `sklearn`). + +**Find if a conda package is available:** + +```shell +conda search -c conda-forge #PackageName (e.g., seisbench) +``` + +**Find if a package is available on PyPI:** + +```shell +python -m pip index versions #PackageName (e.g., seisbench) +``` + +--- + +## Quick Reference + +| What you want to do | Where to do it | +|---|---| +| Add a Python package | `environment.yml` under `dependencies` | +| Add a PyPI-only package | `requirements.txt` | +| Add a system tool | `apt.txt` | +| Run one-time setup commands | `postBuild` (create if needed) | +| Build locally for testing | `docker build --tag my-geolab-image:0.1.0 .` | +| Run locally | `docker run --rm -p 8888:8888 my-geolab-image:0.1.0` | +| Test packages | Run `test_notebook.ipynb` (see [Step 3](#step-3-build-and-test-locally)) | +| Build for GeoLab | `docker build --no-cache --platform linux/amd64 --tag username/image:version .` | +| Publish | `docker push username/my-geolab-image:0.1.0` | + +--- + +## Getting a Personal Access Token (for GHCR) + +Before you can push images to GHCR, you need a **Personal Access Token (PAT)** with package permissions: + +1. Go to **GitHub → Settings → Developer settings → Personal access tokens → Tokens (classic)**. +2. Click **Generate new token (classic)**. +3. Give it a name (e.g. `geolab-image`), set an expiration, and check the **`write:packages`** scope. +4. Click **Generate token** and copy it, as you won't be able to see it again. -GeoLab will pull and launch your custom environment. The first launch may take a bit longer while the image is transferred from the repository. +Save your token somewhere safe (a password manager works well). You'll use it to log in to the registry when publishing. diff --git a/geolab-base/apt.txt b/geolab-base/apt.txt index b2bdaa3..d59bf56 100644 --- a/geolab-base/apt.txt +++ b/geolab-base/apt.txt @@ -4,3 +4,4 @@ git gmt-dcw gmt-gshhg make +nano \ No newline at end of file diff --git a/geolab-base/build_process.png b/geolab-base/build_process.png new file mode 100644 index 0000000..06f639f Binary files /dev/null and b/geolab-base/build_process.png differ diff --git a/geolab-base/environment.yml b/geolab-base/environment.yml index b315c6e..91cc549 100644 --- a/geolab-base/environment.yml +++ b/geolab-base/environment.yml @@ -47,6 +47,9 @@ dependencies: - distributed # --- Visualization --- - matplotlib-base + - graphviz + - pygraphviz + - ipycytoscape - altair - hvplot - holoviews diff --git a/geolab-base/requirements.txt b/geolab-base/requirements.txt index bb232c5..f81ef83 100644 --- a/geolab-base/requirements.txt +++ b/geolab-base/requirements.txt @@ -6,6 +6,8 @@ earthscope-sdk==1.6.1 earthscope-cli==1.2.0 earthscopestraintools - # --- Jupyter add-ons --- jupyterlab_jupyterbook_navigation + +# --- Geophysics --- + diff --git a/geolab-base/test_helpers.py b/geolab-base/test_helpers.py new file mode 100644 index 0000000..ac86288 --- /dev/null +++ b/geolab-base/test_helpers.py @@ -0,0 +1,41 @@ +"""Helpers for smoke-testing installed packages and CLI tools.""" + +import importlib +import shutil +import subprocess + +RESULTS = [] + + +def reset(): + """Clear RESULTS — call before a fresh run in a long-lived kernel.""" + RESULTS.clear() + + +def py(modname, alias=None, smoke=None): + """Import `modname` and optionally run `smoke(mod)` as a sanity check.""" + label = alias or modname + try: + mod = importlib.import_module(modname) + if smoke is not None: + smoke(mod) + version = getattr(mod, '__version__', '') + RESULTS.append((label, 'OK', str(version), '')) + except Exception as exc: + RESULTS.append((label, 'FAIL', '', f'{type(exc).__name__}: {exc}')) + + +def cli(cmd, version_flag='--version'): + """Verify `cmd` is on $PATH and responds to a version flag.""" + path = shutil.which(cmd) + if not path: + RESULTS.append((cmd, 'FAIL', '', 'not on $PATH')) + return + try: + r = subprocess.run([cmd, version_flag], + capture_output=True, text=True, timeout=10) + line = (r.stdout or r.stderr).strip().splitlines() + version = line[0] if line else 'on PATH' + RESULTS.append((cmd, 'OK', version[:80], '')) + except Exception as exc: + RESULTS.append((cmd, 'OK', 'on PATH', f'{type(exc).__name__}')) diff --git a/geolab-base/test_notebook.ipynb b/geolab-base/test_notebook.ipynb index fb15dd6..b7269f0 100644 --- a/geolab-base/test_notebook.ipynb +++ b/geolab-base/test_notebook.ipynb @@ -1,373 +1,355 @@ { - "cells": [ - { - "cell_type": "markdown", - "id": "118af38b", - "metadata": {}, - "source": [ - "# Smoke tests for `geolab-base`\n", - "\n", - "For every package in `environment.yml` (conda + pip): try to import it and\n", - "exercise one minimal API call. CLI-only packages get a `which`/`--version`\n", - "check instead. A failure here means something installed but doesn't load,\n", - "which is usually a sign of an ABI mismatch or a missing system library.\n", - "\n", - "Run all cells. The summary at the bottom lists pass/fail per package." - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Setup" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import importlib\n", - "import shutil\n", - "import subprocess\n", - "import sys\n", - "\n", - "RESULTS = []\n", - "\n", - "\n", - "def py(modname, alias=None, smoke=None):\n", - " \"\"\"Import `modname` and optionally run `smoke(mod)` as a sanity check.\"\"\"\n", - " label = alias or modname\n", - " try:\n", - " mod = importlib.import_module(modname)\n", - " if smoke is not None:\n", - " smoke(mod)\n", - " version = getattr(mod, '__version__', '')\n", - " RESULTS.append((label, 'OK', str(version), ''))\n", - " except Exception as exc:\n", - " RESULTS.append((label, 'FAIL', '', f'{type(exc).__name__}: {exc}'))\n", - "\n", - "\n", - "def cli(cmd, version_flag='--version'):\n", - " \"\"\"Verify `cmd` is on $PATH and responds to a version flag.\"\"\"\n", - " path = shutil.which(cmd)\n", - " if not path:\n", - " RESULTS.append((cmd, 'FAIL', '', 'not on $PATH'))\n", - " return\n", - " try:\n", - " r = subprocess.run([cmd, version_flag],\n", - " capture_output=True, text=True, timeout=10)\n", - " line = (r.stdout or r.stderr).strip().splitlines()\n", - " version = line[0] if line else 'on PATH'\n", - " RESULTS.append((cmd, 'OK', version[:80], ''))\n", - " except Exception as exc:\n", - " RESULTS.append((cmd, 'OK', 'on PATH', f'{type(exc).__name__}'))\n", - "\n", - "\n", - "print(f'Python {sys.version}')\n", - "print(f'sys.prefix: {sys.prefix}')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Cloud & storage" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "705a7197", - "metadata": {}, - "outputs": [], - "source": [ - "cli('aws')\n", - "py('awswrangler')\n", - "py('boto3', smoke=lambda m: m.client('s3', region_name='us-east-1'))\n", - "py('fsspec', smoke=lambda m: m.filesystem('memory'))\n", - "py('obstore')\n", - "py('s3fs', smoke=lambda m: m.S3FileSystem)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Geospatial" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "84bab05a", - "metadata": {}, - "outputs": [], - "source": [ - "py('cartopy.crs', alias='cartopy',\n", - " smoke=lambda m: m.PlateCarree())\n", - "py('contextily')\n", - "py('fiona', smoke=lambda m: m.supported_drivers)\n", - "py('folium',\n", - " smoke=lambda m: m.Map(location=[0, 0], zoom_start=2))\n", - "py('osgeo.gdal', alias='gdal',\n", - " smoke=lambda m: m.VersionInfo('RELEASE_NAME'))\n", - "py('ipyleaflet', smoke=lambda m: m.Map())\n", - "py('lonboard')\n", - "py('pyproj', smoke=lambda m: m.CRS('EPSG:4326'))\n", - "py('shapely.geometry', alias='shapely',\n", - " smoke=lambda m: m.Point(0, 0).buffer(1).area)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Core scientific stack" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "b383bbe0", - "metadata": {}, - "outputs": [], - "source": [ - "py('numpy', smoke=lambda m: int(m.array([1, 2, 3]).sum()))\n", - "py('numba', smoke=lambda m: m.njit(lambda x: x + 1)(1))\n", - "py('scipy.stats', alias='scipy', smoke=lambda m: m.norm.cdf(0))\n", - "py('pandas',\n", - " smoke=lambda m: m.DataFrame({'a': [1, 2]}).shape)\n", - "py('geopandas')\n", - "import matplotlib; matplotlib.use('Agg')\n", - "py('matplotlib', alias='matplotlib-base',\n", - " smoke=lambda m: m.figure.Figure())\n", - "py('xarray',\n", - " smoke=lambda m: m.DataArray([1, 2, 3]).sum().item())\n", - "py('netCDF4', alias='netcdf4')\n", - "py('h5py')\n", - "py('h5netcdf')\n", - "py('pyarrow',\n", - " smoke=lambda m: m.array([1, 2, 3]).to_pylist())\n", - "py('zarr',\n", - " smoke=lambda m: m.zeros((3,), chunks=3, dtype='f4'))\n", - "py('virtualizarr')\n", - "py('bottleneck',\n", - " smoke=lambda m: m.nansum([1.0, 2.0, float('nan'), 3.0]))\n", - "py('flox')\n", - "py('pooch')\n", - "py('dask.array', alias='dask',\n", - " smoke=lambda m: m.ones(10, chunks=5).sum().compute())\n", - "py('distributed')\n", - "py('dask_gateway', alias='dask-gateway')\n", - "py('cvxpy', smoke=lambda m: m.Variable(name='x'))" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Geo / geoscience" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "062000c0", - "metadata": {}, - "outputs": [], - "source": [ - "py('dascore')\n", - "cli('gmt', version_flag='--version')\n", - "py('obspy',\n", - " smoke=lambda m: m.UTCDateTime('2020-01-01').timestamp)\n", - "py('obsplus')\n", - "py('pygmt')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Utilities" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "dc28287d", - "metadata": {}, - "outputs": [], - "source": [ - "py('tqdm',\n", - " smoke=lambda m: list(m.tqdm(range(3), disable=True)))\n", - "py('requests')\n", - "py('yaml', alias='pyyaml',\n", - " smoke=lambda m: m.safe_load('a: 1'))\n", - "cli('gs', version_flag='--version') # ghostscript\n", - "cli('ffmpeg', version_flag='-version')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Dev tools" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "cli('gh')\n", - "cli('gh-scoped-creds')\n", - "py('pytest')\n", - "cli('ruff')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Jupyter stack & extensions" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "6ed8a837", - "metadata": {}, - "outputs": [], - "source": [ - "py('jupyterhub')\n", - "py('jupyter_server')\n", - "py('jupyterlab')\n", - "py('ipykernel')\n", - "py('jupyter_resource_usage', alias='jupyter-resource-usage')\n", - "py('jupyter_ruff', alias='jupyter-ruff')\n", - "py('jupyter_server_proxy', alias='jupyter-server-proxy')\n", - "py('jupyterlab_git', alias='jupyterlab-git')\n", - "py('jupyterlab_myst', alias='jupyterlab-myst')\n", - "py('jupyterlab_code_formatter')\n", - "py('jupyterlab_pygments')\n", - "py('nbdime')" - ] - }, - { - "cell_type": "markdown", - "id": "7788e14a", - "metadata": {}, - "source": [ - "## pip packages & visualization" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "824b272d", - "metadata": {}, - "outputs": [], - "source": [ - "# EarthScope --------------------------------------------------\n", - "py('earthscope_sdk', alias='earthscope-sdk')\n", - "cli('es') # earthscope-cli entry point\n", - "py('earthscopestraintools')\n", - "\n", - "# Jupyter add-ons ---------------------------------------------\n", - "py('jupyterlab_jupyterbook_navigation')\n", - "\n", - "# Visualization & data frames ---------------------------------\n", - "py('altair',\n", - " smoke=lambda m: m.Chart())\n", - "py('plotly')\n", - "py('polars',\n", - " smoke=lambda m: m.DataFrame({'a': [1, 2]}))\n", - "py('vegafusion')\n", - "py('vl_convert', alias='vl-convert-python')\n", - "py('ipympl')\n", - "py('hvplot')\n", - "py('holoviews', alias='holoviews',\n", - " smoke=lambda m: m.Curve([1, 2, 3]))\n", - "py('panel')" - ] - }, - { - "cell_type": "markdown", - "id": "5411c0ef", - "metadata": {}, - "source": [ - "## Interactive widgets" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "id": "d7ba172f", - "metadata": {}, - "outputs": [], - "source": [ - "py('ipywidgets',\n", - " smoke=lambda m: m.IntSlider(value=5, min=0, max=10))\n", - "py('anywidget')\n", - "py('bqplot')\n", - "py('ipytree', smoke=lambda m: m.Node(name='root'))\n", - "py('itables')\n", - "py('ipydatagrid')\n", - "from sidecar import Sidecar # noqa: F401\n", - "py('sidecar')" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Summary" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import pandas as pd\n", - "from IPython.display import display\n", - "\n", - "df = pd.DataFrame(RESULTS,\n", - " columns=['package', 'status', 'version', 'error'])\n", - "\n", - "passed = int((df['status'] == 'OK').sum())\n", - "total = len(df)\n", - "failed = total - passed\n", - "\n", - "print(f'Results: {passed}/{total} OK, {failed} failed')\n", - "if failed:\n", - " print('\\nFailures:')\n", - " for _, row in df[df['status'] == 'FAIL'].iterrows():\n", - " print(f\" {row['package']:35s} {row['error']}\")\n", - "\n", - "df.style.map(\n", - " lambda v: ('color: red; font-weight: bold' if v == 'FAIL'\n", - " else 'color: green'),\n", - " subset=['status']\n", - ")" - ] - } - ], - "metadata": { - "kernelspec": { - "display_name": "Python 3", - "language": "python", - "name": "python3" - }, - "language_info": { - "name": "python", - "pygments_lexer": "ipython3" - } + "cells": [ + { + "cell_type": "markdown", + "id": "118af38b", + "metadata": {}, + "source": [ + "# Smoke tests for `geolab-base`\n", + "\n", + "For every package in `environment.yml` (conda + pip): try to import it and\n", + "exercise one minimal API call. CLI-only packages get a `which`/`--version`\n", + "check instead. A failure here means something installed but doesn't load,\n", + "which is usually a sign of an ABI mismatch or a missing system library.\n", + "\n", + "Run all cells. The summary at the bottom lists pass/fail per package." + ] }, - "nbformat": 4, - "nbformat_minor": 5 + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Setup" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "618283a2", + "metadata": {}, + "outputs": [], + "source": [ + "import sys\n", + "\n", + "import test_helpers as test\n", + "from test_helpers import RESULTS, cli, py\n", + "\n", + "test.RESULTS.clear()\n", + "\n", + "print(f'Python {sys.version}')\n", + "print(f'sys.prefix: {sys.prefix}')" + ] + }, + { + "cell_type": "markdown", + "id": "586f3ed6", + "metadata": {}, + "source": [ + "## Cloud & storage" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "705a7197", + "metadata": {}, + "outputs": [], + "source": [ + "cli('aws')\n", + "py('awswrangler')\n", + "py('boto3', smoke=lambda m: m.client('s3', region_name='us-east-1'))\n", + "py('fsspec', smoke=lambda m: m.filesystem('memory'))\n", + "py('obstore')\n", + "py('s3fs', smoke=lambda m: m.S3FileSystem)" + ] + }, + { + "cell_type": "markdown", + "id": "cbc56927", + "metadata": {}, + "source": [ + "## Geospatial" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "84bab05a", + "metadata": {}, + "outputs": [], + "source": [ + "py('cartopy.crs', alias='cartopy',\n", + " smoke=lambda m: m.PlateCarree())\n", + "py('contextily')\n", + "py('fiona', smoke=lambda m: m.supported_drivers)\n", + "py('folium',\n", + " smoke=lambda m: m.Map(location=[0, 0], zoom_start=2))\n", + "py('osgeo.gdal', alias='gdal',\n", + " smoke=lambda m: m.VersionInfo('RELEASE_NAME'))\n", + "py('ipyleaflet', smoke=lambda m: m.Map())\n", + "py('lonboard')\n", + "py('pyproj', smoke=lambda m: m.CRS('EPSG:4326'))\n", + "py('shapely.geometry', alias='shapely',\n", + " smoke=lambda m: m.Point(0, 0).buffer(1).area)" + ] + }, + { + "cell_type": "markdown", + "id": "91dba753", + "metadata": {}, + "source": [ + "## Core scientific stack" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "b383bbe0", + "metadata": {}, + "outputs": [], + "source": [ + "py('numpy', smoke=lambda m: int(m.array([1, 2, 3]).sum()))\n", + "py('numba', smoke=lambda m: m.njit(lambda x: x + 1)(1))\n", + "py('scipy.stats', alias='scipy', smoke=lambda m: m.norm.cdf(0))\n", + "py('pandas',\n", + " smoke=lambda m: m.DataFrame({'a': [1, 2]}).shape)\n", + "py('geopandas')\n", + "import matplotlib; matplotlib.use('Agg')\n", + "py('matplotlib', alias='matplotlib-base',\n", + " smoke=lambda m: m.figure.Figure())\n", + "cli('dot', version_flag='-V')\n", + "py('pygraphviz', smoke=lambda m: m.AGraph().add_node(1))\n", + "py('xarray',\n", + " smoke=lambda m: m.DataArray([1, 2, 3]).sum().item())\n", + "py('netCDF4', alias='netcdf4')\n", + "py('h5py')\n", + "py('h5netcdf')\n", + "py('pyarrow',\n", + " smoke=lambda m: m.array([1, 2, 3]).to_pylist())\n", + "py('zarr',\n", + " smoke=lambda m: m.zeros((3,), chunks=3, dtype='f4'))\n", + "py('virtualizarr')\n", + "py('bottleneck',\n", + " smoke=lambda m: m.nansum([1.0, 2.0, float('nan'), 3.0]))\n", + "py('flox')\n", + "py('pooch')\n", + "py('dask.array', alias='dask',\n", + " smoke=lambda m: m.ones(10, chunks=5).sum().compute())\n", + "py('distributed')\n", + "py('dask_gateway', alias='dask-gateway')\n", + "py('cvxpy', smoke=lambda m: m.Variable(name='x'))" + ] + }, + { + "cell_type": "markdown", + "id": "a86a0e2a", + "metadata": {}, + "source": [ + "## Geo / geoscience" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "062000c0", + "metadata": {}, + "outputs": [], + "source": [ + "py('dascore')\n", + "cli('gmt', version_flag='--version')\n", + "py('obspy',\n", + " smoke=lambda m: m.UTCDateTime('2020-01-01').timestamp)\n", + "py('obsplus')\n", + "py('pygmt')" + ] + }, + { + "cell_type": "markdown", + "id": "4d531eaf", + "metadata": {}, + "source": [ + "## Utilities" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "dc28287d", + "metadata": {}, + "outputs": [], + "source": [ + "py('tqdm',\n", + " smoke=lambda m: list(m.tqdm(range(3), disable=True)))\n", + "py('requests')\n", + "py('yaml', alias='pyyaml',\n", + " smoke=lambda m: m.safe_load('a: 1'))\n", + "cli('gs', version_flag='--version') # ghostscript\n", + "cli('ffmpeg', version_flag='-version')" + ] + }, + { + "cell_type": "markdown", + "id": "77abdcce", + "metadata": {}, + "source": [ + "## Dev tools" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "a7ccd9e7", + "metadata": {}, + "outputs": [], + "source": [ + "cli('gh')\n", + "cli('gh-scoped-creds')\n", + "py('pytest')\n", + "cli('ruff')" + ] + }, + { + "cell_type": "markdown", + "id": "14ba80ce", + "metadata": {}, + "source": [ + "## Jupyter stack & extensions" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "6ed8a837", + "metadata": {}, + "outputs": [], + "source": [ + "py('jupyterhub')\n", + "py('jupyter_server')\n", + "py('jupyterlab')\n", + "py('ipykernel')\n", + "py('jupyter_resource_usage', alias='jupyter-resource-usage')\n", + "py('jupyter_ruff', alias='jupyter-ruff')\n", + "py('jupyter_server_proxy', alias='jupyter-server-proxy')\n", + "py('jupyterlab_git', alias='jupyterlab-git')\n", + "py('jupyterlab_myst', alias='jupyterlab-myst')\n", + "py('jupyterlab_code_formatter')\n", + "py('jupyterlab_pygments')\n", + "py('nbdime')" + ] + }, + { + "cell_type": "markdown", + "id": "7788e14a", + "metadata": {}, + "source": [ + "## pip packages & visualization" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "824b272d", + "metadata": {}, + "outputs": [], + "source": [ + "# EarthScope --------------------------------------------------\n", + "py('earthscope_sdk', alias='earthscope-sdk')\n", + "cli('es') # earthscope-cli entry point\n", + "py('earthscopestraintools')\n", + "\n", + "# Jupyter add-ons ---------------------------------------------\n", + "py('jupyterlab_jupyterbook_navigation')\n", + "\n", + "# Visualization & data frames ---------------------------------\n", + "py('altair',\n", + " smoke=lambda m: m.Chart())\n", + "py('plotly')\n", + "py('polars',\n", + " smoke=lambda m: m.DataFrame({'a': [1, 2]}))\n", + "py('vegafusion')\n", + "py('vl_convert', alias='vl-convert-python')\n", + "py('ipympl')\n", + "py('hvplot')\n", + "py('holoviews', alias='holoviews',\n", + " smoke=lambda m: m.Curve([1, 2, 3]))\n", + "py('panel')" + ] + }, + { + "cell_type": "markdown", + "id": "5411c0ef", + "metadata": {}, + "source": [ + "## Interactive widgets" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "id": "d7ba172f", + "metadata": {}, + "outputs": [], + "source": [ + "py('ipywidgets',\n", + " smoke=lambda m: m.IntSlider(value=5, min=0, max=10))\n", + "py('anywidget')\n", + "py('bqplot')\n", + "py('ipytree', smoke=lambda m: m.Node(name='root'))\n", + "py('ipycytoscape', smoke=lambda m: m.CytoscapeWidget())\n", + "py('itables')\n", + "py('ipydatagrid')\n", + "from sidecar import Sidecar # noqa: F401\n", + "py('sidecar')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Summary" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import pandas as pd\n", + "from IPython.display import display\n", + "\n", + "df = pd.DataFrame(RESULTS,\n", + " columns=['package', 'status', 'version', 'error'])\n", + "\n", + "passed = int((df['status'] == 'OK').sum())\n", + "total = len(df)\n", + "failed = total - passed\n", + "\n", + "print(f'Results: {passed}/{total} OK, {failed} failed')\n", + "if failed:\n", + " print('\\nFailures:')\n", + " for _, row in df[df['status'] == 'FAIL'].iterrows():\n", + " print(f\" {row['package']:35s} {row['error']}\")\n", + "\n", + "df.style.map(\n", + " lambda v: ('color: red; font-weight: bold' if v == 'FAIL'\n", + " else 'color: green'),\n", + " subset=['status']\n", + ")" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "name": "python", + "pygments_lexer": "ipython3" + } + }, + "nbformat": 4, + "nbformat_minor": 5 } diff --git a/geolab-base/test_packages.py b/geolab-base/test_packages.py deleted file mode 100644 index a25b8f0..0000000 --- a/geolab-base/test_packages.py +++ /dev/null @@ -1,593 +0,0 @@ -""" -Unit tests for packages installed in geolab-base. - -Run inside the container: - pytest test_packages.py -v - -Run with a filter: - pytest test_packages.py -v -k geospatial # tests with 'geospatial' in name - pytest test_packages.py::test_obspy -v # single test - -Each test exercises one package with a minimal API call. Failures usually -indicate ABI mismatches, missing system libraries, or broken installs -- -NOT just missing imports, which a smoke check would also catch. -""" - -import math -import re -import shutil -import subprocess - -import pytest - -# ─── Helpers ────────────────────────────────────────────────── - - -def _cli_version(cmd, version_flag="--version"): - """Run `cmd --version` and return stdout. Fail if not on $PATH.""" - if not shutil.which(cmd): - pytest.fail(f"{cmd} not on $PATH") - r = subprocess.run( - [cmd, version_flag], - capture_output=True, - text=True, - timeout=10, - ) - return (r.stdout or r.stderr).strip() - - -# ─── Cloud & storage ────────────────────────────────────────── - - -def test_aws_cli(): - out = _cli_version("aws") - assert "aws-cli" in out.lower() - - -def test_awswrangler(): - import awswrangler as wr - - assert wr.__version__ - - -def test_boto3(): - import boto3 - - client = boto3.client("s3", region_name="us-east-1") - assert client.meta.service_model.service_name == "s3" - - -def test_obstore(): - import obstore # noqa: F401 - - -def test_fsspec(): - import fsspec - - fs = fsspec.filesystem("memory") - assert fs.protocol == "memory" - - -def test_s3fs(): - import s3fs - - assert s3fs.S3FileSystem is not None - - -# ─── Geospatial ─────────────────────────────────────────────── - - -def test_cartopy(): - import cartopy.crs as ccrs - - proj = ccrs.PlateCarree() - assert proj.proj4_params is not None - - -def test_contextily(): - import contextily as cx - - assert cx.__version__ - - -def test_fiona(): - import fiona - - drivers = fiona.supported_drivers - assert "GPKG" in drivers - assert "ESRI Shapefile" in drivers - - -def test_folium(): - import folium - - m = folium.Map(location=[0, 0], zoom_start=2) - assert m is not None - - -def test_gdal(): - from osgeo import gdal - - release = gdal.VersionInfo("RELEASE_NAME") - assert release # e.g. "3.8.4" - - -def test_pyproj(): - import pyproj - - crs = pyproj.CRS("EPSG:4326") - assert crs.to_epsg() == 4326 - assert "WGS 84" in crs.name - - -def test_shapely(): - from shapely.geometry import Point - - p = Point(0, 0) - assert p.buffer(1).area == pytest.approx(math.pi, abs=0.1) - - -def test_ipyleaflet(): - import ipyleaflet - - m = ipyleaflet.Map() - assert m is not None - - -def test_lonboard(): - import lonboard - - assert lonboard.__version__ - - -# ─── Core scientific stack ──────────────────────────────────── - - -def test_numpy(): - import numpy as np - - assert int(np.array([1, 2, 3]).sum()) == 6 - - -def test_numba(): - from numba import njit - - @njit - def add_one(x): - return x + 1 - - assert add_one(1) == 2 - - -def test_scipy(): - from scipy import stats - - assert stats.norm.cdf(0) == pytest.approx(0.5) - - -def test_pandas(): - import pandas as pd - - df = pd.DataFrame({"a": [1, 2, 3]}) - assert df.shape == (3, 1) - assert df["a"].sum() == 6 - - -def test_geopandas(): - import geopandas as gpd - from shapely.geometry import Point - - gdf = gpd.GeoDataFrame( - {"name": ["a", "b"]}, - geometry=[Point(0, 0), Point(1, 1)], - crs="EPSG:4326", - ) - assert len(gdf) == 2 - assert gdf.crs.to_epsg() == 4326 - - -def test_matplotlib_base(): - import matplotlib - - matplotlib.use("Agg") - import matplotlib.pyplot as plt - - fig, ax = plt.subplots() - ax.plot([0, 1], [0, 1]) - plt.close(fig) - - -def test_xarray(): - import xarray as xr - - da = xr.DataArray([1, 2, 3], dims="x") - assert da.sum().item() == 6 - - -@pytest.mark.filterwarnings("ignore:numpy.ndarray size changed:RuntimeWarning") -def test_netcdf4(tmp_path): - import netCDF4 - - path = tmp_path / "smoke.nc" - with netCDF4.Dataset(path, "w") as ds: - ds.createDimension("x", 3) - v = ds.createVariable("v", "f4", ("x",)) - v[:] = [1.0, 2.0, 3.0] - with netCDF4.Dataset(path, "r") as ds: - assert ds["v"][:].sum() == pytest.approx(6.0) - - -def test_h5py(tmp_path): - import h5py - import numpy as np - - path = tmp_path / "smoke.h5" - with h5py.File(path, "w") as f: - f["arr"] = np.arange(3) - with h5py.File(path, "r") as f: - assert f["arr"][:].sum() == 3 - - -def test_h5netcdf(tmp_path): - import h5netcdf.legacyapi as netCDF4 - import numpy as np - - path = tmp_path / "smoke_h5netcdf.nc" - with netCDF4.Dataset(path, "w") as ds: - ds.createDimension("x", 3) - v = ds.createVariable("v", "f4", ("x",)) - v[:] = np.array([1.0, 2.0, 3.0]) - with netCDF4.Dataset(path, "r") as ds: - assert ds["v"][:].sum() == pytest.approx(6.0) - - -def test_zarr(): - import zarr - - arr = zarr.zeros((3,), chunks=3, dtype="f4") - arr[:] = [1.0, 2.0, 3.0] - assert arr[:].sum() == pytest.approx(6.0) - - -def test_virtualizarr(): - import virtualizarr # noqa: F401 - - assert virtualizarr.__version__ - - -def test_pooch(): - import pooch - - assert pooch.__version__ - - -def test_pyarrow(): - import pyarrow as pa - - arr = pa.array([1, 2, 3]) - assert arr.to_pylist() == [1, 2, 3] - - -def test_bottleneck(): - import bottleneck as bn - - assert bn.nansum([1.0, 2.0, float("nan"), 3.0]) == pytest.approx(6.0) - - -def test_flox(): - import flox - - assert flox.__version__ - - -# ─── Parallel computing ─────────────────────────────────────── - - -def test_dask(): - import dask.array as da - - assert da.ones(10, chunks=5).sum().compute() == pytest.approx(10.0) - - -def test_dask_gateway(): - import dask_gateway # noqa: F401 - - -def test_distributed(): - import distributed - - assert distributed.__version__ - - -# ─── Geo / geoscience ───────────────────────────────────────── - - -def test_dascore(): - import dascore - - assert hasattr(dascore, "__version__") - - -def test_gmt_cli(): - out = _cli_version("gmt") - assert re.match(r"\d+\.\d+", out.strip()) # gmt --version prints bare "6.6.0" - - -def test_obspy(): - from obspy import UTCDateTime - - t = UTCDateTime("2020-01-01T12:30:45") - assert t.year == 2020 - assert t.month == 1 - assert t.hour == 12 - - -def test_pygmt(): - import pygmt - - assert pygmt.__version__ - - -def test_obsplus(): - import obsplus # noqa: F401 - - -# ─── Optimization ───────────────────────────────────────────── - - -def test_cvxpy(): - import cvxpy as cp - - x = cp.Variable() - prob = cp.Problem(cp.Minimize((x - 2) ** 2)) - prob.solve() - assert x.value == pytest.approx(2.0, abs=1e-3) - - -# ─── Visualization ──────────────────────────────────────────── - - -def test_ipympl(): - import ipympl # noqa: F401 - - -def test_hvplot(): - import hvplot # noqa: F401 - - assert hvplot.__version__ - - -def test_holoviews(): - import holoviews as hv - - curve = hv.Curve([1, 2, 3]) - assert curve is not None - - -def test_panel(): - import panel as pn - - assert pn.__version__ - - -def test_vl_convert(): - import vl_convert as vlc - - assert vlc.__version__ - - -# ─── Media & system tools ───────────────────────────────────── - - -def test_ghostscript_cli(): - out = _cli_version("gs", "--version") - assert any(ch.isdigit() for ch in out) - - -def test_ffmpeg_cli(): - out = _cli_version("ffmpeg", "-version") - assert "ffmpeg" in out.lower() - - -# ─── Utilities ──────────────────────────────────────────────── - - -def test_tqdm(): - from tqdm import tqdm - - assert list(tqdm(range(3), disable=True)) == [0, 1, 2] - - -def test_requests(): - import requests - - assert requests.__version__ - - -def test_pyyaml(): - import yaml - - parsed = yaml.safe_load("a: 1\nb: [2, 3]") - assert parsed == {"a": 1, "b": [2, 3]} - - -# ─── Dev tools ──────────────────────────────────────────────── - - -def test_gh_cli(): - out = _cli_version("gh") - assert "gh version" in out.lower() or "github cli" in out.lower() - - -def test_gh_scoped_creds(): - assert shutil.which("gh-scoped-creds") is not None - - -def test_pytest_self(): - # we're in pytest, so importing it must work - assert pytest.__version__ - - -def test_ruff_cli(): - out = _cli_version("ruff") - assert "ruff" in out.lower() - - -# ─── Jupyter stack & extensions ─────────────────────────────── - - -def test_jupyterhub(): - import jupyterhub - - assert jupyterhub.__version__ - - -def test_jupyter_server(): - import jupyter_server - - assert jupyter_server.__version__ - - -def test_jupyterlab(): - import jupyterlab - - assert jupyterlab.__version__ - - -def test_ipykernel(): - import ipykernel - - assert ipykernel.__version__ - - -def test_jupyter_resource_usage(): - import jupyter_resource_usage # noqa: F401 - - -def test_jupyter_ruff(): - import jupyter_ruff # noqa: F401 - - -def test_jupyter_server_proxy(): - import jupyter_server_proxy # noqa: F401 - - -def test_jupyterlab_git(): - import jupyterlab_git # noqa: F401 - - -def test_jupyterlab_myst(): - import jupyterlab_myst # noqa: F401 - - -def test_jupyterlab_code_formatter(): - import jupyterlab_code_formatter # noqa: F401 - - -def test_jupyterlab_pygments(): - import jupyterlab_pygments # noqa: F401 - - -def test_nbdime(): - import nbdime - - assert nbdime.__version__ - - -def test_nbgitpuller(): - import nbgitpuller - - assert nbgitpuller.__version__ - - -# ─── pip packages ───────────────────────────────────────────── - - -def test_earthscope_sdk(): - import earthscope_sdk # noqa: F401 - - -def test_earthscope_cli(): - # `es` is the earthscope-cli entry point - assert shutil.which("es") is not None - - -def test_earthscopestraintools(): - import earthscopestraintools # noqa: F401 - - -def test_jupyterlab_jupyterbook_navigation(): - import jupyterlab_jupyterbook_navigation # noqa: F401 - - -def test_altair(): - import altair as alt - - chart = alt.Chart() - assert chart is not None - - -def test_plotly(): - import plotly - - assert plotly.__version__ - - -def test_polars(): - import polars as pl - - df = pl.DataFrame({"a": [1, 2, 3]}) - assert df.shape == (3, 1) - assert df["a"].sum() == 6 - - -def test_vegafusion(): - import vegafusion # noqa: F401 - - -# ─── Interactive widgets ─────────────────────────────────────── - - -def test_ipywidgets(): - import ipywidgets - - slider = ipywidgets.IntSlider(value=5, min=0, max=10) - assert slider.value == 5 - - -def test_anywidget(): - import anywidget # noqa: F401 - - assert anywidget.__version__ - - -@pytest.mark.filterwarnings( - "ignore:metadata .* was set from the constructor:DeprecationWarning" -) -def test_bqplot(): - import bqplot # noqa: F401 - - assert bqplot.__version__ - - -def test_ipytree(): - from ipytree import Node - - root = Node(name="root") - assert root.name == "root" - - -def test_itables(): - import itables # noqa: F401 - - assert itables.__version__ - - -def test_ipydatagrid(): - import ipydatagrid # noqa: F401 - - assert ipydatagrid.__version__ - - -def test_sidecar(): - from sidecar import Sidecar # noqa: F401