diff --git a/.github/workflows/docker-smoke.yml b/.github/workflows/docker-smoke.yml new file mode 100644 index 0000000000..42f468d728 --- /dev/null +++ b/.github/workflows/docker-smoke.yml @@ -0,0 +1,73 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Build docker/Dockerfile and smoke-test the image (install + inject). +# Runs only when the Dockerfile or this workflow changes. + +--- +name: docker image smoke +on: + push: + branches: [master] + paths: + - 'docker/Dockerfile' + - '.github/workflows/docker-smoke.yml' + pull_request: + types: [opened, synchronize, reopened] + branches: [master] + paths: + - 'docker/Dockerfile' + - '.github/workflows/docker-smoke.yml' + +concurrency: + group: docker-smoke-${{ github.ref }} + cancel-in-progress: true + +permissions: {} + +jobs: + docker-smoke: + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v7 + - name: Build Nutch image + run: docker build -t apache/nutch:ci -f docker/Dockerfile docker/ + - name: Verify installation + run: | + set -euo pipefail + test "$(docker run --rm apache/nutch:ci id -un)" = nutch + test "$(docker run --rm apache/nutch:ci id -u)" != 0 + test "$(docker run --rm apache/nutch:ci printenv JAVA_HOME)" \ + = /usr/lib/jvm/java-17-openjdk + test "$(docker run --rm apache/nutch:ci printenv NUTCH_HOME)" \ + = /opt/nutch/runtime/local + docker run --rm -i apache/nutch:ci bash -ic 'test -n "$JAVA_HOME"' + # nutch with no args exits 1, so only check the launchers are on PATH + docker run --rm apache/nutch:ci bash -c \ + 'command -v nutch && command -v crawl' + - name: Inject seed URL + run: | + set -euo pipefail + docker run --rm apache/nutch:ci bash -c ' + set -euo pipefail + cd "$NUTCH_HOME" + mkdir -p urls + printf "%s\n" "https://nutch.apache.org/" > urls/seed.txt + nutch inject -Dhttp.agent.name=NutchDockerSmoke crawldb urls + STATS=$(nutch readdb crawldb -stats 2>&1) + printf "%s\n" "$STATS" + printf "%s\n" "$STATS" | grep -E "TOTAL urls:[[:space:]]*[1-9]" + ' diff --git a/docker/Dockerfile b/docker/Dockerfile index 392dfdb86f..c9c8a399c0 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -28,29 +28,30 @@ LABEL org.opencontainers.image.title="Apache Nutch 1.x Docker Image" LABEL org.opencontainers.image.url="https://hub.docker.com/r/apache/nutch" LABEL org.opencontainers.image.vendor="Apache Nutch https://nutch.apache.org" -WORKDIR /root/ - -# Install dependencies -RUN apk update -RUN apk --no-cache add apache-ant bash git openjdk17 - -# Establish environment variables -RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.bashrc -RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.ashrc ENV JAVA_HOME='/usr/lib/jvm/java-17-openjdk' -ENV NUTCH_HOME='/root/nutch_source/runtime/local' - -# Checkout and build the Nutch master branch (1.x) -RUN git clone https://github.com/apache/nutch.git nutch_source && \ - cd nutch_source && \ - ant runtime && \ - rm -rf build/ && \ - rm -rf /root/.ivy2/ - -# Create symlinks for runtime/local/bin/nutch and runtime/local/bin/crawl -RUN ln -sf $NUTCH_HOME/bin/nutch /usr/local/bin/ -RUN ln -sf $NUTCH_HOME/bin/crawl /usr/local/bin/ - -RUN echo "Successfully built image, see https://s.apache.org/m5933 for guidance on running a container instance." +ENV NUTCH_HOME='/opt/nutch/runtime/local' + +# Install dependencies, build Nutch, and drop privileges for runtime +RUN apk --no-cache add \ + apache-ant=1.10.14-r0 \ + bash=5.2.21-r0 \ + git=2.43.7-r0 \ + openjdk17=17.0.14_p7-r0 \ + && addgroup -S nutch \ + && adduser -S -D -H -G nutch -h /opt/nutch nutch \ + && git clone https://github.com/apache/nutch.git /opt/nutch \ + && ant -f /opt/nutch/build.xml runtime \ + && rm -rf /opt/nutch/build/ \ + && rm -rf /root/.ivy2/ \ + && ln -sf "${NUTCH_HOME}/bin/nutch" /usr/local/bin/ \ + && ln -sf "${NUTCH_HOME}/bin/crawl" /usr/local/bin/ \ + && echo "export JAVA_HOME=${JAVA_HOME}" >> /opt/nutch/.bashrc \ + && echo "export JAVA_HOME=${JAVA_HOME}" >> /opt/nutch/.ashrc \ + && chown -R nutch:nutch /opt/nutch \ + && echo "Successfully built image, see https://s.apache.org/m5933 for guidance on running a container instance." + +WORKDIR /opt/nutch + +USER nutch CMD ["/bin/bash"] diff --git a/docker/README.md b/docker/README.md index 6c63044812..1c3f57df68 100644 --- a/docker/README.md +++ b/docker/README.md @@ -20,7 +20,9 @@ Nutch can run on a single machine, but gains a lot of its strength from running Current configuration of this image consists of components: -* Nutch 1.x (branch "master") +* Nutch 1.x (branch "master") +* Default runtime user `nutch` (not root) +* `NUTCH_HOME=/opt/nutch/runtime/local` (sources under `/opt/nutch`) ## Base Image @@ -49,9 +51,19 @@ $(boot2docker shellinit | grep export) #may not be necessary docker build -t apache/nutch . ``` +Pull requests that change `docker/Dockerfile` (or the +[docker image smoke](../.github/workflows/docker-smoke.yml) workflow) build +this image and inject `https://nutch.apache.org/` into a CrawlDb. They do not +run fetch. + ## Security and plugin directories -Nutch loads executable code from the directories configured as `plugin.folders` (see `nutch-default.xml`). For production and shared images, treat those paths as **trusted**: mount them read-only where possible, rebuild images to change plugins, and run the crawl process under a dedicated low-privilege user so the filesystem cannot be abused to drop unexpected JARs or `plugin.xml` files into that tree. +Nutch loads executable code from the directories configured as `plugin.folders` +(see `nutch-default.xml`). For production and shared images, treat those paths +as **trusted**: mount them read-only where possible, rebuild images to change +plugins, and keep the crawl process on the image default user `nutch` so the +filesystem cannot be abused to drop unexpected JARs or `plugin.xml` files into +that tree. Use `docker run --user root ...` only for privileged debugging. User-defined JEXL in configuration (for example `index.jexl.filter`, generator expressions, and `hostdb.filter.expression`) is evaluated in a **sandboxed** engine by default. The property `nutch.jexl.disable.sandbox` disables that protection and must not be set in untrusted environments. @@ -63,12 +75,14 @@ boot2docker up $(boot2docker shellinit | grep export) ``` -Run a container interactively (`nutch` and `crawl` are on `PATH`; default command is `bash`): +Run a container interactively (`nutch` and `crawl` are on `PATH`; default command is `bash`; process user is `nutch`): ```bash docker run -t -i --name nutchcontainer apache/nutch ``` +The previous image layout used `/root/nutch_source`. Bind mounts that pointed at that path must be updated to `/opt/nutch`. + In another terminal, attach to a running container if needed: ```bash