From ea5bd7652c7d3567958adeb76d0cfeef3b6bcc1c Mon Sep 17 00:00:00 2001 From: totza2010 Date: Thu, 6 Nov 2025 22:09:08 +0700 Subject: [PATCH] Supports multiple rclone remotes and dynamic mounts Refactors logic to enable handling multiple rclone remotes, mountpoints, and ports using arrays and dynamic environment variable parsing. Improves mount, serve, healthcheck, and refresh workflows to work with multiple remotes, enabling parallel operations and per-remote scheduling. Cleans up legacy single-remote logic and simplifies configuration, making extension and maintenance easier. --- .github/workflows/buildx.yml | 10 +- Dockerfile | 11 +- root/etc/cont-finish.d/10-unmount | 9 +- root/etc/cont-init.d/50-rclone | 187 ++++++++++++++++--- root/etc/s6-overlay/s6-rc.d/cron/run | 31 ++- root/etc/s6-overlay/s6-rc.d/mount/data/check | 10 +- root/etc/s6-overlay/s6-rc.d/mount/finish | 10 +- root/etc/s6-overlay/s6-rc.d/mount/run | 39 +++- root/etc/s6-overlay/s6-rc.d/serve/run | 21 ++- root/usr/local/bin/healthcheck | 11 +- root/usr/local/bin/rclone_refresh.script | 39 ++-- root/usr/local/bin/variables | 71 +++++-- 12 files changed, 357 insertions(+), 92 deletions(-) diff --git a/.github/workflows/buildx.yml b/.github/workflows/buildx.yml index 9661b1d..29b5ddb 100644 --- a/.github/workflows/buildx.yml +++ b/.github/workflows/buildx.yml @@ -10,8 +10,8 @@ on: pull_request: env: - DOCKERHUB_USER: wiserain - IMAGE_NAME: wiserain/rclone + DOCKERHUB_USER: totza2010 + IMAGE_NAME: totza2010/rclone jobs: buildx: @@ -21,13 +21,15 @@ jobs: if: github.event_name != 'pull_request' strategy: matrix: - rclone-type: [latest, mod] + rclone-type: [latest, tgdrive, mod] steps: - name: Check if buildx required run: | if [ ${{ matrix.rclone-type }} = "latest" ]; then rclone_install_script_url="https://rclone.org/install.sh" + elif [ ${{ matrix.rclone-type }} = "tgdrive" ]; then + rclone_install_script_url="instl.vercel.app/rclone"; elif [ ${{ matrix.rclone-type }} = "mod" ]; then rclone_install_script_url="https://raw.githubusercontent.com/wiserain/rclone/mod/install.sh" fi @@ -139,6 +141,8 @@ jobs: run: | if [ ${{ matrix.rclone-type }} = "latest" ]; then rclone_install_script_url="https://rclone.org/install.sh" + elif [ ${{ matrix.rclone-type }} = "tgdrive" ]; then + rclone_install_script_url="instl.vercel.app/rclone"; elif [ ${{ matrix.rclone-type }} = "mod" ]; then rclone_install_script_url="https://raw.githubusercontent.com/wiserain/rclone/mod/install.sh" fi diff --git a/Dockerfile b/Dockerfile index 2455d33..60e2ace 100644 --- a/Dockerfile +++ b/Dockerfile @@ -22,7 +22,9 @@ RUN \ if [ "${RCLONE_TYPE}" = "latest" ]; then \ rclone_install_script_url="https://rclone.org/install.sh"; \ elif [ "${RCLONE_TYPE}" = "mod" ]; then \ - rclone_install_script_url="https://raw.githubusercontent.com/wiserain/rclone/mod/install.sh"; fi && \ + rclone_install_script_url="https://raw.githubusercontent.com/wiserain/rclone/mod/install.sh"; \ + elif [ "${RCLONE_TYPE}" = "tgdrive" ]; then \ + rclone_install_script_url="instl.vercel.app/rclone"; fi && \ curl -fsSL $rclone_install_script_url | bash # @@ -50,7 +52,6 @@ RUN \ mkdir -p \ /bar/cache \ /bar/log \ - /bar/cloud \ /bar/data \ /bar/local \ && \ @@ -123,14 +124,14 @@ ENV \ S6_SERVICES_GRACETIME=5000 \ S6_KILL_GRACETIME=5000 \ RCLONE_CONFIG=/config/rclone.conf \ - UFS_BRANCHES="/local=RW:/cloud=RO" \ - MFS_BRANCHES="/local=RW:/cloud=NC" \ + UFS_BRANCHES="/local=RW:" \ + MFS_BRANCHES="/local=RW:" \ UFS_USER_OPTS="cow,direct_io,nonempty,auto_cache,sync_read" \ MFS_USER_OPTS="rw,use_ino,func.getattr=newest,category.action=all,category.create=ff,cache.files=auto-full,dropcacheonclose=true" \ KEEP_EMPTY_DIRS=0 \ DATE_FORMAT="+%4Y/%m/%d %H:%M:%S" -VOLUME /config /cache /log /cloud /data /local +VOLUME /config /cache /log /data /local HEALTHCHECK --interval=30s --timeout=30s --start-period=10s --retries=3 \ CMD /usr/local/bin/healthcheck diff --git a/root/etc/cont-finish.d/10-unmount b/root/etc/cont-finish.d/10-unmount index c2dd245..b976337 100644 --- a/root/etc/cont-finish.d/10-unmount +++ b/root/etc/cont-finish.d/10-unmount @@ -1,9 +1,16 @@ #!/usr/bin/with-contenv bash +. /usr/local/bin/variables unmount() { if findmnt "$1" | grep -q fuse; then fusermount -uz "$1" ; fi } # unmount fuse unmount /data -[[ -n "${POOLING_FS:-}" ]] && unmount /cloud +echo ">>> unmounting rclone mountpoint: /data" +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + echo ">>> unmounting rclone mountpoint: ${mountpoint}" + unmount "${mountpoint}" +done exit 0 diff --git a/root/etc/cont-init.d/50-rclone b/root/etc/cont-init.d/50-rclone index 11996df..01860ee 100644 --- a/root/etc/cont-init.d/50-rclone +++ b/root/etc/cont-init.d/50-rclone @@ -15,33 +15,108 @@ if [[ ! "${RCLONE_REMOTE_PATH}" == :* ]]; then done fi -echo "*** checking rclone remote path: ${RCLONE_REMOTE_PATH}" -if [[ ! "${RCLONE_REMOTE_PATH}" == *:* ]]; then - echo "ERROR: Missing colon (:) in RCLONE_REMOTE_PATH (${RCLONE_REMOTE_PATH})" - exit 1 -fi -if [[ ! "${RCLONE_REMOTE_PATH}" == :* ]]; then - rclone_remote="$(echo "$RCLONE_REMOTE_PATH" | cut -d: -f1):" - if ! rclone listremotes | grep -q "${rclone_remote}"; then - echo "ERROR: Invalid rclone remote name (${rclone_remote})" - echo "ERROR: Available rclone remotes ($(rclone listremotes | xargs))" +echo "*** DYNAMIC MOUNTPOINT MANAGEMENT (Cleanup and Creation) ***" +declare -a EXPECTED_MOUNTPOINTS +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + + if [[ -n "$mountpoint" ]]; then + EXPECTED_MOUNTPOINTS+=("$mountpoint") + fi +done + +for current_dir in /cloud*; do + if [[ -d "$current_dir" ]] && [[ "$current_dir" != "/cloud" ]]; then + + IS_EXPECTED=0 + for expected_mp in "${EXPECTED_MOUNTPOINTS[@]}"; do + if [[ "$current_dir" == "$expected_mp" ]]; then + IS_EXPECTED=1 + break + fi + done + + if [[ $IS_EXPECTED -eq 0 ]]; then + echo " - Cleaning up unused mountpoint: $current_dir" + rmdir "$current_dir" || echo " (Note: Could not remove $current_dir, likely not empty)" + fi + fi +done + +declare -a ALL_DIRS_FOR_CHOWN +ALL_DIRS_FOR_CHOWN=("/config" "/log" "/data") + +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + opts="${value##*|}" + + if [[ -n "$mountpoint" ]] && [[ ! -d "$mountpoint" ]]; then + echo " - Creating required mountpoint: $mountpoint" + mkdir -p "$mountpoint" + fi + if [[ -n "$mountpoint" ]]; then + ALL_DIRS_FOR_CHOWN+=("$mountpoint") + fi + + cache_dir=$(echo "$opts" | grep -oP '\-\-cache\-dir=\K[^ ]+' || true) + if [[ -n "$cache_dir" ]] && [[ ! -d "$cache_dir" ]]; then + echo " - Creating required cache directory: $cache_dir" + mkdir -p "$cache_dir" + fi + if [[ -n "$cache_dir" ]]; then + ALL_DIRS_FOR_CHOWN+=("$cache_dir") + fi +done + +echo "*** checking rclone ${#RCLONE_PATHS[@]} remote(s):" +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + remote="${value%%|*}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + opts="${value##*|}" + echo " - ${key} → ${remote} to ${mountpoint} with options: $(echo $opts)" +done +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + remote="${value%%|*}" + echo ">>> checking rclone remote path: ${remote}" + if [[ ! "${remote}" == *:* ]]; then + echo "ERROR: Missing colon (:) in RCLONE_REMOTE_PATH (${remote})" exit 1 fi -fi -echo "*** checking rclone mountpoint: ${rclone_mountpoint}" -while findmnt "${rclone_mountpoint}" | grep -q fuse.rclone; do - echo "ERROR: rclone mountpoint (${rclone_mountpoint}) already mounted" - fusermount -uz "${rclone_mountpoint}" - echo "Retrying in 30s ..." - sleep 30s + if [[ ! "${remote}" == :* ]]; then + rclone_remote="$(echo "$remote" | cut -d: -f1):" + if ! rclone listremotes | grep -q "${rclone_remote}"; then + echo "ERROR: Invalid rclone remote name (${rclone_remote})" + echo "ERROR: Available rclone remotes ($(rclone listremotes | xargs))" + exit 1 + fi + fi +done +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + echo ">>> checking rclone mountpoint: ${mountpoint}" + while findmnt "$mountpoint" | grep -q fuse.rclone; do + echo "ERROR: rclone mountpoint (${mountpoint}) already mounted" + fusermount -uz "$mountpoint" + echo "Retrying in 30s ..." + sleep 30s + done done # checking redefined args redefined="" -for opt in $rclone_mount_basic_opts; do - opt="${opt%=*}" - case " $RCLONE_MOUNT_USER_OPTS " in *" ${opt}="* | *" ${opt} "* ) redefined="$redefined ${opt}";; esac +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + opts="${value##*|}" + for opt in $opts; do + opt="${opt%=*}" + case " $RCLONE_MOUNT_USER_OPTS " in *" ${opt}="* | *" ${opt} "* ) redefined="$redefined ${opt}";; esac + done done if [ -n "${redefined}" ]; then echo "*** ERROR: Some of the arguments you have specified are already built-in. Please remove them from the RCLONE_MOUNT_USER_OPTS environment variable:${redefined}" @@ -49,17 +124,68 @@ if [ -n "${redefined}" ]; then fi # checking duplicates in RCLONE_REFRESH_CRON@ -crons=() -for cron in "${!RCLONE_REFRESH_CRON@}"; do - crons+=("${!cron}") -done +# ---------------------------------------------------------------------- +# DYNAMIC CRON DUPLICATE CHECK (Adapted to original format) +# ---------------------------------------------------------------------- +declare -a crons # Array เพื่อเก็บ "Cron Value|Remote Index" +declare -a all_var_names # Array เพื่อเก็บชื่อตัวแปรที่เกี่ยวข้อง + +# 1. วนลูปผ่าน Environment Variables เพื่อสร้าง "คีย์ผสม" +while IFS='=' read -r name value; do + # ตรวจสอบเฉพาะตัวแปร RCLONE_REFRESH_CRON* + [[ "$name" =~ ^RCLONE_REFRESH_CRON(_[0-9]+)?(_[A-Za-z0-9]+)?$ ]] || continue + + if [[ -n "$value" ]]; then + # ดึง Remote Index + remote_num="${BASH_REMATCH[1]}" + index_num="${remote_num#_}" + index_num="${index_num:-1}" # ถ้าว่างเปล่า ให้ใช้ 1 + + # สร้างคีย์ผสม (Key): CronValue|RemoteIndex + check_key="${value}|${index_num}" + + # เก็บทั้งคีย์และชื่อตัวแปร + crons+=("$check_key") + all_var_names+=("$name") + fi +done < <(printenv) + + +# 2. ใช้ AWK ตรวจสอบความซ้ำซ้อนในคีย์ผสม +# ndup คือจำนวนคีย์ที่ไม่ซ้ำกัน ndup=$(printf '%s\n' "${crons[@]}"|awk '!($0 in seen){seen[$0];c++} END {print c}') + +# 3. ตรวจสอบว่ามีจำนวนซ้ำกันหรือไม่ if [ ${#crons[@]} -ne 0 ] && [ ${#crons[@]} -ne "$ndup" ]; then - echo "*** ERROR: found duplicates in your RCLONE_REFRESH_CRON*" - while IFS= read -r dup; do - printf " - \"%s\" -> " "$dup" - for cron in "${!RCLONE_REFRESH_CRON@}"; do - [ "$dup" = "${!cron}" ] && printf '%s ' "${cron}" + echo "*** ERROR: found duplicates in your RCLONE_REFRESH_CRON settings." + + # 4. ใช้ AWK เดิมเพื่อหาค่าที่ซ้ำซ้อน + # AWK นี้จะแสดงเฉพาะค่า (คีย์ผสม) ที่ซ้ำออกมา + while IFS= read -r dup_key; do + # แยก Cron Value และ Remote Index + IFS='|' read -r dup_cron_value dup_remote_index <<< "$dup_key" + + printf " - Remote Index %s: \"%s\" is used by -> " "$dup_remote_index" "$dup_cron_value" + + # 5. วนลูปผ่านชื่อตัวแปรทั้งหมดเพื่อหาตัวแปรที่ตรงกับคีย์ซ้ำนี้ + i=0 + for name in "${all_var_names[@]}"; do + # สร้างคีย์ผสมจากชื่อตัวแปรปัจจุบันเพื่อเปรียบเทียบ + current_cron_value="${!name}" # ดึงค่า Cron Expression + + [[ "$name" =~ ^RCLONE_REFRESH_CRON(_[0-9]+)?(_[A-Za-z0-9]+)?$ ]] + current_remote_num="${BASH_REMATCH[1]}" + current_index_num="${current_remote_num#_}" + current_index_num="${current_index_num:-1}" + + current_check_key="${current_cron_value}|${current_index_num}" + + # เปรียบเทียบกับคีย์ที่ซ้ำ + if [ "$dup_key" = "$current_check_key" ]; then + printf '%s ' "$name" + fi + + i=$((i+1)) done printf "\n" done <<< "$(printf '%s\n' "${crons[@]}"|awk '!($0 in seen){seen[$0];next} 1')" @@ -67,8 +193,7 @@ if [ ${#crons[@]} -ne 0 ] && [ ${#crons[@]} -ne "$ndup" ]; then fi # permissions -chown -R abc:abc \ - /config /log /cloud /data +chown -R abc:abc "${ALL_DIRS_FOR_CHOWN[@]}" chown abc:abc /cache /local echo "*** rclone v$(rclone version | head -n1 | cut -f2 -dv) ready!" diff --git a/root/etc/s6-overlay/s6-rc.d/cron/run b/root/etc/s6-overlay/s6-rc.d/cron/run index 0d96df1..7833ba8 100644 --- a/root/etc/s6-overlay/s6-rc.d/cron/run +++ b/root/etc/s6-overlay/s6-rc.d/cron/run @@ -1,4 +1,5 @@ #!/usr/bin/with-contenv bash +. /usr/local/bin/variables s6-svwait -U /var/run/service/mount @@ -15,13 +16,31 @@ if [[ -n "${MOVE_LOCAL_CRON:-}" ]]; then fi # any ENVs starting with RCLONE_REFRESH_CRON -for cron in "${!RCLONE_REFRESH_CRON@}"; do - dirs="RCLONE_REFRESH_DIRS${cron//RCLONE_REFRESH_CRON/}" - if [[ -n "${!cron}" ]] && [[ -n "${!dirs}" ]]; then - echo "*** scheduling job: vfs/refresh for '${!dirs}'" - go-cron "${!cron}" /usr/local/bin/rclone_refresh "${!dirs}" & +while IFS='=' read -r name value; do + [[ "$name" =~ ^RCLONE_REFRESH_CRON(_[0-9]+)?(_[A-Za-z0-9]+)?$ ]] || continue + remote_num="${BASH_REMATCH[1]}" + job_name="${BASH_REMATCH[2]}" + + dirs_var="RCLONE_REFRESH_DIRS${remote_num}${job_name}" + remote_key="RCLONE_REMOTE_PATH${remote_num}" + + if [[ -n "$value" ]] && [[ -n "${!dirs_var:-}" ]] && [[ -n "${RCLONE_PATHS[$remote_key]:-}" ]]; then + + rc_path_value="${RCLONE_PATHS[$remote_key]}" + opts="${rc_path_value##*|}" + + rc_port=$(echo "$opts" | grep -oP '\-\-rc\-addr=:\K[0-9]+' || true) + + if [[ -z "$rc_port" ]]; then + echo "!!! ERROR: Skipping cron for remote $remote_key. Missing --rc-addr=:PORT in options." + continue + fi + + echo "*** scheduling job: vfs/refresh [${job_name}] for '${!dirs_var}' on RC port $rc_port" + + go-cron "$value" /usr/local/bin/rclone_refresh "${!dirs_var}" "$rc_port" & sleep 2 fi -done +done < <(printenv) exec wait diff --git a/root/etc/s6-overlay/s6-rc.d/mount/data/check b/root/etc/s6-overlay/s6-rc.d/mount/data/check index 4977358..c46f14e 100644 --- a/root/etc/s6-overlay/s6-rc.d/mount/data/check +++ b/root/etc/s6-overlay/s6-rc.d/mount/data/check @@ -2,9 +2,13 @@ . /usr/local/bin/variables set -eo pipefail -if ! findmnt "$rclone_mountpoint" | grep -q fuse.rclone; then - exit 1 -fi +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + if ! findmnt "$mountpoint" | grep -q fuse.rclone; then + exit 1 + fi +done trap run_exit_commands EXIT trap 'run_commands "${POST_COMMANDS_FAILURE:-}"; exit 1' ERR diff --git a/root/etc/s6-overlay/s6-rc.d/mount/finish b/root/etc/s6-overlay/s6-rc.d/mount/finish index cb9a4be..58888e6 100644 --- a/root/etc/s6-overlay/s6-rc.d/mount/finish +++ b/root/etc/s6-overlay/s6-rc.d/mount/finish @@ -1,7 +1,15 @@ #!/usr/bin/with-contenv bash +. /usr/local/bin/variables unmount() { if findmnt "$1" | grep -q fuse; then fusermount -uz "$1" ; fi } # unmount fuse unmount /data -[[ -n "${POOLING_FS:-}" ]] && unmount /cloud +echo ">>> unmounting rclone mountpoint: /data" +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + echo ">>> unmounting rclone mountpoint: ${mountpoint}" + + unmount "${mountpoint}" +done diff --git a/root/etc/s6-overlay/s6-rc.d/mount/run b/root/etc/s6-overlay/s6-rc.d/mount/run index 2ff3afa..9548bd1 100644 --- a/root/etc/s6-overlay/s6-rc.d/mount/run +++ b/root/etc/s6-overlay/s6-rc.d/mount/run @@ -3,7 +3,38 @@ run_commands "${PRE_COMMANDS:-}" -mount_command="rclone mount ${RCLONE_REMOTE_PATH} ${rclone_mountpoint} $(echo $rclone_mount_basic_opts) $RCLONE_MOUNT_USER_OPTS" -echo "*** mounting => $mount_command" -exec s6-notifyoncheck -d -s 3000 \ - s6-setuidgid abc $mount_command +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + remote="${value%%|*}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + opts="${value##*|}" + + mount_command="rclone mount ${remote} ${mountpoint} $(echo $opts) ${RCLONE_MOUNT_USER_OPTS:-}" + echo "*** mounting => $mount_command" + + s6-setuidgid abc $mount_command & +done + +echo "*** run: waiting for all rclone mounts to be ready before calling check..." + +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + + ATTEMPTS=0 + until findmnt "$mountpoint" | grep -q fuse.rclone; do + if [ $ATTEMPTS -ge 60 ]; then + echo "ERROR: Mountpoint $mountpoint failed to appear after 60 seconds." >&2 + exit 1 + fi + echo " - RUN: Waiting for mountpoint: $mountpoint (Attempt $ATTEMPTS)..." + sleep 1 + ATTEMPTS=$((ATTEMPTS + 1)) + done + echo " - RUN: Mountpoint $mountpoint is ready." +done + +echo "*** run: all rclone mounts are ready. Notifying s6 to call check." +s6-notifyoncheck true & + +wait diff --git a/root/etc/s6-overlay/s6-rc.d/serve/run b/root/etc/s6-overlay/s6-rc.d/serve/run index fe5243d..f8c385d 100644 --- a/root/etc/s6-overlay/s6-rc.d/serve/run +++ b/root/etc/s6-overlay/s6-rc.d/serve/run @@ -9,6 +9,21 @@ case "$RCLONE_SERVE_MODE" in ;; esac -serve_command="rclone serve $RCLONE_SERVE_MODE ${rclone_mountpoint} ${RCLONE_SERVE_ARGS:---use-mmap --addr=:5678}" -echo "*** serving => $serve_command" -exec s6-setuidgid abc $serve_command +START_PORT="${RCLONE_SERVE_START_PORT:-5678}" +CURRENT_PORT=$START_PORT +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + remote="${value%%|*}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + opts="${value##*|}" + + serve_command="rclone serve $RCLONE_SERVE_MODE ${mountpoint} ${RCLONE_SERVE_ARGS:---use-mmap} --addr=:$CURRENT_PORT" + echo "*** serving => $serve_command" + s6-setuidgid abc $serve_command & + + CURRENT_PORT=$((CURRENT_PORT + 1)) +done + +echo "*** s6: Waiting for all rclone serve processes to finish..." + +wait diff --git a/root/usr/local/bin/healthcheck b/root/usr/local/bin/healthcheck index 8f3942a..04f8285 100644 --- a/root/usr/local/bin/healthcheck +++ b/root/usr/local/bin/healthcheck @@ -1,9 +1,14 @@ #!/usr/bin/with-contenv bash . /usr/local/bin/variables -if [ "$(findmnt "${rclone_mountpoint}" | grep fuse.rclone -c)" -ne 1 ]; then - exit 1 -fi +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + + if [ "$(findmnt "${mountpoint}" | grep fuse.rclone -c)" -ne 1 ]; then + exit 1 + fi +done if [ "$POOLING_FS" == "mergerfs" ]; then if [ "$(findmnt /data | grep fuse.mergerfs -c)" -ne 1 ]; then diff --git a/root/usr/local/bin/rclone_refresh.script b/root/usr/local/bin/rclone_refresh.script index dc68283..f552867 100644 --- a/root/usr/local/bin/rclone_refresh.script +++ b/root/usr/local/bin/rclone_refresh.script @@ -17,20 +17,9 @@ set -euo pipefail # ----------------------------------------------------------------------------- logf() { printf '%s REFRES: %s\n' "$(date "$DATE_FORMAT")" "$*"; } -# ----------------------------------------------------------------------------- -# Prevent concurrent execution using flock -# ----------------------------------------------------------------------------- -LOCKFILE="/tmp/$(basename "$0").lock" -exec 9>"$LOCKFILE" -if ! flock -n 9; then - logf "Already running. Skipping new refresh request." - exit 3 -fi - # ----------------------------------------------------------------------------- # Refresh logic # ----------------------------------------------------------------------------- -refresh_cmd_base=(rclone rc vfs/refresh --url 127.0.0.1:5574 --fast-list --timeout 300m) has_content() { local path="$1" @@ -41,18 +30,18 @@ do_refresh() { local args=("$@") local rc_out if ! rc_out="$("${refresh_cmd_base[@]}" "${args[@]}" 2>&1)"; then - logf "rclone rc command failed: $rc_out" + logf "[RC:$RC_PORT] rclone rc command failed: $rc_out" return 1 fi # parse and log result safely local jq_out if ! jq_out=$(jq -r -c 'try (.result | to_entries[] | [.key, .value] | @tsv) catch "parse error"' <<<"$rc_out" 2>/dev/null); then - logf "jq parse failed or invalid rclone output" + logf "[RC:$RC_PORT] jq parse failed or invalid rclone output" return 1 fi while IFS=$'\t' read -r dir result; do - [[ -n "$dir" ]] && logf "\"${dir}\": ${result}" + [[ -n "$dir" ]] && logf "[RC:$RC_PORT] \"${dir}\": ${result}" done <<<"$jq_out" } @@ -72,7 +61,7 @@ do_deep_refresh() { while IFS= read -r -d $'\0' subpath; do if ! has_content "$subpath"; then subdir="$RDIR/$(basename "$subpath")" - logf "\"$subdir\": appears empty - refreshing it" + logf "[RC:$RC_PORT] \"$subdir\": appears empty - refreshing it" do_refresh "dir=$subdir" "recursive=true" || true fi done < <(find "$mount_path" -maxdepth 1 -mindepth 1 -type d -print0) @@ -82,14 +71,26 @@ do_deep_refresh() { # ----------------------------------------------------------------------------- # Main logic # ----------------------------------------------------------------------------- -if [[ $# -lt 1 ]]; then +if [[ $# -lt 2 ]]; then exit 0 fi +RC_PORT="$2" +refresh_cmd_base=(rclone rc vfs/refresh --url 127.0.0.1:$RC_PORT --fast-list --timeout 300m) + +# ----------------------------------------------------------------------------- +# Prevent concurrent execution using flock +# ----------------------------------------------------------------------------- +LOCKFILE="/tmp/$(basename "$0")_$RC_PORT.lock" +exec 9>"$LOCKFILE" +if ! flock -n 9; then + logf "[RC:$RC_PORT] Already running. Skipping new refresh request." + exit 3 +fi stime=$(date +%s) if [[ "$1" == "." || "$1" == "/" ]]; then - logf ">>> refreshing root" + logf "[RC:$RC_PORT] >>> refreshing root" if [[ "$RCLONE_REFRESH_METHOD" == "default" || ! -d "$mount_root" ]]; then do_refresh "recursive=true" || true @@ -102,7 +103,7 @@ if [[ "$1" == "." || "$1" == "/" ]]; then done < <(find "$mount_root" -maxdepth 1 -mindepth 1 -type d -print0) fi else - logf ">>> refreshing \"$1\"" + logf "[RC:$RC_PORT] >>> refreshing \"$1\"" # replace all newlines with |, then split on | IFS="|" read -r -a raw_dirs <<< "${1//$'\n'/|}" @@ -147,4 +148,4 @@ etime=$(( $(date +%s) - stime )) hh=$((etime / 3600)) mm=$(((etime % 3600) / 60)) ss=$(((etime % 3600) % 60)) -logf "<<< done in $(printf "%02d:%02d:%02d\n" "$hh" "$mm" "$ss")" +logf "[RC:$RC_PORT] <<< done in $(printf "%02d:%02d:%02d\n" "$hh" "$mm" "$ss")" diff --git a/root/usr/local/bin/variables b/root/usr/local/bin/variables index dd3a3f1..64e35c8 100644 --- a/root/usr/local/bin/variables +++ b/root/usr/local/bin/variables @@ -1,18 +1,63 @@ #!/usr/bin/with-contenv bash -[[ -n "${POOLING_FS:-}" ]] && rclone_mountpoint="/cloud" || rclone_mountpoint="/data" - -rclone_mount_basic_opts=" ---uid=${PUID:-911} ---gid=${PGID:-911} ---cache-dir=/cache ---use-mmap ---allow-other ---umask=002 ---rc ---rc-no-auth ---rc-addr=:5574 -" +base_mount="/data" +[[ -n "${POOLING_FS:-}" ]] && base_mount="/cloud" + +rclone_mountpoint="$base_mount" + +declare -gA RCLONE_PATHS +index=0 + +while IFS='=' read -r name value; do + [[ "$name" =~ ^RCLONE_REMOTE_PATH(_[0-9]+)?$ ]] || continue + num="${BASH_REMATCH[1]}" + num="${num#_}" + mountpoint="${base_mount}${num:+$num}" + port=$((5574 + index)) + + rclone_mount_basic_opts="\ + --uid=${PUID:-911} \ + --gid=${PGID:-911} \ + --cache-dir=/cache \ + --use-mmap \ + --allow-other \ + --umask=002 \ + --rc \ + --rc-no-auth \ + --rc-addr=:${port} + " + + opts="${RCLONE_REMOTE_OPTS[$num]:-$rclone_mount_basic_opts}" + + RCLONE_PATHS["$name"]="${value}|${mountpoint}|${opts}" + ((index++)) +done < <(printenv) + +if (( index == 0 )) && [[ -n "${RCLONE_REMOTE_PATH:-}" ]]; then + rclone_mount_basic_opts="\ + --uid=${PUID:-911} \ + --gid=${PGID:-911} \ + --cache-dir=/cache \ + --use-mmap \ + --allow-other \ + --umask=002 \ + --rc \ + --rc-no-auth \ + --rc-addr=:5574 + " + + RCLONE_PATHS["RCLONE_REMOTE_PATH"]="${RCLONE_REMOTE_PATH}|${base_mount}|${rclone_mount_basic_opts}" +fi + +for key in $(printf '%s\n' "${!RCLONE_PATHS[@]}" | sort -t_ -k4n); do + value="${RCLONE_PATHS[$key]}" + mountpoint="${value#*|}"; mountpoint="${mountpoint%%|*}" + + if [[ -n "$mountpoint" ]]; then + UFS_BRANCHES+="${mountpoint}=RO:" + MFS_BRANCHES+="${mountpoint}=NC:" + fi +done mfs_basic_opts="uid=${PUID:-911},gid=${PGID:-911},umask=022,allow_other" ufs_basic_opts="uid=${PUID:-911},gid=${PGID:-911},umask=022,allow_other"