diff --git a/.gitignore b/.gitignore index 72491bd..b47a87d 100644 --- a/.gitignore +++ b/.gitignore @@ -221,3 +221,6 @@ docs # HuRI client outputs *.wav +/.huri-local/ +/assets/ +config/*.generated.yaml diff --git a/README.md b/README.md index 6bb36fa..acf7965 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,81 @@ Speech-to-Text (STT) and Text-to-Speech (TTS), Retrieval-Augmented Generation (R pip install -r requirements.txt ``` +#### Local install (single machine, no Kubernetes) + +`scripts/install_local.sh` is the bare-metal counterpart of the Docker images in +`deploy/` and the Helm chart in `helm/`. It probes the machine (GPU vendor, VRAM, +RAM, disk, Python), works out which parts of the pipeline actually fit, and then +installs only those: system packages, a virtualenv with the right torch build, +the model weights, Qdrant and Ollama, plus a Ray Serve config matching the plan. + +```sh +scripts/install_local.sh --plan-only # what would run on this machine? +scripts/install_local.sh # show the plan, then install it +``` + +The plan decides per module — TTS and gesture generation need an NVIDIA GPU, +STT runs fine on CPU, and the LLM tier is picked from the VRAM (or RAM) left +over. Modules that do not fit are dropped from `HURI_MODULES`, so no Serve +deployment is created for them. Override with `--force-tts`, `--force-gesture`, +`--llm-model`, `--stt-model`, … (`--help` lists everything). + +It generates: + +| Path | What | +| --- | --- | +| `config/huri_local.generated.yaml` | Ray Serve config for this machine | +| `config/client_local.generated.yaml` | matching client config | +| `.huri-local/huri.env` | the same runtime env vars as a plain `.env` | +| `.huri-local/secrets.env` | LLM API key, `0600`, never in the configs | +| `.huri-local/env.sh` | venv + `huri.env` + `secrets.env` in one `source` | +| `.huri-local/start.sh` / `stop.sh` / `status.sh` | run the stack | +| `.huri-local/plan.env` | the hardware plan it acted on | + +```sh +.huri-local/start.sh # services + ray head + serve deploy +source .huri-local/env.sh +python -m src.client --config config/client_local.generated.yaml +.huri-local/status.sh # what is up, locally and remotely +.huri-local/stop.sh --all +``` + +`source .huri-local/env.sh` gives a shell configured exactly like a Serve +replica (model paths, `HURI_MODULES`, TTS/gesture tuning, API key), so +`python -m src.launch_huri` and `python -m src.modules.rag.ingestion` work +without `serve deploy`. + +TTS additionally needs a reference voice sample: +`--voice-sample voice.wav --voice-transcript "exactly what is said in it"`. + +#### Using a remote LLM instead of a local one + +Any endpoint that is not on localhost is treated as already running: it is not +installed, not started, and costs no local VRAM — which is what frees a small +GPU to run TTS and gesture generation. + +```sh +# self-hosted vLLM over HTTPS with a private CA / self-signed cert +scripts/install_local.sh \ + --llm-url https://llm.example.lan --llm-provider vllm \ + --llm-model Qwen3.5-4B-GGUF --no-verify-ssl \ + --embed-url https://embedding.example.lan --embed-model bge-large-en-v1.5-gguf-Q4_K_M + +# hosted OpenAI-compatible API (key goes to .huri-local/secrets.env, 0600) +scripts/install_local.sh \ + --llm-url https://api.example.com --llm-provider api \ + --llm-model some-model --llm-api-key "$MY_KEY" + +# remote Qdrant too — then nothing but HuRI itself runs locally +scripts/install_local.sh --qdrant-url https://qdrant.example.lan +``` + +The provider selects the wire protocol (`ollama` → `/api/chat`, `vllm`/`api` → +`/v1/chat/completions`, only `api` sends the bearer token) and is inferred from +the URL and whether a key was given. Re-point an existing install without +reinstalling anything: `scripts/install_local.sh --only config --llm-url ... --llm-model ...` +then restart (`.huri-local/stop.sh && .huri-local/start.sh`). + ## Usage #### Launch HuRI server: diff --git a/constraints.txt b/constraints.txt index 835369d..17d65e2 100644 --- a/constraints.txt +++ b/constraints.txt @@ -10,3 +10,17 @@ # google-api-core / googleapis-common-protos / proto-plus / opentelemetry-proto / # google-cloud-storage, so it satisfies the whole dependency graph. protobuf==4.25.8 + +# fastapi: Ray Serve's @serve.ingress does pickle_dumps(app) to check the ASGI +# app is serializable. FastAPI 0.139.2 added a threading.Lock to its internal +# route-cache dataclass, so that check now fails with +# TypeError: cannot pickle '_thread.lock' object +# on every FastAPI app, even a bare one. No fix released yet. +# https://github.com/ray-project/ray/issues/64939 +fastapi<0.139.2 + +# setuptools: pkg_resources was removed outright in setuptools 82.0.0 +# (2026-02-08). webrtcvad==2.0.10 still does `import pkg_resources` at import +# time, so anything newer breaks STT startup with +# ModuleNotFoundError: No module named 'pkg_resources' +setuptools<82 diff --git a/deploy/examples/local_nvidia_amd/values.yaml b/deploy/examples/local_nvidia_amd/values.yaml index e980c57..1139d5d 100644 --- a/deploy/examples/local_nvidia_amd/values.yaml +++ b/deploy/examples/local_nvidia_amd/values.yaml @@ -74,6 +74,12 @@ ray: llm_provider: "vllm" llm_model: "Qwen3.5-4B-GGUF" verify_ssl: false + # Widened from the 3-attempt/1s-base default (~3s total) so a + # single dropped packet on the LAN to *.huri.lan doesn't surface + # as a user-facing error — see incident 2026-09-03 19:28:50. + embed_retries: 5 + llm_retries: 5 + retry_base_delay: 1.5 # GPU split (manual override knob): TTS and Gesture share one NVIDIA # GPU. num_gpus/resources are Ray *scheduling* fractions — they let # both replicas pack onto the same device and bias the split. Audio diff --git a/requirements-nvidia.txt b/requirements-nvidia.txt index 9ff2e43..e8288cd 100644 --- a/requirements-nvidia.txt +++ b/requirements-nvidia.txt @@ -21,9 +21,10 @@ einops==0.8.2 tiktoken==0.13.0 # cosyvoice/tokenizer pyarrow==18.1.0 # imported by cli paths via dataset utils? actually only dataset/processor — can drop protobuf==4.25.8 # keep <5 so Ray Serve's _proto_to_dict works; 4.25.8 satisfies onnxruntime>=4.25.8 (also enforced by constraints.txt) -pydantic==2.7.0 # transitive (transformers/fastapi), but pinning avoids drift +pydantic==2.13.5 # transitive (transformers/fastapi); fastapi<0.139.2 (see constraints.txt) requires pydantic>=2.9.0 regex==2025.11.3 tqdm==4.67.3 +rich==13.7.1 # cosyvoice/flow/flow_matching.py -> matcha.utils.rich_utils # --- RAG / LLM extras --- httpx==0.27.2 diff --git a/scripts/install_local.sh b/scripts/install_local.sh new file mode 100755 index 0000000..c30c061 --- /dev/null +++ b/scripts/install_local.sh @@ -0,0 +1,1870 @@ +#!/usr/bin/env bash +# ============================================================================= +# HuRI — local (bare-metal) installer +# ============================================================================= +# +# Installs and configures the whole HuRI stack on a single machine, without +# Kubernetes, Helm or Docker images. It is the local counterpart of: +# +# deploy/Dockerfile.{base,nvidia,amd} → the Python environment +# helm/templates/*-model-init-job.yaml → the model weight downloads +# deploy/examples/*/values.yaml → the Ray Serve config +# +# Before installing anything it *probes the machine* (GPU vendor, VRAM, RAM, +# disk, CPU, Python) and computes which parts of the pipeline can actually run +# here. Components that do not fit are moved to CPU or dropped from the module +# set, and the generated Ray Serve config reflects that decision. +# +# Usage: +# scripts/install_local.sh --plan-only # just tell me what would run here +# scripts/install_local.sh # probe, show the plan, install +# scripts/install_local.sh --yes # no prompts +# scripts/install_local.sh --help +# +# Everything it generates lives in .huri-local/ plus config/*.generated.yaml, +# so the install is inspectable and removable (`rm -rf .huri-local`). +# ============================================================================= + +set -Eeuo pipefail + +# --- Paths ------------------------------------------------------------------- + +SCRIPT_PATH="$(readlink -f "${BASH_SOURCE[0]}")" +REPO_ROOT="$(dirname "$(dirname "$SCRIPT_PATH")")" +STATE_DIR="$REPO_ROOT/.huri-local" +ASSETS_DIR="$REPO_ROOT/assets" +MODELS_DIR="$ASSETS_DIR/models" +VENV_DIR="$REPO_ROOT/.venv" +LOG_FILE="$STATE_DIR/install.log" + +# --- Pinned versions (kept in sync with deploy/) ----------------------------- + +# deploy/Dockerfile.nvidia +COSYVOICE_REPO="https://github.com/FunAudioLLM/CosyVoice.git" +COSYVOICE_COMMIT="074ca6dc9e80a2f424f1f74b48bdd7d3fea531cc" +TORCH_CUDA_VERSION="2.3.1" # requirements-nvidia.txt +TORCH_CUDA_INDEX="https://download.pytorch.org/whl/cu121" +TORCH_CPU_INDEX="https://download.pytorch.org/whl/cpu" +# deploy/Dockerfile.amd +ROCM_VERSION="7.2" +ROCM_TORCH_WHEEL="torch-2.8.0+rocm7.2.0.lw.gitbf943426" +ROCM_TORCHAUDIO_WHEEL="torchaudio-2.8.0+rocm7.2.0.git6e1c7fe9" +ROCM_TRITON_VERSION="3.4.0+rocm7.2.0.git0cace8d2" +CT2_ROCM_URL="https://github.com/OpenNMT/CTranslate2/releases/download/v4.7.1/rocm-python-wheels-Linux.zip" + +# helm/templates/*-model-init-job.yaml + deploy/examples/*/values.yaml +COSYTTS_MODEL_ID="FunAudioLLM/Fun-CosyVoice3-0.5B-2512" # modelscope +EMAGE_REPO_ID="H-Liu1997/emage_audio" # huggingface +EMOTION_REPO_ID="superb/hubert-large-superb-er" # huggingface +WHISPER_REPO_PREFIX="Systran/faster-whisper" # + - + +QDRANT_VERSION="v1.12.4" +QDRANT_IMAGE="qdrant/qdrant:$QDRANT_VERSION" + +# ============================================================================= +# Resource cost model +# ============================================================================= +# All figures in MiB, steady-state, measured/estimated on the pinned versions. +# They are deliberately a little pessimistic: the point is to refuse an install +# that would OOM at the first utterance, not to squeeze the last megabyte. +# +# TTS CosyVoice3-0.5B: LM (0.5B) + flow-matching + HiFi vocoder + the +# zero-shot prompt cache. fp16 on CUDA (the code enables it when +# CUDA is present), fp32 otherwise. +# GESTURE EMAGE audio: 4 VQ-VAEs + global AE + the audio transformer, plus +# the sliding-window activations. +# STT faster-whisper via CTranslate2, incl. beam/encoder buffers. +# EMO hubert-large prosody classifier — always CPU (the module never +# moves tensors to a device), and it is instantiated *per session*. +# LLM/EMB Ollama, Q4_K_M weights + KV cache at the default context. + +VRAM_TTS_FP16=4500 +VRAM_TTS_FP32=7000 +VRAM_GESTURE=2200 +declare -A VRAM_STT=( [base]=1000 [small]=1600 [medium]=3200 [large-v3]=6000 ) +declare -A RAM_STT=( [base]=700 [small]=1100 [medium]=2400 [large-v3]=4500 ) +RAM_TTS_CPU=6000 +RAM_GESTURE_CPU=2500 +RAM_EMO_PER_SESSION=1600 +RAM_BASE=3000 # ray head + serve controller + proxy + HuRI actor + +# LLM tiers, largest first: "||" +LLM_TIERS=( + "qwen2.5:14b|9500|11000" + "mistral:7b|5500|6500" + "llama3.2:3b|2800|3600" + "qwen2.5:1.5b|1600|2200" +) +EMBED_MODEL_DEFAULT="bge-m3" +VRAM_EMBED=1300 + +# Rough on-disk footprint (MiB) used for the free-space check. +DISK_VENV_BASE=2500 +DISK_TORCH_CUDA=5000 +DISK_TORCH_ROCM=7000 +DISK_TORCH_CPU=900 +DISK_COSYVOICE=6000 # repo + submodules + model snapshot +DISK_EMAGE=1500 +DISK_EMOTION=1300 +declare -A DISK_STT=( [base]=200 [small]=600 [medium]=1800 [large-v3]=3300 ) +DISK_SERVICES=3000 # qdrant image/binary + ollama runtime + +# ============================================================================= +# Options +# ============================================================================= + +DRY_RUN=0 +PLAN_ONLY=0 +ASSUME_YES=0 +PROFILE="auto" +GPU_INDEX=0 +VRAM_OVERRIDE="" +RESERVE_VRAM=700 +STT_SIZE="base" +LLM_MODEL="" +EMBED_MODEL="$EMBED_MODEL_DEFAULT" +LLM_URL="http://localhost:11434" +LLM_PROVIDER="" +LLM_API_KEY="" +EMBED_URL="" +QDRANT_URL="http://localhost:6333" +VERIFY_SSL=1 +VOICE_SAMPLE="" +VOICE_TRANSCRIPT="Instinct creates its own oppressors and bids us rise up against them." +PYTHON_BIN="" +FORCE_TTS=0 +FORCE_GESTURE=0 +FORCE_EMO=0 +SKIP_SYSTEM=0 +SKIP_PYTHON=0 +SKIP_MODELS=0 +SKIP_SERVICES=0 +ONLY_STAGES="" + +usage() { + cat <<'USAGE' +HuRI local installer — installs the full stack on this machine (no Kubernetes). + + scripts/install_local.sh [options] + +Planning + -n, --dry-run Print every command instead of running it. + --plan-only Probe the machine, print the capability plan, exit. + --profile P auto | nvidia | amd | cpu (default: auto) + --gpu-index N Which GPU to budget against (default: 0) + --vram MB Override detected VRAM (for GPUs the tools can't read) + --reserve-vram MB VRAM left free for driver/context (default: 700) + --stt-model SIZE base | small | medium | large-v3 (default: base) + --llm-model TAG Model name; default is an Ollama tag picked from the + free VRAM/RAM. Required with a remote --llm-url. + --embed-model TAG Embedding model (default: bge-m3) + +Remote endpoints (anything not on localhost is treated as already running: +no Ollama/Qdrant is installed for it, and it costs no local VRAM/RAM) + --llm-url URL LLM base URL, http(s) (default: http://localhost:11434) + --llm-provider P ollama | vllm | api (default: inferred from the URL + and whether an API key was given) + --llm-api-key KEY Bearer token. Stored in .huri-local/secrets.env (0600) + and exported as HURI_LLM_API_KEY — never written into + the generated config. + --embed-url URL Embeddings base URL (default: same as --llm-url) + --qdrant-url URL Qdrant URL (default: http://localhost:6333) + --no-verify-ssl Skip TLS verification (self-signed internal endpoints) + --force-tts Keep TTS even when the plan puts it on CPU + --force-gesture Keep gesture generation even when it lands on CPU + --force-emo Keep prosody/emotion even when RAM is tight + +Install + -y, --yes Don't ask for confirmation + --python PATH Interpreter used to create the venv (needs 3.10–3.12) + --venv PATH Virtualenv location (default: ./.venv) + --voice-sample PATH Reference voice sample for TTS -> assets/voice.wav + --voice-transcript T Its exact transcript (CosyVoice3 zero-shot prompt) + --skip-system Don't touch system packages (apt/dnf/pacman) + --skip-python Don't create/populate the virtualenv + --skip-models Don't download model weights + --skip-services Don't install/start Qdrant and Ollama + --only S[,S...] Run only these stages: + system,python,models,services,config,verify + -h, --help This message + +Outputs + .huri-local/ plan.env, env.sh, start.sh, stop.sh, status.sh + config/huri_local.generated.yaml Ray Serve config for this machine + config/client_local.generated.yaml matching client config +USAGE +} + +while [[ $# -gt 0 ]]; do + case "$1" in + -n|--dry-run) DRY_RUN=1 ;; + --plan-only) PLAN_ONLY=1 ;; + -y|--yes) ASSUME_YES=1 ;; + --profile) PROFILE="${2:?}"; shift ;; + --gpu-index) GPU_INDEX="${2:?}"; shift ;; + --vram) VRAM_OVERRIDE="${2:?}"; shift ;; + --reserve-vram) RESERVE_VRAM="${2:?}"; shift ;; + --stt-model) STT_SIZE="${2:?}"; shift ;; + --llm-model) LLM_MODEL="${2:?}"; shift ;; + --embed-model) EMBED_MODEL="${2:?}"; shift ;; + --llm-url) LLM_URL="${2:?}"; shift ;; + --llm-provider) LLM_PROVIDER="${2:?}"; shift ;; + --llm-api-key) LLM_API_KEY="${2:?}"; shift ;; + --embed-url) EMBED_URL="${2:?}"; shift ;; + --qdrant-url) QDRANT_URL="${2:?}"; shift ;; + --no-verify-ssl) VERIFY_SSL=0 ;; + --voice-sample) VOICE_SAMPLE="${2:?}"; shift ;; + --voice-transcript) VOICE_TRANSCRIPT="${2:?}"; shift ;; + --python) PYTHON_BIN="${2:?}"; shift ;; + --venv) VENV_DIR="$(readlink -f "${2:?}")"; shift ;; + --force-tts) FORCE_TTS=1 ;; + --force-gesture) FORCE_GESTURE=1 ;; + --force-emo) FORCE_EMO=1 ;; + --skip-system) SKIP_SYSTEM=1 ;; + --skip-python) SKIP_PYTHON=1 ;; + --skip-models) SKIP_MODELS=1 ;; + --skip-services) SKIP_SERVICES=1 ;; + --only) ONLY_STAGES="${2:?}"; shift ;; + -h|--help) usage; exit 0 ;; + *) echo "unknown option: $1" >&2; usage >&2; exit 2 ;; + esac + shift +done + +[[ -n "${VRAM_STT[$STT_SIZE]:-}" ]] || { echo "unknown --stt-model '$STT_SIZE'" >&2; exit 2; } +case "$PROFILE" in auto|nvidia|amd|cpu) ;; *) echo "unknown --profile '$PROFILE'" >&2; exit 2 ;; esac + +# An API key given on an earlier run lives in .huri-local/secrets.env; reload it +# so `--only config` re-runs keep working without re-passing the secret. +if [[ -z "$LLM_API_KEY" && -r "$STATE_DIR/secrets.env" ]]; then + # shellcheck disable=SC1091 + source "$STATE_DIR/secrets.env" + LLM_API_KEY="${HURI_LLM_API_KEY:-}" +fi + +# rag.py falls back to llm_url when embedding_url is empty; mirror that here so +# the plan and the generated config agree on which endpoint is used. +EMBED_URL_EFF="${EMBED_URL:-$LLM_URL}" + +if [[ -z "$LLM_PROVIDER" ]]; then + # /api/chat (ollama) vs /v1/chat/completions (vllm, api). Only "api" sends the + # Authorization header, so a key implies it. See rag.py::_llm_stream. + if [[ "$LLM_URL" == *:11434* ]]; then LLM_PROVIDER="ollama" + elif [[ -n "$LLM_API_KEY" ]]; then LLM_PROVIDER="api" + else LLM_PROVIDER="vllm" + fi +fi +case "$LLM_PROVIDER" in ollama|vllm|api) ;; *) echo "unknown --llm-provider '$LLM_PROVIDER'" >&2; exit 2 ;; esac + +# ============================================================================= +# Output helpers +# ============================================================================= + +if [[ -t 1 ]]; then + C_RST=$'\033[0m'; C_B=$'\033[1m'; C_DIM=$'\033[2m' + C_R=$'\033[31m'; C_G=$'\033[32m'; C_Y=$'\033[33m'; C_C=$'\033[36m' +else + C_RST=; C_B=; C_DIM=; C_R=; C_G=; C_Y=; C_C= +fi + +_log() { [[ -d "$STATE_DIR" ]] && printf '%s %s\n' "$(date -Is)" "$*" >>"$LOG_FILE" || true; } +info() { printf '%s\n' "$*"; _log "INFO $*"; } +ok() { printf ' %s✓%s %s\n' "$C_G" "$C_RST" "$*"; _log "OK $*"; } +warn() { printf ' %s!%s %s\n' "$C_Y" "$C_RST" "$*"; _log "WARN $*"; } +err() { printf ' %s✗%s %s\n' "$C_R" "$C_RST" "$*" >&2; _log "ERR $*"; } +die() { err "$*"; exit 1; } +step() { printf '\n%s==>%s %s%s%s\n' "$C_C" "$C_RST" "$C_B" "$*" "$C_RST"; _log "STEP $*"; } +note() { printf ' %s%s%s\n' "$C_DIM" "$*" "$C_RST"; } + +# run — echo in dry-run, execute otherwise (stdout/stderr also logged). +run() { + if (( DRY_RUN )); then + printf ' %s$ %s%s\n' "$C_DIM" "$(printf '%q ' "$@")" "$C_RST" + return 0 + fi + _log "RUN $*" + "$@" +} + +# Write a file (respecting --dry-run). Content on stdin. +write_file() { + local path="$1" + if (( DRY_RUN )); then + printf ' %s$ write %s%s\n' "$C_DIM" "$path" "$C_RST" + cat >/dev/null + return 0 + fi + mkdir -p "$(dirname "$path")" + cat >"$path" + _log "WRITE $path" +} + +have() { command -v "$1" >/dev/null 2>&1; } + +# is_local_url — true when the URL points at this machine, i.e. when the +# installer is responsible for providing the service behind it. +is_local_url() { + local host + host="$(printf '%s' "$1" | sed -E 's#^[a-zA-Z][a-zA-Z0-9+.-]*://##; s#^[^/@]*@##; s#[/?].*##; s#:[0-9]+$##; s#^\[|\]$##g')" + case "$host" in + localhost|127.0.0.1|0.0.0.0|::1|"$(hostname)"|"$(hostname -s 2>/dev/null)") return 0 ;; + *) return 1 ;; + esac +} + +confirm() { + local prompt="$1" + (( ASSUME_YES )) && return 0 + (( DRY_RUN )) && return 0 + local reply + read -r -p "$prompt [y/N] " reply || true + [[ "$reply" == [yY]* ]] +} + +stage_enabled() { + [[ -z "$ONLY_STAGES" ]] && return 0 + [[ ",$ONLY_STAGES," == *",$1,"* ]] +} + +mb_to_gb() { awk -v m="$1" 'BEGIN{printf "%.1f", m/1024}'; } + +trap 'err "failed at line $LINENO (see $LOG_FILE)"' ERR + +# ============================================================================= +# 1. Detection +# ============================================================================= + +OS_NAME=""; OS_VERSION=""; PKG_MGR=""; IS_WSL=0 +CPU_CORES=0; RAM_TOTAL_MB=0; RAM_FREE_MB=0; DISK_FREE_MB=0 +GPU_VENDOR="none"; GPU_NAME=""; GPU_COUNT=0; GPU_VRAM_MB=0; GPU_DRIVER="" +PY_BIN=""; PY_VERSION="" +HAS_DOCKER=""; HAS_OLLAMA=0 + +detect_os() { + if [[ -r /etc/os-release ]]; then + # shellcheck disable=SC1091 + OS_NAME="$(. /etc/os-release && echo "${ID:-unknown}")" + OS_VERSION="$(. /etc/os-release && echo "${VERSION_ID:-}")" + else + OS_NAME="$(uname -s)" + fi + grep -qiE 'microsoft|wsl' /proc/version 2>/dev/null && IS_WSL=1 || true + if have apt-get; then PKG_MGR="apt" + elif have dnf; then PKG_MGR="dnf" + elif have pacman; then PKG_MGR="pacman" + elif have zypper; then PKG_MGR="zypper" + else PKG_MGR="" + fi +} + +detect_host() { + CPU_CORES="$(nproc 2>/dev/null || echo 1)" + RAM_TOTAL_MB=$(( $(awk '/MemTotal/{print $2}' /proc/meminfo 2>/dev/null || echo 0) / 1024 )) + RAM_FREE_MB=$(( $(awk '/MemAvailable/{print $2}' /proc/meminfo 2>/dev/null || echo 0) / 1024 )) + DISK_FREE_MB=$(( $(df -Pk "$REPO_ROOT" 2>/dev/null | awk 'NR==2{print $4}' || echo 0) / 1024 )) +} + +detect_nvidia() { + have nvidia-smi || return 1 + local out + out="$(nvidia-smi --query-gpu=index,name,memory.total,driver_version \ + --format=csv,noheader,nounits 2>/dev/null)" || return 1 + [[ -n "$out" ]] || return 1 + GPU_COUNT="$(printf '%s\n' "$out" | grep -c .)" + local line + line="$(printf '%s\n' "$out" | awk -F', *' -v i="$GPU_INDEX" '$1+0==i{print;exit}')" + [[ -n "$line" ]] || die "no NVIDIA GPU with index $GPU_INDEX (found $GPU_COUNT)" + GPU_VENDOR="nvidia" + GPU_NAME="$(awk -F', *' '{print $2}' <<<"$line")" + GPU_VRAM_MB="$(awk -F', *' '{print int($3)}' <<<"$line")" + GPU_DRIVER="$(awk -F', *' '{print $4}' <<<"$line")" + return 0 +} + +detect_amd() { + local bytes="" + if have amd-smi; then + bytes="$(amd-smi static --json 2>/dev/null \ + | grep -oE '"(total|size)"[[:space:]]*:[[:space:]]*[0-9]+' | head -1 \ + | grep -oE '[0-9]+' || true)" + fi + if [[ -z "$bytes" ]] && have rocm-smi; then + # Output shape moves between ROCm releases; take the first big integer that + # follows a "Total" VRAM label, in bytes. + bytes="$(rocm-smi --showmeminfo vram 2>/dev/null \ + | grep -iE 'vram total memory' | grep -oE '[0-9]{7,}' | head -1 || true)" + fi + if [[ -z "$bytes" ]]; then + # No ROCm tooling: is there an AMD display/compute device at all? + have lspci && lspci 2>/dev/null | grep -qiE 'VGA|3D|Display' \ + && lspci 2>/dev/null | grep -iE 'VGA|3D|Display' | grep -qi 'AMD/ATI' || return 1 + GPU_VENDOR="amd"; GPU_COUNT=1 + GPU_NAME="$(lspci 2>/dev/null | grep -iE 'VGA|3D|Display' | grep -i 'AMD/ATI' | head -1 | cut -d':' -f3- | sed 's/^ //')" + GPU_VRAM_MB=0 + return 0 + fi + GPU_VENDOR="amd"; GPU_COUNT=1 + GPU_VRAM_MB=$(( bytes / 1024 / 1024 )) + GPU_NAME="$(have rocm-smi && rocm-smi --showproductname 2>/dev/null | grep -iE 'card series|product name' | head -1 | cut -d':' -f2- | sed 's/^ *//' || true)" + [[ -n "$GPU_NAME" ]] || GPU_NAME="AMD GPU" + return 0 +} + +detect_gpu() { + case "$PROFILE" in + cpu) GPU_VENDOR="none"; return ;; + nvidia) detect_nvidia || die "--profile nvidia but nvidia-smi reports no usable GPU" ;; + amd) detect_amd || die "--profile amd but no AMD GPU found (install rocm-smi, or pass --vram)" ;; + auto) detect_nvidia || detect_amd || GPU_VENDOR="none" ;; + esac + if [[ -n "$VRAM_OVERRIDE" ]]; then + GPU_VRAM_MB="$VRAM_OVERRIDE" + [[ "$GPU_VENDOR" == "none" ]] && GPU_VENDOR="${PROFILE}" + fi +} + +detect_python() { + local candidates=() + [[ -n "$PYTHON_BIN" ]] && candidates+=("$PYTHON_BIN") + [[ -x "$VENV_DIR/bin/python" ]] && candidates+=("$VENV_DIR/bin/python") + candidates+=(python3.12 python3.11 python3.10 python3) + local c v major minor + for c in "${candidates[@]}"; do + have "$c" || [[ -x "$c" ]] || continue + v="$("$c" -c 'import sys;print("%d.%d.%d"%sys.version_info[:3])' 2>/dev/null)" || continue + major="${v%%.*}"; minor="$(cut -d. -f2 <<<"$v")" + if [[ "$major" == 3 ]] && (( minor >= 10 && minor <= 12 )); then + PY_BIN="$(command -v "$c" || echo "$c")"; PY_VERSION="$v"; return 0 + fi + done + return 1 +} + +detect_tools() { + if have docker && docker info >/dev/null 2>&1; then HAS_DOCKER="docker" + elif have podman; then HAS_DOCKER="podman" + else HAS_DOCKER=""; fi + have ollama && HAS_OLLAMA=1 || HAS_OLLAMA=0 +} + +report_detection() { + step "Machine" + printf ' %-14s %s %s\n' "os" "$OS_NAME" "$OS_VERSION$( ((IS_WSL)) && echo ' (WSL2)')" + printf ' %-14s %s cores, %s GiB RAM (%s GiB available)\n' "cpu/ram" \ + "$CPU_CORES" "$(mb_to_gb "$RAM_TOTAL_MB")" "$(mb_to_gb "$RAM_FREE_MB")" + printf ' %-14s %s GiB free at %s\n' "disk" "$(mb_to_gb "$DISK_FREE_MB")" "$REPO_ROOT" + if [[ "$GPU_VENDOR" == "none" ]]; then + printf ' %-14s %snone detected%s — CPU-only plan\n' "gpu" "$C_Y" "$C_RST" + else + printf ' %-14s %s x%s, %s GiB VRAM%s\n' "gpu" \ + "${GPU_NAME:-$GPU_VENDOR}" "${GPU_COUNT:-1}" "$(mb_to_gb "$GPU_VRAM_MB")" \ + "${GPU_DRIVER:+, driver $GPU_DRIVER}" + fi + printf ' %-14s %s\n' "python" "${PY_BIN:-} ${PY_VERSION:+($PY_VERSION)}" + printf ' %-14s %s\n' "container" "${HAS_DOCKER:-none}" + printf ' %-14s %s\n' "ollama" "$( ((HAS_OLLAMA)) && echo installed || echo 'not installed')" + + if [[ "$GPU_VENDOR" == "amd" && $IS_WSL == 1 ]]; then + warn "ROCm is not usable under WSL2 — planning as CPU-only." + GPU_VENDOR="none" + fi + if [[ "$GPU_VENDOR" != "none" && "$GPU_VRAM_MB" -eq 0 ]]; then + if [[ "$PROFILE" == "auto" ]]; then + warn "found a $GPU_VENDOR GPU but no tooling reports its VRAM — planning as CPU-only." + note "Install the vendor tools (rocm-smi / nvidia-smi), or re-run with --vram ." + GPU_VENDOR="none" + else + die "found a $GPU_VENDOR GPU but could not read its VRAM — re-run with --vram " + fi + fi +} + +# ============================================================================= +# 2. Planning — what can this machine actually run? +# ============================================================================= + +# Per-component decisions filled in by plan(). +P_STT_DEV=""; P_STT_WHY="" +P_TTS_DEV=""; P_TTS_WHY="" +P_GES_DEV=""; P_GES_WHY="" +P_EMO_DEV=""; P_EMO_WHY="" +P_LLM_DEV=""; P_LLM_WHY="" +P_TTS_VRAM=0; P_GES_VRAM=0; P_STT_VRAM=0; P_LLM_VRAM=0; P_LLM_RAM=0 +P_VERDICT=""; P_MODULES=""; P_RAM_NEED=0; P_DISK_NEED=0 +P_EMBED_DEV=""; LLM_REMOTE=0; EMBED_REMOTE=0; QDRANT_REMOTE=0; NEED_OLLAMA=0 +P_TTS_FRAC="0"; P_GES_FRAC="0"; P_STT_FRAC="0"; P_RAY_GPUS=0 + +# frac — that component's share of the GPU, as a Ray num_gpus fraction +# rounded up to 2 decimals (Ray fractions are a scheduling/packing hint). +# frac_ceil — round up: used for HURI_GESTURE_GPU_MEM_FRACTION, a hard +# allocation cap, where rounding down would starve the model it is sizing. +frac_ceil() { + awk -v m="$1" -v t="$GPU_VRAM_MB" 'BEGIN{ + if (t <= 0) { print "0"; exit } + f = (int(m * 100 / t) + 1) / 100 + if (f > 1) f = 1 + printf "%.2f", f + }' +} + +frac() { + awk -v m="$1" -v t="$GPU_VRAM_MB" 'BEGIN{ + if (t <= 0) { print "0"; exit } + f = int(m * 100 / t) / 100 # floor: the shares already sum to < 1 + if (f < 0.01) f = 0.01 + if (f > 1) f = 1 + printf "%.2f", f + }' +} + +# Which endpoints this machine has to host itself. +resolve_endpoints() { + is_local_url "$LLM_URL" && LLM_REMOTE=0 || LLM_REMOTE=1 + is_local_url "$EMBED_URL_EFF" && EMBED_REMOTE=0 || EMBED_REMOTE=1 + is_local_url "$QDRANT_URL" && QDRANT_REMOTE=0 || QDRANT_REMOTE=1 + NEED_OLLAMA=0 + (( LLM_REMOTE )) || NEED_OLLAMA=1 + (( EMBED_REMOTE )) || NEED_OLLAMA=1 + if (( LLM_REMOTE )) && [[ -z "$LLM_MODEL" ]]; then + die "--llm-url points at $LLM_URL — pass --llm-model " + fi + # Only the "api" provider attaches the bearer token (rag.py::_llm_stream), so + # a key with any other provider would silently never be sent. + if [[ -n "$LLM_API_KEY" && "$LLM_PROVIDER" != "api" ]]; then + warn "an API key is set but --llm-provider is '$LLM_PROVIDER', which sends no" + note "Authorization header. Use --llm-provider api if the endpoint needs the key." + fi +} + +plan() { + local pool=0 + if [[ "$GPU_VENDOR" != "none" ]]; then + pool=$(( GPU_VRAM_MB - RESERVE_VRAM )) + (( pool < 0 )) && pool=0 + fi + + # --- TTS (CosyVoice3) ------------------------------------------------------ + # ROCm is deliberately excluded: requirements-amd.txt states CosyVoice2/3 and + # EMAGE run on the NVIDIA worker only, so we do not pretend otherwise here. + local tts_cost=$VRAM_TTS_FP16 + if [[ "$GPU_VENDOR" == "nvidia" ]] && (( pool >= tts_cost )); then + P_TTS_DEV="gpu"; P_TTS_VRAM=$tts_cost; pool=$(( pool - tts_cost )) + P_TTS_WHY="fp16 on $GPU_NAME" + elif [[ "$GPU_VENDOR" == "amd" ]]; then + if (( FORCE_TTS )); then + P_TTS_DEV="cpu"; P_TTS_WHY="forced; ROCm build has no CosyVoice stack (see requirements-amd.txt)" + else + P_TTS_DEV="off"; P_TTS_WHY="CosyVoice is not supported on ROCm (--force-tts runs it on CPU)" + fi + elif (( FORCE_TTS )); then + P_TTS_DEV="cpu"; P_TTS_WHY="forced onto CPU — expect several seconds per sentence" + else + P_TTS_DEV="off" + if [[ "$GPU_VENDOR" == "none" ]]; then + P_TTS_WHY="no GPU; CPU synthesis is far slower than realtime (--force-tts to try)" + else + P_TTS_WHY="needs $(mb_to_gb $tts_cost) GiB VRAM, only $(mb_to_gb $pool) GiB left" + fi + fi + + # --- LLM (Ollama) ---------------------------------------------------------- + # Picked before gesture but *after* reserving gesture's slice when both fit: + # a smaller LLM that keeps gestures beats a bigger one that kills them. + local reserve_ges=0 tier tag tvram tram + if [[ "$GPU_VENDOR" == "nvidia" ]] && (( pool >= VRAM_GESTURE + 1600 )); then + reserve_ges=$VRAM_GESTURE + fi + # Embeddings only cost local VRAM when Ollama is the one serving them. + local embed_cost=0 + (( EMBED_REMOTE )) || embed_cost=$VRAM_EMBED + local llm_budget=$(( pool - reserve_ges - embed_cost )) + (( llm_budget < 0 )) && llm_budget=0 + + # RAM left for a CPU-served LLM once the rest of the stack is accounted for. + # STT is charged at its CPU cost and EMO at one session even if they later land + # elsewhere — over-reserving here only makes the choice safer. + local cpu_other=$(( RAM_BASE + RAM_STT[$STT_SIZE] + RAM_EMO_PER_SESSION )) + (( FORCE_TTS )) && cpu_other=$(( cpu_other + RAM_TTS_CPU )) + (( FORCE_GESTURE )) && cpu_other=$(( cpu_other + RAM_GESTURE_CPU )) + local llm_ram_budget=$(( RAM_TOTAL_MB - cpu_other - 1024 )) # 1 GiB margin + (( llm_ram_budget < 0 )) && llm_ram_budget=0 + + if (( LLM_REMOTE )); then + # Someone else's GPU: no local budget, and the freed VRAM stays available + # to TTS/gesture above. + P_LLM_DEV="remote" + P_LLM_WHY="$LLM_PROVIDER endpoint $LLM_URL" + [[ -n "$LLM_API_KEY" ]] && P_LLM_WHY="$P_LLM_WHY (key from secrets.env)" + elif [[ -n "$LLM_MODEL" ]]; then + # User-chosen tag: honour it, budget with the closest known tier. + tvram=5500; tram=6500 + for tier in "${LLM_TIERS[@]}"; do + IFS='|' read -r tag tvram tram <<<"$tier" + [[ "$tag" == "$LLM_MODEL" ]] && break + tvram=5500; tram=6500 + done + if [[ "$GPU_VENDOR" == "nvidia" ]] && (( llm_budget >= tvram )); then + P_LLM_DEV="gpu"; P_LLM_VRAM=$tvram; pool=$(( pool - tvram - embed_cost )) + P_LLM_WHY="user-selected, fits in VRAM" + else + P_LLM_DEV="cpu"; P_LLM_RAM=$tram + P_LLM_WHY="user-selected; served from RAM by Ollama" + (( tram > llm_ram_budget )) && P_LLM_WHY="user-selected; $(mb_to_gb $tram) GiB needed but only $(mb_to_gb $llm_ram_budget) GiB spare RAM" + fi + else + for tier in "${LLM_TIERS[@]}"; do + IFS='|' read -r tag tvram tram <<<"$tier" + if [[ "$GPU_VENDOR" == "nvidia" ]] && (( llm_budget >= tvram )); then + LLM_MODEL="$tag"; P_LLM_DEV="gpu"; P_LLM_VRAM=$tvram + pool=$(( pool - tvram - embed_cost )) + P_LLM_WHY="largest tier fitting the remaining VRAM" + break + fi + done + if [[ -z "$LLM_MODEL" ]]; then + # CPU inference: pick by the RAM left after the rest of the stack. + for tier in "${LLM_TIERS[@]}"; do + IFS='|' read -r tag tvram tram <<<"$tier" + if (( llm_ram_budget >= tram )); then + LLM_MODEL="$tag"; P_LLM_DEV="cpu"; P_LLM_RAM=$tram + P_LLM_WHY="no VRAM headroom; largest tier fitting $(mb_to_gb $llm_ram_budget) GiB spare RAM" + break + fi + done + fi + fi + # Embeddings: served by the same local Ollama, or by the remote endpoint. + if (( EMBED_REMOTE )); then + P_EMBED_DEV="remote" + elif [[ "$GPU_VENDOR" == "nvidia" ]] && (( pool >= embed_cost )); then + P_EMBED_DEV="gpu" + (( LLM_REMOTE )) && pool=$(( pool - embed_cost )) # not yet charged above + else + P_EMBED_DEV="cpu" + fi + + if [[ -z "$LLM_MODEL" ]]; then + LLM_MODEL="qwen2.5:1.5b"; P_LLM_DEV="off" + if (( FORCE_TTS || FORCE_GESTURE )); then + P_LLM_WHY="no RAM left: --force-tts/--force-gesture reserved it for CPU inference" + else + P_LLM_WHY="not enough RAM for any tier — RAG cannot answer" + fi + fi + + # --- Gesture (EMAGE) ------------------------------------------------------- + if [[ "$GPU_VENDOR" == "nvidia" ]] && (( pool >= VRAM_GESTURE )); then + P_GES_DEV="gpu"; P_GES_VRAM=$VRAM_GESTURE; pool=$(( pool - VRAM_GESTURE )) + P_GES_WHY="shares the GPU with TTS (memory fraction capped at runtime)" + elif (( FORCE_GESTURE )); then + P_GES_DEV="cpu"; P_GES_WHY="forced onto CPU — motion will lag the audio" + else + P_GES_DEV="off" + if [[ "$GPU_VENDOR" == "nvidia" ]]; then + P_GES_WHY="needs $(mb_to_gb $VRAM_GESTURE) GiB VRAM, only $(mb_to_gb $pool) GiB left" + elif [[ "$GPU_VENDOR" == "amd" ]]; then + P_GES_WHY="EMAGE is not part of the ROCm build (--force-gesture runs it on CPU)" + else + P_GES_WHY="no GPU; CPU inference is slower than realtime (--force-gesture to try)" + fi + fi + + # --- STT (faster-whisper) -------------------------------------------------- + # CPU int8 is genuinely realtime for base/small, so STT only takes GPU when + # there is room left over. On ROCm it is the one component that *is* supported. + local stt_cost="${VRAM_STT[$STT_SIZE]}" + if [[ "$GPU_VENDOR" == "amd" ]] && (( pool >= stt_cost )); then + P_STT_DEV="gpu"; P_STT_VRAM=$stt_cost; pool=$(( pool - stt_cost )) + P_STT_WHY="CTranslate2 ROCm build" + elif [[ "$GPU_VENDOR" == "nvidia" ]] && (( pool >= stt_cost )); then + P_STT_DEV="gpu"; P_STT_VRAM=$stt_cost; pool=$(( pool - stt_cost )) + P_STT_WHY="spare VRAM after TTS/LLM/gesture" + else + P_STT_DEV="cpu"; P_STT_WHY="int8 on $CPU_CORES cores — realtime for '$STT_SIZE'" + fi + + # --- Emotion (hubert-large, CPU, per session) ------------------------------ + local emo_headroom=$(( RAM_TOTAL_MB - RAM_BASE - RAM_EMO_PER_SESSION )) + if (( emo_headroom >= 2000 )) || (( FORCE_EMO )); then + P_EMO_DEV="cpu"; P_EMO_WHY="hubert-large, ~$(mb_to_gb $RAM_EMO_PER_SESSION) GiB RAM per session" + else + P_EMO_DEV="off"; P_EMO_WHY="needs ~$(mb_to_gb $RAM_EMO_PER_SESSION) GiB RAM per session (--force-emo to keep)" + fi + + # --- Module allow-list (HURI_MODULES) ------------------------------------- + local mods=("mic" "stt" "tag" "qag" "rag") + [[ "$P_EMO_DEV" != "off" ]] && mods+=("emo" "eag") + [[ "$P_TTS_DEV" != "off" ]] && mods+=("tts") + [[ "$P_GES_DEV" != "off" && "$P_TTS_DEV" != "off" ]] && mods+=("gesture") + if [[ "$P_GES_DEV" != "off" && "$P_TTS_DEV" == "off" ]]; then + P_GES_DEV="off"; P_GES_WHY="gesture is driven by TTS audio, which is disabled" + fi + P_MODULES="$(IFS=,; echo "${mods[*]}")" + + # --- Ray resources --------------------------------------------------------- + if [[ "$GPU_VENDOR" != "none" ]]; then + [[ "$P_TTS_DEV" == "gpu" ]] && P_TTS_FRAC="$(frac "$P_TTS_VRAM")" + [[ "$P_GES_DEV" == "gpu" ]] && P_GES_FRAC="$(frac "$P_GES_VRAM")" + [[ "$P_STT_DEV" == "gpu" ]] && P_STT_FRAC="$(frac "$P_STT_VRAM")" + P_RAY_GPUS=1 + fi + + # --- RAM / disk requirement ------------------------------------------------ + P_RAM_NEED=$RAM_BASE + [[ "$P_STT_DEV" == "cpu" ]] && P_RAM_NEED=$(( P_RAM_NEED + RAM_STT[$STT_SIZE] )) + [[ "$P_TTS_DEV" == "cpu" ]] && P_RAM_NEED=$(( P_RAM_NEED + RAM_TTS_CPU )) + [[ "$P_GES_DEV" == "cpu" ]] && P_RAM_NEED=$(( P_RAM_NEED + RAM_GESTURE_CPU )) + [[ "$P_EMO_DEV" == "cpu" ]] && P_RAM_NEED=$(( P_RAM_NEED + RAM_EMO_PER_SESSION )) + [[ "$P_LLM_DEV" == "cpu" ]] && P_RAM_NEED=$(( P_RAM_NEED + P_LLM_RAM )) + + P_DISK_NEED=$(( DISK_VENV_BASE + DISK_STT[$STT_SIZE] )) + case "$GPU_VENDOR" in + nvidia) P_DISK_NEED=$(( P_DISK_NEED + DISK_TORCH_CUDA )) ;; + amd) P_DISK_NEED=$(( P_DISK_NEED + DISK_TORCH_ROCM )) ;; + *) P_DISK_NEED=$(( P_DISK_NEED + DISK_TORCH_CPU )) ;; + esac + [[ "$P_TTS_DEV" != "off" ]] && P_DISK_NEED=$(( P_DISK_NEED + DISK_COSYVOICE )) + [[ "$P_GES_DEV" != "off" ]] && P_DISK_NEED=$(( P_DISK_NEED + DISK_EMAGE )) + [[ "$P_EMO_DEV" != "off" ]] && P_DISK_NEED=$(( P_DISK_NEED + DISK_EMOTION )) + if (( ! SKIP_SERVICES )); then + if (( ! QDRANT_REMOTE )); then P_DISK_NEED=$(( P_DISK_NEED + 200 )); fi + if (( NEED_OLLAMA )); then P_DISK_NEED=$(( P_DISK_NEED + DISK_SERVICES )); fi + fi + + # --- Verdict --------------------------------------------------------------- + if [[ "$P_LLM_DEV" == "off" ]]; then + P_VERDICT="blocked" + elif [[ "$P_TTS_DEV" != "off" && "$P_GES_DEV" != "off" ]]; then + P_VERDICT="full" + elif [[ "$P_TTS_DEV" != "off" ]]; then + P_VERDICT="voice" + else + P_VERDICT="text" + fi +} + +# row . The device colour is +# applied around a pre-padded field so escape codes never shift the columns. +row() { + local color + case "$3" in + gpu) color="$C_G" ;; + cpu) color="$C_Y" ;; + off) color="$C_R" ;; + remote) color="$C_C" ;; + *) color="" ;; + esac + printf ' %-9s %-26s %s%-7s%s %-12s %s\n' "$1" "$2" "$color" "$3" "$C_RST" "$4" "$5" +} + +report_plan() { + # Budget shown per component: VRAM when it lands on the GPU, otherwise the + # RAM it will occupy on the host (— when it is not loaded at all). + local stt_budget llm_budget tts_budget ges_budget emo_budget + if [[ "$P_STT_DEV" == "gpu" ]]; then stt_budget="$(mb_to_gb "$P_STT_VRAM") GiB vram" + else stt_budget="$(mb_to_gb "${RAM_STT[$STT_SIZE]}") GiB ram"; fi + case "$P_LLM_DEV" in + gpu) llm_budget="$(mb_to_gb "$P_LLM_VRAM") GiB vram" ;; + cpu) llm_budget="$(mb_to_gb "$P_LLM_RAM") GiB ram" ;; + *) llm_budget="-" ;; + esac + local llm_backend="ollama $LLM_MODEL" + [[ "$P_LLM_DEV" == "remote" ]] && llm_backend="$LLM_PROVIDER $LLM_MODEL" + local embed_backend="ollama $EMBED_MODEL" + [[ "$P_EMBED_DEV" == "remote" ]] && embed_backend="$EMBED_MODEL" + local mem_backend="qdrant $QDRANT_VERSION" mem_dev="cpu" mem_why + if (( QDRANT_REMOTE )); then + mem_backend="qdrant (remote)"; mem_dev="remote"; mem_why="$QDRANT_URL" + elif [[ -n "$HAS_DOCKER" ]]; then + mem_why="via $HAS_DOCKER" + else + mem_why="standalone binary" + fi + case "$P_TTS_DEV" in + gpu) tts_budget="$(mb_to_gb "$P_TTS_VRAM") GiB vram" ;; + cpu) tts_budget="$(mb_to_gb "$RAM_TTS_CPU") GiB ram" ;; + *) tts_budget="-" ;; + esac + case "$P_GES_DEV" in + gpu) ges_budget="$(mb_to_gb "$P_GES_VRAM") GiB vram" ;; + cpu) ges_budget="$(mb_to_gb "$RAM_GESTURE_CPU") GiB ram" ;; + *) ges_budget="-" ;; + esac + [[ "$P_EMO_DEV" == "cpu" ]] && emo_budget="$(mb_to_gb "$RAM_EMO_PER_SESSION") GiB ram" || emo_budget="-" + + step "Capability plan" + printf ' %-9s %-26s %-7s %-12s %s\n' "MODULE" "BACKEND" "DEVICE" "BUDGET" "WHY" + printf ' %s\n' "$(printf '─%.0s' {1..96})" + row "stt" "faster-whisper $STT_SIZE" "$P_STT_DEV" "$stt_budget" "$P_STT_WHY" + row "rag/llm" "$llm_backend" "$P_LLM_DEV" "$llm_budget" "$P_LLM_WHY" + row "tts" "CosyVoice3-0.5B" "$P_TTS_DEV" "$tts_budget" "$P_TTS_WHY" + row "gesture" "EMAGE audio" "$P_GES_DEV" "$ges_budget" "$P_GES_WHY" + row "emo" "hubert-large-superb-er" "$P_EMO_DEV" "$emo_budget" "$P_EMO_WHY" + row "embed" "$embed_backend" "$P_EMBED_DEV" \ + "$([[ "$P_EMBED_DEV" == "gpu" ]] && echo "$(mb_to_gb $VRAM_EMBED) GiB vram" || echo "-")" \ + "$( (( EMBED_REMOTE )) && echo "$EMBED_URL_EFF" || echo "OpenAI-compatible /v1/embeddings" )" + row "memory" "$mem_backend" "$mem_dev" "-" "$mem_why" + + echo + printf ' %-22s %s\n' "modules registered" "$P_MODULES" + printf ' %-22s %s GiB needed / %s GiB present\n' "ram" "$(mb_to_gb $P_RAM_NEED)" "$(mb_to_gb $RAM_TOTAL_MB)" + printf ' %-22s %s GiB needed / %s GiB free\n' "disk" "$(mb_to_gb $P_DISK_NEED)" "$(mb_to_gb $DISK_FREE_MB)" + if [[ "$GPU_VENDOR" != "none" ]]; then + printf ' %-22s ray --num-gpus=%s · fractions tts=%s gesture=%s stt=%s\n' \ + "gpu scheduling" "$P_RAY_GPUS" "$P_TTS_FRAC" "$P_GES_FRAC" "$P_STT_FRAC" + fi + + echo + case "$P_VERDICT" in + full) printf ' %sVERDICT: full pipeline%s — voice in, voice + gesture out.\n' "$C_G$C_B" "$C_RST" ;; + voice) printf ' %sVERDICT: voice pipeline%s — voice in, voice out, no gesture.\n' "$C_G$C_B" "$C_RST" ;; + text) printf ' %sVERDICT: text pipeline%s — speech or text in, text out (no local TTS).\n' "$C_Y$C_B" "$C_RST" ;; + blocked) printf ' %sVERDICT: cannot run here%s — not enough RAM for even the smallest LLM.\n' "$C_R$C_B" "$C_RST" ;; + esac +} + +check_hard_limits() { + local fatal=0 + if (( RAM_TOTAL_MB < P_RAM_NEED )); then + err "plan needs $(mb_to_gb $P_RAM_NEED) GiB RAM, machine has $(mb_to_gb $RAM_TOTAL_MB) GiB" + fatal=1 + fi + if (( DISK_FREE_MB < P_DISK_NEED )); then + err "plan needs $(mb_to_gb $P_DISK_NEED) GiB free disk, only $(mb_to_gb $DISK_FREE_MB) GiB available" + fatal=1 + fi + [[ "$P_VERDICT" == "blocked" ]] && fatal=1 + if (( fatal )); then + echo + note "Options: drop --force-tts/--force-gesture (they reserve CPU RAM)," + note "--stt-model base, --llm-model qwen2.5:1.5b, --skip-services, or free disk space." + return 1 + fi + return 0 +} + +save_plan() { + write_file "$STATE_DIR/plan.env" < — reuse the repo's pinned version + local pkg="$1" file="$2" line + line="$(grep -iE "^[[:space:]]*${pkg}[=<>~]" "$REPO_ROOT/$file" 2>/dev/null | head -1 | sed 's/[[:space:]]*#.*//')" + printf '%s\n' "${line:-$pkg}" | tr -d ' ' +} + +create_venv() { + if [[ -x "$VENV_DIR/bin/python" ]]; then + local existing + existing="$("$VENV_DIR/bin/python" -c 'import sys;print("%d.%d"%sys.version_info[:2])' 2>/dev/null || echo "?")" + case "$existing" in + 3.10|3.11|3.12) ok "virtualenv exists: $VENV_DIR (python $existing)"; return 0 ;; + *) die "$VENV_DIR runs python $existing (need 3.10–3.12) — delete it, or pass --venv " ;; + esac + fi + info " creating virtualenv at $VENV_DIR (python $PY_VERSION)" + run "$PY_BIN" -m venv "$VENV_DIR" +} + +install_python_deps() { + stage_enabled python || return 0 + (( SKIP_PYTHON )) && { note "python env skipped (--skip-python)"; return 0; } + step "Python environment" + + create_venv + (( DRY_RUN )) && { VPY="python"; PIP="pip"; } + + run "$PIP" install --upgrade pip setuptools wheel + + local C=(-c "$REPO_ROOT/constraints.txt") + + # 1. Vendor torch FIRST, so every later resolution (sentence-transformers, + # transformers, faster-whisper) sees a satisfying torch and does not pull + # the default PyPI CPU wheel just to have it replaced. + case "$GPU_VENDOR" in + nvidia) + info " torch $TORCH_CUDA_VERSION (cu121)" + run "$PIP" install "${C[@]}" --index-url "$TORCH_CUDA_INDEX" \ + "torch==$TORCH_CUDA_VERSION" "torchaudio==$TORCH_CUDA_VERSION" + ;; + amd) + info " torch 2.8 (ROCm $ROCM_VERSION) from repo.radeon.com" + run "$PIP" install "${C[@]}" \ + "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/triton-${ROCM_TRITON_VERSION}-cp312-cp312-linux_x86_64.whl" + run "$PIP" install "${C[@]}" --extra-index-url https://repo.radeon.com/rocm/pypi/ \ + "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/${ROCM_TORCH_WHEEL}-cp312-cp312-linux_x86_64.whl" \ + "https://repo.radeon.com/rocm/manylinux/rocm-rel-${ROCM_VERSION}/${ROCM_TORCHAUDIO_WHEEL}-cp312-cp312-linux_x86_64.whl" + ;; + none) + info " torch $TORCH_CUDA_VERSION (cpu)" + run "$PIP" install "${C[@]}" --index-url "$TORCH_CPU_INDEX" \ + "torch==$TORCH_CUDA_VERSION" "torchaudio==$TORCH_CUDA_VERSION" + ;; + esac + + # 2. Server + client base (ray[serve], faster-whisper, qdrant, sounddevice…). + info " base requirements" + run "$PIP" install "${C[@]}" -r "$REPO_ROOT/requirements.txt" + + # 3. Per-vendor GPU stack, mirroring deploy/Dockerfile.{amd,nvidia}. + if [[ "$GPU_VENDOR" == "amd" ]]; then + install_ct2_rocm + run "$PIP" install "${C[@]}" -r "$REPO_ROOT/requirements-amd.txt" + fi + + # 4. TTS/gesture extras. On NVIDIA use the pinned file verbatim (same set as + # the image); elsewhere derive a CPU-safe variant from it so the pins stay + # in one place. + if [[ "$P_TTS_DEV" != "off" || "$P_GES_DEV" != "off" || "$P_EMO_DEV" != "off" ]]; then + if [[ "$GPU_VENDOR" == "nvidia" && "$P_TTS_DEV" != "off" ]]; then + info " CosyVoice/EMAGE stack (requirements-nvidia.txt)" + run "$PIP" install "${C[@]}" --extra-index-url "$TORCH_CUDA_INDEX" \ + --extra-index-url https://pypi.ngc.nvidia.com -r "$REPO_ROOT/requirements-nvidia.txt" + else + generate_cpu_requirements + info " inference extras (.huri-local/requirements-local.generated.txt)" + run "$PIP" install "${C[@]}" -r "$STATE_DIR/requirements-local.generated.txt" + fi + fi + + # 5. CosyVoice source tree (no setup.py upstream → clone + PYTHONPATH). + [[ "$P_TTS_DEV" != "off" ]] && install_cosyvoice + + ok "python environment ready" +} + +generate_cpu_requirements() { + # Derived from requirements-nvidia.txt: drop the torch pins (installed above + # from the CPU/ROCm index), swap onnxruntime-gpu for the CPU build, and drop + # the EMAGE *rendering/training* extras — src/modules/gesture/emage only needs + # torch + transformers + omegaconf + huggingface_hub. + local out="$STATE_DIR/requirements-local.generated.txt" p + local drop='^(torch|torchaudio|onnxruntime-gpu|smplx|pyrender|trimesh|imageio|lightning|gdown|wget|pyworld)([=<>~]|$)' + { + echo "# Generated by scripts/install_local.sh from requirements-nvidia.txt." + echo "# Vendor: ${GPU_VENDOR}. Do not edit — re-run the installer instead." + if [[ "$P_TTS_DEV" != "off" ]]; then + grep -vE "$drop" "$REPO_ROOT/requirements-nvidia.txt" \ + | sed 's/[[:space:]]*#.*//' | grep -vE '^[[:space:]]*$' + echo "onnxruntime==1.18.0" + else + # No CosyVoice: gesture + emotion only. + for p in transformers librosa soundfile omegaconf huggingface_hub numpy; do + pin_of "$p" requirements-nvidia.txt + done + fi + } | write_file "$out" +} + +install_ct2_rocm() { + info " CTranslate2 (ROCm wheel)" + local tmp="$STATE_DIR/ct2" + run mkdir -p "$tmp" + run curl -fsSL "$CT2_ROCM_URL" -o "$tmp/ct2-rocm.zip" + run unzip -o -j "$tmp/ct2-rocm.zip" 'temp-linux/ctranslate2-4.7.1-cp312-*manylinux*x86_64.whl' -d "$tmp" + if (( ! DRY_RUN )); then + local whl; whl="$(find "$tmp" -name 'ctranslate2-4.7.1-cp312-*.whl' | head -1)" + [[ -n "$whl" ]] || die "no CTranslate2 ROCm wheel in $CT2_ROCM_URL" + run "$PIP" install "$whl" + fi +} + +install_cosyvoice() { + local dir="$ASSETS_DIR/cosyvoice" + if [[ -d "$dir/.git" ]]; then + ok "CosyVoice checkout present ($dir)" + else + info " cloning CosyVoice @ ${COSYVOICE_COMMIT:0:8}" + run git clone "$COSYVOICE_REPO" "$dir" + fi + run git -C "$dir" fetch --depth 1 origin "$COSYVOICE_COMMIT" || true + run git -C "$dir" checkout "$COSYVOICE_COMMIT" + run git -C "$dir" submodule update --init --recursive +} + +# ============================================================================= +# 5. Stage: model weights (mirrors helm/templates/*-model-init-job.yaml) +# ============================================================================= + +hf_snapshot() { # hf_snapshot + run "$VPY" - "$1" "$2" <<'PY' +import sys +from huggingface_hub import snapshot_download +snapshot_download(sys.argv[1], local_dir=sys.argv[2]) +print("downloaded", sys.argv[1], "->", sys.argv[2]) +PY +} + +download_models() { + stage_enabled models || return 0 + (( SKIP_MODELS )) && { note "model download skipped (--skip-models)"; return 0; } + step "Model weights" + run mkdir -p "$MODELS_DIR" + + # --- STT: faster-whisper (presence marker: model.bin, as in the Helm job) --- + local whisper_repo="${WHISPER_REPO_PREFIX}-${STT_SIZE}" + local whisper_dir="$MODELS_DIR/whisper/$whisper_repo" + if [[ -f "$whisper_dir/model.bin" ]]; then + ok "whisper: already at $whisper_dir" + else + info " whisper $STT_SIZE → $whisper_dir" + hf_snapshot "$whisper_repo" "$whisper_dir" + fi + + # --- TTS: CosyVoice3 from ModelScope (markers: cosyvoice3.yaml + llm.pt) --- + if [[ "$P_TTS_DEV" != "off" ]]; then + local cosy_dir="$MODELS_DIR/cosytts/$COSYTTS_MODEL_ID" + if [[ -f "$cosy_dir/cosyvoice3.yaml" && -f "$cosy_dir/llm.pt" ]]; then + ok "cosyvoice3: already at $cosy_dir" + else + info " CosyVoice3 → $cosy_dir (several GiB)" + run "$VPY" - "$COSYTTS_MODEL_ID" "$cosy_dir" <<'PY' +import sys +from modelscope import snapshot_download +snapshot_download(sys.argv[1], local_dir=sys.argv[2]) +print("downloaded", sys.argv[1], "->", sys.argv[2]) +PY + fi + fi + + # --- Gesture: EMAGE --------------------------------------------------------- + if [[ "$P_GES_DEV" != "off" ]]; then + local emage_dir="$MODELS_DIR/emage/$EMAGE_REPO_ID" + if [[ -d "$emage_dir" && -n "$(ls -A "$emage_dir" 2>/dev/null)" ]]; then + ok "emage: already at $emage_dir" + else + info " EMAGE → $emage_dir" + hf_snapshot "$EMAGE_REPO_ID" "$emage_dir" + fi + fi + + # --- Emotion: prefetched into the HF cache (the module loads it by repo id) -- + if [[ "$P_EMO_DEV" != "off" ]]; then + info " prosody model $EMOTION_REPO_ID → HF cache" + run "$VPY" - "$EMOTION_REPO_ID" <<'PY' +import sys +from huggingface_hub import snapshot_download +snapshot_download(sys.argv[1]) +print("cached", sys.argv[1]) +PY + fi + + # --- Reference voice sample ------------------------------------------------- + if [[ "$P_TTS_DEV" != "off" ]]; then + if [[ -n "$VOICE_SAMPLE" ]]; then + [[ -f "$VOICE_SAMPLE" ]] || die "--voice-sample '$VOICE_SAMPLE' not found" + run cp "$VOICE_SAMPLE" "$ASSETS_DIR/voice.wav" + ok "voice sample installed at $ASSETS_DIR/voice.wav" + elif [[ -f "$ASSETS_DIR/voice.wav" ]]; then + ok "voice sample present at $ASSETS_DIR/voice.wav" + else + warn "no $ASSETS_DIR/voice.wav — TTS will start but zero-shot synthesis needs it." + note "Add one with: scripts/install_local.sh --only models,config \\" + note " --voice-sample /path/to/voice.wav --voice-transcript 'exact words spoken'" + note "(--only config alone will NOT copy the file — 'models' is the stage that does)" + fi + fi + ok "models ready" +} + +# ============================================================================= +# 6. Stage: services (Qdrant + Ollama) +# ============================================================================= + +start_qdrant() { + if curl -fsS --max-time 2 http://localhost:6333/readyz >/dev/null 2>&1; then + ok "qdrant already listening on :6333"; return 0 + fi + local data="$STATE_DIR/qdrant" + run mkdir -p "$data" + if [[ -n "$HAS_DOCKER" ]]; then + info " starting qdrant via $HAS_DOCKER" + if (( ! DRY_RUN )) && "$HAS_DOCKER" ps -a --format '{{.Names}}' 2>/dev/null | grep -qx huri-qdrant; then + run "$HAS_DOCKER" start huri-qdrant + else + run "$HAS_DOCKER" run -d --name huri-qdrant --restart unless-stopped \ + -p 6333:6333 -p 6334:6334 -v "$data:/qdrant/storage" "$QDRANT_IMAGE" + fi + else + info " installing standalone qdrant binary (no container runtime found)" + local url="https://github.com/qdrant/qdrant/releases/download/${QDRANT_VERSION#v}/qdrant-x86_64-unknown-linux-gnu.tar.gz" + run mkdir -p "$STATE_DIR/bin" + if [[ ! -x "$STATE_DIR/bin/qdrant" ]]; then + run curl -fsSL "$url" -o "$STATE_DIR/qdrant.tar.gz" \ + || { warn "qdrant download failed — install it manually, RAG memory stays offline"; return 0; } + run tar -xzf "$STATE_DIR/qdrant.tar.gz" -C "$STATE_DIR/bin" + run chmod +x "$STATE_DIR/bin/qdrant" + fi + # The binary resolves ./config/config.yaml relative to its working directory + # (start.sh runs it from .huri-local), unlike the image which ships one. + write_file "$STATE_DIR/config/config.yaml" </dev/null 2>&1 +} + +pull_ollama_models() { + (( HAS_OLLAMA )) || return 0 + if ! ollama_up; then + info " starting 'ollama serve' in the background" + if (( ! DRY_RUN )); then + nohup ollama serve >"$STATE_DIR/ollama.log" 2>&1 & + local i + for i in $(seq 1 30); do ollama_up && break; sleep 1; done + fi + fi + if ! ollama_up && (( ! DRY_RUN )); then + warn "ollama is not answering on :11434 — pull the models manually:" + note "ollama pull $LLM_MODEL && ollama pull $EMBED_MODEL" + return 0 + fi + if (( ! LLM_REMOTE )); then + info " pulling $LLM_MODEL (several GiB)" + run ollama pull "$LLM_MODEL" + fi + if (( ! EMBED_REMOTE )); then + info " pulling $EMBED_MODEL" + run ollama pull "$EMBED_MODEL" + fi +} + +# probe_remote