#!/usr/bin/env bash # AI Admin Panel Installer — universal, multi-distro (AI-349) # # NOTE: scripts/install.sh is GENERATED from scripts/install.sh.in + scripts/lib/*.sh # by scripts/build-installer.sh. Edit the .in template and the lib modules, then # run scripts/build-installer.sh to regenerate — do not hand-edit install.sh. # # Usage: # curl -fsSL https://get.aiadminpanel.com -o install.sh && bash install.sh # bash install.sh [--dry-run] [--verbose] [--unattended] # # Certified: Ubuntu 22.04/24.04, Debian 12/13, Rocky/AlmaLinux/CentOS Stream 9/10 # Best-effort: Fedora, Oracle Linux, Amazon Linux 2023 # Document-only: openSUSE Leap/SLES, Arch (detection is by package-manager # capability + version floor, so newer releases auto-classify) # Architectures: x86_64 (amd64), aarch64/arm64 # # --dry-run: Log all actions without executing them (safe for testing) # --verbose: Enable debug output # --unattended: Non-interactive mode (claims a free temporary domain when PANEL_DOMAIN is unset; set MANAGED_DOMAIN=off to require it) # # Environment variables (skip interactive prompts when set): # PANEL_DOMAIN - Domain for the panel (e.g., panel.example.com) # ACME_EMAIL - Email for Let's Encrypt certificates # PANEL_VERSION - Version to install (default: latest) # CF_DNS_API_TOKEN - Cloudflare DNS API token for wildcard certs (optional) # ADMIN_PASSWORD - Override auto-generated admin password (optional) # LICENSE_KEY - License key for activation (optional, skip for 14-day trial) # GOMEMLIMIT - Go memory limit (optional) # TLS_MODE - letsencrypt (default) | cloudflare (skips the DNS A-record # pre-flight check; use with CF_DNS_API_TOKEN for DNS-01) # AAP_FIX_ROUTING - set to 1 to auto-fix a detected dual-default-route hazard # (multi-homed VPS such as OVH/Hetzner: a private NIC's DHCP # default route at a metric <= the public default breaks Docker # container networking). netplan hosts only; otherwise warn. # AAP_PROVIDER_AI_API_KEY - bundle a central "Provider AI" inference endpoint # (§6.6). When set, the panel seeds a Provider AI source on # first boot so deployed AI apps have a working upstream with # no key pasting. Always editable in Settings afterwards. # AAP_PROVIDER_AI_BASE_URL - the OpenAI-compatible base URL for the above # (e.g. https://inference.example.com/v1). # AAP_PROVIDER_AI_MODEL - default model id for the bundled Provider AI. # AAP_PROVIDER_AI_PROVIDER_TYPE - openai (default) | openrouter | anthropic | ollama. # AAP_SKIP_MODEL_PULL - set to "true" to skip auto-pulling the bundled metered # model at install (AI-622). The RAM-gated litellm-config is # still written; only the download is skipped, so the first # metered call 404s until the model is pulled from the AI # Models page. For air-gapped installs and CI (the installer # matrix sets it so jobs don't download multi-GB models). # MANAGED_DOMAIN - set to "off" to disable the temporary-domain claim path: # a missing PANEL_DOMAIN then fails exactly as before AI-544. # Any other value (or unset) leaves the claim path enabled. # MANAGED_DNS_URL - claim service base URL (default # https://dns.aiadminpanel.com; staging override for testing). # Claims are IPv4-only; the installer never sets # CF-Connecting-IP (Cloudflare manages that header itself). set -euo pipefail # ── Constants ───────────────────────────────────────────────────────────────── PANEL_DIR="/opt/aiadminpanel" CONFIG_DIR="/etc/aiadminpanel" LOG_DIR="/var/log/aiadminpanel" # Secret SOURCE files live on PERSISTENT storage under the panel dir, NOT the # tmpfs /run/secrets (which is wiped on every reboot). Docker still mounts them # into each container at /run/secrets/ via docker-compose.yml — that is the # in-container target and is unaffected. Sourcing from /run pre-AI-397 left # postgres/panel/keycloak unable to start after any host reboot. (AI-397) SECRETS_DIR="${PANEL_DIR}/secrets" INSTALL_LOG="${LOG_DIR}/install.log" GITHUB_RELEASE_BASE="https://github.com/aiadminpanel/ai-admin-panel/releases" GITHUB_DOWNLOAD_BASE="${GITHUB_RELEASE_BASE}/latest/download" GET_BASE="https://get.aiadminpanel.com" PANEL_VERSION="${PANEL_VERSION:-latest}" PANEL_DOMAIN="${PANEL_DOMAIN:-}" ACME_EMAIL="${ACME_EMAIL:-}" TOTAL_STEPS=17 CURRENT_STEP=0 INSTALL_START=$(date +%s) # ── Flags ───────────────────────────────────────────────────────────────────── DRY_RUN=false VERBOSE=false UNATTENDED=false for arg in "$@"; do case "$arg" in --dry-run) DRY_RUN=true ;; --verbose) VERBOSE=true ;; --unattended) UNATTENDED=true ;; *) echo "Unknown argument: $arg" >&2; exit 1 ;; esac done # ── Bundled libraries: logging, platform detection, secrets, docker, firewall, # preflight — inlined from scripts/lib/*.sh by scripts/build-installer.sh. # Edit the lib modules or install.sh.in, NOT the generated install.sh. # ───────────────────────────────────────────────────────────────────────── # BUNDLED LIBRARY CODE — generated from scripts/lib/*.sh by build-installer.sh. # DO NOT EDIT below this banner. Edit scripts/lib/.sh or # scripts/install.sh.in, then run scripts/build-installer.sh to regenerate. # ───────────────────────────────────────────────────────────────────────── # ===== scripts/lib/log.sh ===== # scripts/lib/log.sh — shared logging, colors, step counter, and the dry-run # `run` wrapper for the AI Admin Panel installer (AI-349). Single source of these # helpers for install.sh and every sibling lib module (their guarded fallbacks go # inert once this is sourced/inlined first). # # Consumes globals owned by the installer main section: DRY_RUN, VERBOSE, # INSTALL_LOG, TOTAL_STEPS, CURRENT_STEP. Pure function definitions + color setup. # ── Color support ───────────────────────────────────────────────────────────── COLOR_GREEN="" COLOR_YELLOW="" COLOR_RED="" COLOR_CYAN="" COLOR_BOLD="" COLOR_RESET="" setup_colors() { if [ -t 1 ] && command -v tput &>/dev/null && [ "$(tput colors 2>/dev/null || echo 0)" -ge 8 ]; then COLOR_GREEN=$(tput setaf 2) COLOR_YELLOW=$(tput setaf 3) COLOR_RED=$(tput setaf 1) COLOR_CYAN=$(tput setaf 6) COLOR_BOLD=$(tput bold) COLOR_RESET=$(tput sgr0) fi } # ── Logging ─────────────────────────────────────────────────────────────────── log() { local level="$1" shift local msg="$*" local ts ts=$(date -u '+%Y-%m-%dT%H:%M:%SZ') local line="[$ts] [$level] $msg" case "$level" in "INFO ") echo "${COLOR_GREEN}${line}${COLOR_RESET}" ;; "WARN ") echo "${COLOR_YELLOW}${line}${COLOR_RESET}" ;; "ERROR") echo "${COLOR_RED}${line}${COLOR_RESET}" ;; "DRY ") echo "${COLOR_CYAN}${line}${COLOR_RESET}" ;; *) echo "$line" ;; esac if [ -n "${INSTALL_LOG:-}" ] && [ -d "$(dirname "$INSTALL_LOG")" ]; then echo "$line" >>"$INSTALL_LOG" 2>/dev/null || true fi } info() { log "INFO " "$@"; } warn() { log "WARN " "$@"; } error() { log "ERROR" "$@" exit 1 } debug() { [ "${VERBOSE:-false}" = "true" ] && log "DEBUG" "$@" || true; } dryrun() { log "DRY " "[WOULD] $*"; } step() { CURRENT_STEP=$((${CURRENT_STEP:-0} + 1)) echo "" echo "${COLOR_BOLD}[${CURRENT_STEP}/${TOTAL_STEPS:-?}] $*${COLOR_RESET}" } # run — execute a command, or just narrate it under --dry-run. Used by the lib # modules' side-effecting functions (docker.sh, firewall.sh) and the main script. run() { if [ "${DRY_RUN:-false}" = "true" ]; then dryrun "$@" else debug "Running: $*" "$@" fi } # ===== scripts/lib/platform.sh ===== # shellcheck disable=SC2034 # this library's job is to SET globals (OS_*, OS_FAMILY, # PLATFORM_TIER/REASON, PKG_MANAGER, ARCH, BINARY_SUFFIX) consumed by install.sh and # sibling lib modules — shellcheck can't see those cross-file reads. # scripts/lib/platform.sh — universal distro / package-manager / architecture # detection for the AI Admin Panel installer (AI-349). # # Replaces install.sh's Ubuntu/Debian-only allowlist with capability + family # detection driven by /etc/os-release ID/ID_LIKE/VERSION_ID and a `command -v` # package-manager probe. Pure function definitions only — no side effects at # source time; the bootstrap (install.sh) owns `set -euo pipefail`. # # Public functions (set shell globals, bash-style): # detect_platform → OS_ID OS_ID_LIKE OS_VERSION_ID OS_VERSION_MAJOR OS_PRETTY # OS_FAMILY (debian|rhel|suse|arch|alpine|unknown) # PLATFORM_TIER (certified|best-effort|document|reject) # PLATFORM_REASON (human-readable) # detect_pm → PKG_MANAGER (apt|dnf|yum|zypper|pacman|apk|unknown) # detect_arch → ARCH BINARY_SUFFIX (amd64|arm64); returns 1 on unsupported # # Test override hooks: OS_RELEASE_FILE (path), ARCH_OVERRIDE (uname -m value). # ── os-release parsing (no `source` of the file — avoid executing/clobbering) ── _osr_get() { # $1=file $2=key → echo the unquoted value of KEY=... (last match wins) local file="$1" key="$2" line val="" while IFS= read -r line || [ -n "$line" ]; do case "$line" in "$key="*) val="${line#*=}" ;; *) continue ;; esac done <"$file" val="${val%\"}"; val="${val#\"}" val="${val%\'}"; val="${val#\'}" printf '%s' "$val" } # ── family from ID, then ID_LIKE fall-through ───────────────────────────────── _family_of() { local id="$1" id_like="$2" case "$id" in debian | ubuntu | raspbian | linuxmint | pop | zorin | elementary | devuan | kali | mx | neon) echo debian; return ;; rhel | centos | rocky | almalinux | fedora | ol | oracle | amzn | scientific | cloudlinux | virtuozzo | rocky-linux) echo rhel; return ;; opensuse | opensuse-leap | opensuse-tumbleweed | opensuse-slowroll | sles | sled | suse) echo suse; return ;; arch | manjaro | endeavouros | artix | cachyos | garuda | arcolinux) echo arch; return ;; alpine) echo alpine; return ;; esac # Unknown ID — classify by ID_LIKE (space-delimited list per os-release spec). local hay=" $id_like " case "$hay" in *" ubuntu "* | *" debian "*) echo debian; return ;; *" rhel "* | *" centos "* | *" fedora "*) echo rhel; return ;; *" suse "* | *" opensuse "*) echo suse; return ;; *" arch "*) echo arch; return ;; *" alpine "*) echo alpine; return ;; esac echo unknown } # ── tier classification (pure: id, family, numeric-major → globals) ─────────── _classify() { local id="$1" family="$2" major="$3" PLATFORM_TIER=""; PLATFORM_REASON="" if [ "$family" = "alpine" ]; then PLATFORM_TIER="reject" PLATFORM_REASON="Alpine/musl/OpenRC is unsupported: the panel stack needs systemd-managed Docker. Use Ubuntu 22.04+, Debian 12+, or Rocky/AlmaLinux 9+." return fi case "$id" in ubuntu) if [ "${major:-0}" -ge 22 ]; then PLATFORM_TIER="certified"; PLATFORM_REASON="Ubuntu ${major}.x (>= 22.04 supported)" else PLATFORM_TIER="reject"; PLATFORM_REASON="Ubuntu ${major}.x is below the 22.04 floor and out of support — upgrade to Ubuntu 22.04 LTS or newer." fi return ;; debian) if [ "${major:-0}" -ge 12 ]; then PLATFORM_TIER="certified"; PLATFORM_REASON="Debian ${major} (>= 12 supported)" else PLATFORM_TIER="reject"; PLATFORM_REASON="Debian ${major} is below the 12 floor and out of support — upgrade to Debian 12 or newer." fi return ;; rocky | almalinux | rhel | centos) if [ "${major:-0}" -ge 9 ]; then PLATFORM_TIER="certified"; PLATFORM_REASON="${id} ${major} (RHEL-family >= 9 certified)" elif [ "${major:-0}" -eq 8 ]; then PLATFORM_TIER="best-effort"; PLATFORM_REASON="${id} 8 is below the certified floor (9); best-effort only — prefer 9 or 10." else PLATFORM_TIER="reject"; PLATFORM_REASON="${id} ${major} is EOL/unsupported (CentOS Linux 7/8, RHEL < 8). Use Rocky/AlmaLinux/RHEL 9 or newer." fi return ;; fedora) PLATFORM_TIER="best-effort"; PLATFORM_REASON="Fedora ${major}: best-effort (short EOL cadence — prefer Rocky/AlmaLinux for long-lived servers)." return ;; ol | oracle) if [ "${major:-0}" -ge 8 ]; then PLATFORM_TIER="best-effort"; PLATFORM_REASON="Oracle Linux ${major}: best-effort (RHEL-compatible)." else PLATFORM_TIER="reject"; PLATFORM_REASON="Oracle Linux ${major} is EOL." fi return ;; amzn) if [ "${major:-0}" -ge 2023 ]; then PLATFORM_TIER="best-effort"; PLATFORM_REASON="Amazon Linux 2023: best-effort." else PLATFORM_TIER="reject"; PLATFORM_REASON="Amazon Linux ${major} (AL2) is EOL in 2026 — use Amazon Linux 2023." fi return ;; opensuse | opensuse-leap | opensuse-tumbleweed | opensuse-slowroll | sles | sled | suse) PLATFORM_TIER="document"; PLATFORM_REASON="SUSE family: partially supported (no official Docker CE repo) — see docs for manual steps." return ;; arch | manjaro | endeavouros | artix | cachyos | garuda | arcolinux) PLATFORM_TIER="document"; PLATFORM_REASON="Arch family: best-effort/document-only (rolling release) — see docs." return ;; esac # ID not specifically known — fall back to family (matched via ID_LIKE). case "$family" in debian | rhel) PLATFORM_TIER="best-effort" PLATFORM_REASON="'${id}' is not explicitly tested but looks ${family}-compatible (ID_LIKE) — proceeding best-effort." ;; suse | arch) PLATFORM_TIER="document"; PLATFORM_REASON="'${id}' (${family}-like): document-only." ;; *) PLATFORM_TIER="reject" PLATFORM_REASON="Unrecognized distribution '${id}'. Supported: Ubuntu 22.04+, Debian 12+, Rocky/AlmaLinux/RHEL 9+ (best-effort: Fedora, Oracle Linux, Amazon Linux 2023)." ;; esac } detect_platform() { local osr="${OS_RELEASE_FILE:-/etc/os-release}" if [ ! -r "$osr" ]; then OS_FAMILY="unknown"; PLATFORM_TIER="reject" PLATFORM_REASON="Cannot detect OS: ${osr} not found or unreadable." return 1 fi OS_ID="$(_osr_get "$osr" ID)"; OS_ID="${OS_ID,,}" OS_ID_LIKE="$(_osr_get "$osr" ID_LIKE)"; OS_ID_LIKE="${OS_ID_LIKE,,}" OS_VERSION_ID="$(_osr_get "$osr" VERSION_ID)" OS_PRETTY="$(_osr_get "$osr" PRETTY_NAME)" [ -n "$OS_PRETTY" ] || OS_PRETTY="${OS_ID} ${OS_VERSION_ID}" local maj="${OS_VERSION_ID%%.*}" case "$maj" in *[!0-9]* | "") maj=0 ;; esac OS_VERSION_MAJOR="$maj" OS_FAMILY="$(_family_of "$OS_ID" "$OS_ID_LIKE")" _classify "$OS_ID" "$OS_FAMILY" "$OS_VERSION_MAJOR" return 0 } # ── package manager (ordered probe; dnf preferred over yum) ─────────────────── detect_pm() { local pm for pm in apt-get dnf yum zypper pacman apk; do if command -v "$pm" >/dev/null 2>&1; then case "$pm" in apt-get) PKG_MANAGER="apt" ;; *) PKG_MANAGER="$pm" ;; esac return 0 fi done PKG_MANAGER="unknown" return 1 } # ── architecture ───────────────────────────────────────────────────────────── detect_arch() { ARCH="${ARCH_OVERRIDE:-$(uname -m)}" case "$ARCH" in x86_64 | amd64) ARCH="x86_64"; BINARY_SUFFIX="amd64" ;; aarch64 | arm64) ARCH="aarch64"; BINARY_SUFFIX="arm64" ;; *) echo "Unsupported architecture: ${ARCH} (need x86_64 or aarch64/arm64)" >&2 return 1 ;; esac return 0 } # ===== scripts/lib/secrets.sh ===== # scripts/lib/secrets.sh — portable, hardened CSPRNG secret generation and # .env mutation for the AI Admin Panel universal installer (AI-349). # # Replaces install.sh's generate_secrets / create_admin_user password gen / # generate_env_file with implementations that fix the audited findings # (.planning/universal-installer/AUDIT.md): # # gomemlimit-env-clobber (CRITICAL): `echo X > .env` truncates the whole # file, wiping every OIDC/DB/Keycloak/domain var install.sh wrote. # env_set() appends-or-replaces a single line in place — never truncates. # db-password-644-leak (HIGH): db_password + keycloak_admin_password were # chmod 644 (world-readable). generate_secrets chmod 600 EVERY secret. # base64-w0-busybox / xxd-not-default: `base64 -w0` is GNU-only and `xxd` # (vim) is absent on minimal installs. We use openssl/od/POSIX tr only. # admin-pw-restricted-alphabet: fragile base64|tr|head admin-pw pattern # replaced by unbiased `tr -dc 'A-Za-z0-9' /dev/null 2>&1; then openssl rand -hex "$nbytes" return 0 fi # Portable fallback: od emits space/newline-separated hex bytes; strip both. od -An -tx1 -N "$nbytes" /dev/urandom | tr -d ' \n' printf '\n' } # gen_secret_alnum [nchars=24] → nchars of unbiased [A-Za-z0-9] from /dev/urandom. # `tr -dc` rejects (not maps) non-matching bytes, so the alphabet is uniform — # no modulo bias. LC_ALL=C keeps the byte classes ASCII regardless of locale. gen_secret_alnum() { local nchars="${1:-24}" out="" chunk # Read BOUNDED chunks from /dev/urandom with head as the FIRST pipeline stage # (reading a device, no upstream writer) so nothing ever gets SIGPIPE'd — the # naive `tr /dev/null || true return 0 fi # Create with restrictive perms BEFORE writing content (no 644 window). ( umask 077; printf '%s' "$value" >"$path" ) chmod 600 "$path" } # generate_secrets secrets_dir → idempotently create every panel secret. # master_key : 32 random bytes hex-encoded (64 hex chars) # db_password : 32 unbiased alnum chars # keycloak_admin_password: 32 unbiased alnum chars # litellm_master_key : sk-<40 unbiased alnum chars> # managed_domain_token : empty placeholder (real token written by # manageddns.sh on managed installs) # openbao_unseal_key : 32 random bytes hex-encoded (64 hex chars), # chmod 644 — see the in-function comment # Directory is created 0700; every secret file is chmod 600 except # openbao_unseal_key (644 — the in-container non-root server must read it # through the ro secret mount; host confidentiality comes from the 0700 dir). generate_secrets() { local secrets_dir="$1" ( umask 077; mkdir -p "$secrets_dir" ) chmod 700 "$secrets_dir" _write_secret_file "$secrets_dir/master_key" "$(gen_secret_hex 32)" _write_secret_file "$secrets_dir/db_password" "$(gen_secret_alnum 32)" _write_secret_file "$secrets_dir/keycloak_admin_password" "$(gen_secret_alnum 32)" _write_secret_file "$secrets_dir/litellm_master_key" "sk-$(gen_secret_alnum 40)" # AI-553 (AI-544 plan 3): the panel container mounts managed_domain_token as # a compose secret, and compose hard-fails on a missing secret file. Managed # installs wrote the real token earlier (md_acquire_domain runs before us); # BYO installs get an empty 0600 placeholder, which the panel reads as # "not a managed install". Never overwrite an existing token. if [ ! -f "$secrets_dir/managed_domain_token" ]; then ( umask 077; : > "$secrets_dir/managed_domain_token" ) chmod 600 "$secrets_dir/managed_domain_token" fi # OpenBao static-seal unseal key (AI-557 OpenBao Phase 1). 32 random bytes # hex-encoded; mounted into the openbao container as a compose secret. # # 0644, NOT 0600: compose file-secrets are ro bind mounts that preserve the # host inode's perms, and the openbao image su-execs to its non-root # `openbao` user BEFORE the server parses the seal config — a 0600 # root:root key is unreadable in-container and the server exits with # "error reading file at file:///run/secrets/openbao_unseal_key: # permission denied" (crash-loop, panel blocked on service_healthy; # reproduced against openbao/openbao:2.5.5 — the PR #458 install-matrix # failure on all 9 distros). postgres never hits this because its # entrypoint reads db_password while still root. Host confidentiality # comes from the untraversable 0700 secrets dir above, not the file bits. # chmod runs unconditionally to self-heal pre-fix 0600 keys on re-install. if [ ! -f "$secrets_dir/openbao_unseal_key" ]; then ( umask 022; printf '%s' "$(gen_secret_hex 32)" > "$secrets_dir/openbao_unseal_key" ) fi chmod 644 "$secrets_dir/openbao_unseal_key" } # migrate_legacy_secrets dest_dir [legacy_dir=/run/secrets] → one-time migration # for installs created before AI-397, when secrets lived in the tmpfs # /run/secrets (wiped on every reboot). If the persistent dest_dir is missing a # secret but the legacy dir still holds it — a box that hasn't rebooted yet — # copy the value across so the password the postgres data volume was initialized # with is preserved instead of being regenerated into a mismatch. # # Idempotent and safe to call before generate_secrets: a no-op when dest_dir # already has the secret, when the legacy dir is gone (already rebooted, nothing # to recover), or when dest_dir IS the legacy dir (unchanged install). migrate_legacy_secrets() { local dest_dir="$1" legacy_dir="${2:-/run/secrets}" name [ "$dest_dir" = "$legacy_dir" ] && return 0 [ -d "$legacy_dir" ] || return 0 ( umask 077; mkdir -p "$dest_dir" ) chmod 700 "$dest_dir" 2>/dev/null || true for name in master_key db_password keycloak_admin_password litellm_master_key admin_password; do if [ -f "$legacy_dir/$name" ] && [ ! -f "$dest_dir/$name" ]; then cp -p "$legacy_dir/$name" "$dest_dir/$name" chmod 600 "$dest_dir/$name" fi done } # ── .env mutation (NEVER truncate) ──────────────────────────────────────────── # env_set env_file KEY VALUE → set KEY=VALUE in env_file. # If a line `^KEY=` exists, replace ONLY that line (preserving every other line # and its order); otherwise append `KEY=VALUE`. The file is created 600 if # missing. This is the fix for gomemlimit-env-clobber: callers must use this # instead of `echo KEY=VALUE > .env`, which wipes the file. env_set() { local env_file="$1" key="$2" value="$3" # Create with restrictive perms if absent so we never widen perms later. if [ ! -f "$env_file" ]; then ( umask 077; : >"$env_file" ) chmod 600 "$env_file" fi local newline="${key}=${value}" # Does the key already exist? Match the literal `KEY=` prefix at line start. if grep -q "^${key}=" "$env_file" 2>/dev/null; then # Replace just that line. Build the new file in a temp, then atomic mv. # We avoid sed/awk metacharacter pitfalls in VALUE by reading line by line # and substituting the matching line with the pre-built literal. local tmp tmp="$(mktemp "${env_file}.XXXXXX")" local line replaced=0 while IFS= read -r line || [ -n "$line" ]; do case "$line" in "${key}="*) if [ "$replaced" -eq 0 ]; then printf '%s\n' "$newline" >>"$tmp" replaced=1 fi # Drop any duplicate KEY= lines beyond the first. ;; *) printf '%s\n' "$line" >>"$tmp" ;; esac done <"$env_file" chmod 600 "$tmp" mv "$tmp" "$env_file" else printf '%s\n' "$newline" >>"$env_file" fi } # ── permission self-healing ─────────────────────────────────────────────────── # harden_perms secrets_dir env_file → re-assert restrictive perms. # Idempotent and re-runnable: 700 on the secrets dir, 600 on every secret file # inside it and on the env file. Safe to call repeatedly (self-healing). # Exception: openbao_unseal_key stays 644 — harden_perms runs AFTER # generate_secrets in the install flow, and sweeping the key to 600 would # re-break the openbao boot in the same run (the image's non-root server # reads it through the ro secret mount; see generate_secrets). The 0700 dir # is what keeps it host-confidential. harden_perms() { local secrets_dir="$1" env_file="$2" f if [ -d "$secrets_dir" ]; then chmod 700 "$secrets_dir" for f in "$secrets_dir"/*; do [ -f "$f" ] || continue if [ "$(basename "$f")" = "openbao_unseal_key" ]; then chmod 644 "$f" else chmod 600 "$f" fi done fi [ -f "$env_file" ] && chmod 600 "$env_file" return 0 } # ===== scripts/lib/openbao.sh ===== # OpenBao config rendering. Sourced by install.sh (bundled via build-installer.sh). # The unseal key itself is generated by generate_secrets (secrets.sh) so all # secret material lives in one place with one permission model. # write_openbao_config # Renders the OpenBao server HCL. Idempotent: rewrites only when content differs. # # Permissions are deliberately world-readable (dir 755, file 644): the # openbao/openbao image's entrypoint drops privileges to its non-root # `openbao` user, and its chown -R of /openbao/config silently fails on our # :ro bind mount — a root-only 640 config makes `bao server` crash-loop on # "permission denied" and the panel (depends_on: service_healthy) never # starts. This is safe: the config contains NO secret material, only a # file:// pointer to the separately-protected 0600 # /run/secrets/openbao_unseal_key. (BAO_SKIP_DROP_ROOT would also "fix" it, # but running OpenBao as root is a worse posture than a world-readable # pointer file.) write_openbao_config() { local config_dir="$1" mkdir -p "${config_dir}" chmod 755 "${config_dir}" local target="${config_dir}/openbao.hcl" local tmp # mktemp NEXT TO the target (same filesystem): a /tmp temp file breaks the # final mv with "Invalid cross-device link" when /tmp is a separate mount # (tmpfs, hardened hosts). Same idiom as secrets.sh env_set(). tmp="$(mktemp "${target}.XXXXXX")" cat > "${tmp}" <<'EOF' # Generated by the AAP installer. Managed file - do not edit by hand. # static seal: unattended auto-unseal from the compose-mounted key file. # Protects data-at-rest (stolen volume/backup); NOT a defense against full # host compromise - see docs/runbooks/openbao.md for the threat model. ui = false listener "tcp" { address = "0.0.0.0:8200" tls_disable = true } # Raft lives in /openbao/file: it's the ONLY storage dir the image # pre-creates (openbao:openbao) and its entrypoint chowns for the non-root # server user. The image ships no /openbao/data - mounting a volume there # creates a root-owned mountpoint the dropped-privilege server can't write # ("open /openbao/data/node-id: permission denied" crash-loop). storage "raft" { path = "/openbao/file" } seal "static" { current_key_id = "install-1" current_key = "file:///run/secrets/openbao_unseal_key" } # Declarative audit device: OpenBao (unlike upstream Vault) hard-rejects # API-based audit creation (400 "use declarative, config-based audit device # management instead" - verified against a live 2.5.5 container, AI-558), so # the spec-mandated file audit device must be declared here; the panel's # Bootstrapper only verifies it exists at first boot. The log lives under # /openbao/logs - alongside /openbao/file it's one of only two dirs the image # pre-creates for (and its entrypoint chowns to) the non-root server user; # anywhere else is unwritable (same failure class as the raft /openbao/data # crash-loop above). Compose mounts a named volume there so the audit trail # survives container recreation. audit "file" "file" { description = "AAP file audit device" options { file_path = "/openbao/logs/audit.log" } } cluster_addr = "http://openbao:8201" api_addr = "http://openbao:8200" disable_mlock = true EOF if [ -f "${target}" ] && cmp -s "${tmp}" "${target}"; then rm -f "${tmp}" return 0 fi mv "${tmp}" "${target}" chmod 644 "${target}" } # write_openbao_log_rotate_script # Renders the audit-log rotation script the compose `openbao_log_rotator` # sidecar runs. Idempotent: rewrites only when content differs. # # WHY (AI-611): OpenBao's declarative file audit device appends every audited # request to /openbao/logs/audit.log with NO built-in rotation or size cap. # On a long-running box that file grows unbounded — the demo hit 128 GB, filled # the disk, and crash-looped Postgres + OpenBao ("No space left on device"). # OpenBao has no native size limit and rejects API-based audit management # (AI-558), so the file must be bounded externally. # # The rotator uses copytruncate (cp then `: >`) rather than a rename: the audit # device holds audit.log open with O_APPEND, so truncating the SAME inode in # place keeps its open fd writing to the now-small live file. A mv/rename would # leave the device writing to the rotated-away inode until a SIGHUP — which we # can't send across containers without the Docker socket. Written NEXT TO the # config dir (not inside it) so the ":ro config holds only openbao.hcl" contract # and `bao server -config=` (which ignores siblings) both stay intact. write_openbao_log_rotate_script() { local openbao_dir="$1" mkdir -p "${openbao_dir}" local target="${openbao_dir}/openbao-log-rotate.sh" local tmp tmp="$(mktemp "${target}.XXXXXX")" cat > "${tmp}" <<'EOF' #!/bin/sh # Generated by the AAP installer. Managed file - do not edit by hand. # Bounds the OpenBao file audit device's audit.log (AI-611). Runs in the # openbao_log_rotator compose sidecar (reusing the openbao image). Config via # env: AUDIT_LOG_PATH, AUDIT_LOG_MAX_BYTES, AUDIT_LOG_KEEP, AUDIT_LOG_INTERVAL. set -u # rotate_once: if the audit log exceeds the cap, shift the kept backups # (.N-1 -> .N ... .1 -> .2), copy the live log to .1, then truncate it IN PLACE # so the audit device's open O_APPEND fd keeps writing to the same (now-empty) # inode. Reads config from the environment on every call. rotate_once() { f="${AUDIT_LOG_PATH:-/openbao/logs/audit.log}" max="${AUDIT_LOG_MAX_BYTES:-104857600}" keep="${AUDIT_LOG_KEEP:-5}" [ -f "$f" ] || return 0 sz="$(stat -c %s "$f" 2>/dev/null || wc -c < "$f" 2>/dev/null || echo 0)" [ "$sz" -le "$max" ] && return 0 i="$keep" while [ "$i" -gt 1 ]; do prev=$((i - 1)) [ -f "$f.$prev" ] && mv -f "$f.$prev" "$f.$i" i="$prev" done if [ "$keep" -ge 1 ]; then cp "$f" "$f.1" # copytruncate step 1: snapshot current log fi : > "$f" # copytruncate step 2: empty in place (same inode) } # AUDIT_LOG_DEFINE_ONLY lets the bats suite source this file for rotate_once # without entering the daemon loop. if [ "${AUDIT_LOG_DEFINE_ONLY:-0}" = "1" ]; then # shellcheck disable=SC2317 # reachable when EXECUTED (not sourced) with the flag return 0 2>/dev/null || exit 0 fi while :; do rotate_once sleep "${AUDIT_LOG_INTERVAL:-120}" done EOF if [ -f "${target}" ] && cmp -s "${tmp}" "${target}"; then rm -f "${tmp}" chmod 755 "${target}" return 0 fi mv "${tmp}" "${target}" chmod 755 "${target}" } # ===== scripts/lib/manageddns.sh ===== # scripts/lib/manageddns.sh — temporary-domain claim path (AI-544 plan 2). # When the user has no domain, claims an aap-…aiadminpanel.host name from the # managed-dns claim service (infra/managed-dns/ — FROZEN API contract in its # README; this module encodes but never drifts it) and persists the claim token # for the panel backend's lease renewal (plan 3). # # Contract facts encoded here: # POST /v1/claims no auth, rate limit 3/day/IP, IPv4 only (curl -4), # 201 {"domain","token","leaseExpiresAt"} # GET /v1/claims/status Authorization: Bearer , # 200 {"domain","emailVerified","leaseExpiresAt","lastRenewedAt"} # 401/410 = the claim is dead (lapsed/released) # NEVER send CF-Connecting-IP — Cloudflare rejects spoofed copies of its # managed header (edge error 1000/403). Real callers just POST; CF sets it. # # SECURITY (Rule 11): the token is shown exactly once by the API. It is written # 0600 under SECRETS_DIR immediately and NEVER logged — no debug/info of any # response body in this module. # # Consumes: info/warn (lib/log.sh); _resolve_domain (lib/preflight.sh) from # md_wait_for_dns — bundle order is irrelevant, functions resolve at call time. # Consumed by: prompt_domain / generate_env_file / print_success (install.sh.in). # Guarded logging fallback: inert once log.sh is bundled/sourced first (real # installer), active when this module is unit-tested in isolation. Mirrors # scripts/lib/docker.sh + preflight.sh so the bats file never sources log.sh # (whose run() would clobber bats' own run() test helper). if ! declare -F info >/dev/null 2>&1; then info() { printf '[INFO] %s\n' "$*"; } warn() { printf '[WARN] %s\n' "$*" >&2; } fi MANAGED_DNS_URL="${MANAGED_DNS_URL:-https://dns.aiadminpanel.com}" # Sleep seconds between claim attempts (1 initial + one retry per entry ≈ 2 min # total, per spec). Tests override with "0 0 0". MD_RETRY_DELAYS="${MD_RETRY_DELAYS:-20 40 60}" # Forward-declared cross-file state consumed by sibling bundled modules # (install.sh.in prompt_domain/generate_env_file/print_success, Tasks 2-3). # Exported (docker.sh:238 AAP_NVIDIA_RUNTIME_READY precedent) so the SC2034 # "appears unused" warning stays clean — the linter can't see cross-file reads. # Secret-safe: MD_TOKEN_FILE is a path; the token itself stays a local in Task 2. export MD_CLAIMED=false # true once this run claimed (or reused) a managed domain export MD_BASE_URL="" # base URL the claim actually used → .env MANAGED_DNS_URL export MD_LEASE_EXPIRES="" # leaseExpiresAt from the claim/status response → success text export MD_FAIL_REASON="" # human-readable failure for the caller to warn/error with export MD_REUSED_DOMAIN="" # set by md_reuse_claim export MD_TOKEN_FILE="" # set by md_acquire_domain from SECRETS_DIR # md_json_field — extract a string field from a flat JSON object # without jq (not installed at domain-prompt time; ensure_updater_deps runs much # later). Safe ONLY for our own Worker's responses: values are URL/hostname/ISO # material and never contain escaped quotes. md_json_field() { local json="$1" key="$2" printf '%s' "$json" | sed -n 's/.*"'"$key"'"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' } # md_claim — POST /v1/claims. Prints the 201 body on success. # Returns: 0 claimed · 1 unreachable/5xx after all retries · 2 rate-limited # 3 ipv6-unsupported · 4 other 400. Only network errors / 5xx retry. md_claim() { local out code body delay # shellcheck disable=SC2086 # intentional word-split of the delay list for delay in 0 $MD_RETRY_DELAYS; do if [ "$delay" -gt 0 ]; then info "Claim service not reachable — retrying in ${delay}s..." sleep "$delay" fi out=$(curl -4 -sS --max-time 15 -X POST "${MANAGED_DNS_URL}/v1/claims" \ -H 'Content-Type: application/json' -d '{}' \ -w $'\n%{http_code}' 2>/dev/null) || out="" code="${out##*$'\n'}" body="${out%$'\n'*}" case "$code" in 201) printf '%s' "$body"; return 0 ;; 429) return 2 ;; 400) case "$body" in *ipv6_unsupported*) return 3 ;; *) return 4 ;; esac ;; *) : ;; # empty (network error) or 5xx — retry esac done return 1 } # md_reuse_claim — if a token from a previous run exists, ask the service which # domain it belongs to and reuse it: installer re-runs after a mid-install # failure must never burn the 3/day/IP quota or orphan the earlier claim. # rc 0: sets MD_REUSED_DOMAIN + MD_LEASE_EXPIRES. rc 1: nothing reusable # (a dead token — 401/410 — is deleted; transient errors fall through to the # claim path, which carries the retry/fail-loud policy). md_reuse_claim() { MD_REUSED_DOMAIN="" [ -f "$MD_TOKEN_FILE" ] || return 1 local token out code body token="$(cat "$MD_TOKEN_FILE")" if [ -z "$token" ]; then rm -f "$MD_TOKEN_FILE" return 1 fi out=$(curl -4 -sS --max-time 15 "${MANAGED_DNS_URL}/v1/claims/status" \ -H "Authorization: Bearer ${token}" -w $'\n%{http_code}' 2>/dev/null) || out="" code="${out##*$'\n'}" body="${out%$'\n'*}" case "$code" in 200) MD_REUSED_DOMAIN="$(md_json_field "$body" domain)" MD_LEASE_EXPIRES="$(md_json_field "$body" leaseExpiresAt)" [ -n "$MD_REUSED_DOMAIN" ] || return 1 return 0 ;; 401 | 410) warn "Stored temporary-domain token is no longer valid — claiming a fresh domain." rm -f "$MD_TOKEN_FILE" return 1 ;; *) return 1 ;; esac } # md_acquire_domain — reuse-or-claim a managed temporary domain. # rc 0: PANEL_DOMAIN set, MD_CLAIMED=true, MD_BASE_URL set, token persisted 0600. # rc 1: MD_FAIL_REASON set; the caller decides re-prompt (interactive) vs # error (unattended/piped). # Call DIRECTLY, never in $(…) — it sets globals. Call it only in an # errexit-exempt context (a bare `md_acquire_domain || …` or an `if`/`elif` # condition, as both current callers do): under `set -e` the internal # `body="$(md_claim)"; rc=$?` capture would abort the script on a non-zero # claim before the MD_FAIL_REASON logic runs, silently breaking the # fail-reason UX. bash disables -e for the whole dynamic extent of a function # invoked in those contexts, which is why the current call sites are safe. md_acquire_domain() { MD_TOKEN_FILE="${SECRETS_DIR:-/opt/aiadminpanel/secrets}/managed_domain_token" MD_BASE_URL="$MANAGED_DNS_URL" MD_FAIL_REASON="" local body rc domain token if md_reuse_claim; then PANEL_DOMAIN="$MD_REUSED_DOMAIN" MD_CLAIMED=true info "Reusing temporary domain from a previous run: ${PANEL_DOMAIN}" return 0 fi info "Claiming a free temporary domain from ${MANAGED_DNS_URL} ..." body="$(md_claim)" rc=$? if [ "$rc" -ne 0 ]; then case "$rc" in 2) MD_FAIL_REASON="Temporary-domain limit reached (3 per IP per day). Enter your own domain, or retry after 24 hours." ;; 3) MD_FAIL_REASON="This box has no IPv4 egress (temporary domains are IPv4-only for now). Use your own domain instead." ;; 4) MD_FAIL_REASON="The claim service rejected the request. Re-run the installer, or set PANEL_DOMAIN to your own domain." ;; *) MD_FAIL_REASON="Claim service unreachable after 3 retries (~2 min). Re-run the installer, or set PANEL_DOMAIN to your own domain." ;; esac return 1 fi domain="$(md_json_field "$body" domain)" token="$(md_json_field "$body" token)" MD_LEASE_EXPIRES="$(md_json_field "$body" leaseExpiresAt)" if [ -z "$domain" ] || [ -z "$token" ]; then MD_FAIL_REASON="Claim service returned an unexpected response. Re-run the installer, or set PANEL_DOMAIN to your own domain." return 1 fi # Persist the token IMMEDIATELY — the API shows it exactly once and losing it # loses lease renewal. create_directories runs later in main(), so make the # secrets dir ourselves. Never log the token (Rule 11). mkdir -p "$(dirname "$MD_TOKEN_FILE")" chmod 700 "$(dirname "$MD_TOKEN_FILE")" printf '%s' "$token" > "$MD_TOKEN_FILE" chmod 600 "$MD_TOKEN_FILE" PANEL_DOMAIN="$domain" MD_CLAIMED=true info "Claimed temporary domain: ${PANEL_DOMAIN}" info "Claim token stored at ${MD_TOKEN_FILE} (root-only; used for lease renewal — keep it)." md_wait_for_dns "$PANEL_DOMAIN" return 0 } # md_wait_for_dns — absorb resolver lag on a seconds-old record so the # unchanged check_dns preflight doesn't hard-fail. Skips in cloudflare TLS mode # (check_dns skips there too). Warn-and-continue on timeout — check_dns stays # the hard gate. MD_DNS_WAIT_INTERVAL exists for tests (default 5s × 12 = 60s). md_wait_for_dns() { local domain="$1" i resolved [ "${TLS_MODE:-letsencrypt}" = "cloudflare" ] && return 0 info "Waiting for ${domain} to resolve (up to 60s)..." for ((i = 1; i <= 12; i++)); do resolved="$(_resolve_domain "$domain")" if [ -n "$resolved" ]; then info "DNS ready: ${domain} resolves." return 0 fi sleep "${MD_DNS_WAIT_INTERVAL:-5}" done warn "${domain} does not resolve yet — continuing; the pre-flight DNS check is the hard gate." return 0 } # ===== scripts/lib/docker.sh ===== # scripts/lib/docker.sh — universal Docker Engine + Compose/Buildx install (AI-349). # # Replaces install.sh's get.docker.com + apt-only-plugin path (which is redundant # where get.docker.com works and broken everywhere else). Strategy: # 1. Fast-path: real Docker >= 24 with compose + buildx and NOT a podman shim. # 2. get.docker.com for the IDs it supports (it bundles compose + buildx). # 3. Native per-package-manager repo branches for the IDs it refuses # (AlmaLinux, Oracle, Amazon Linux, SUSE, Arch). # 4. Always `systemctl enable --now docker`, then ASSERT tooling. # # Relies on platform.sh globals: OS_ID, OS_FAMILY, OS_VERSION_MAJOR, OS_PRETTY. # Pure function definitions only (the bootstrap owns `set -euo pipefail`). # Self-contained logging fallbacks for standalone use — install.sh provides # richer versions and (because these are guarded) its definitions win when this # is sourced into it. NOTE: deliberately does NOT define `run` — that would # clobber bats's own `run` helper in unit tests; the side-effecting functions # get `run` from install.sh (lib/log.sh) at integration time. if ! declare -F info >/dev/null 2>&1; then info() { printf '[INFO] %s\n' "$*"; } warn() { printf '[WARN] %s\n' "$*" >&2; } error() { printf '[ERROR] %s\n' "$*" >&2 return 1 } fi # ── Strategy selection (pure) ──────────────────────────────────────────────── # Strategy by ID. get.docker.com cleanly handles the apt family (ubuntu/debian/ # raspbian) and Fedora (clean $releasever major). For the RHEL family we # DELIBERATELY use our own native dnf branch instead of get.docker.com: Docker's # repo file embeds $releasever, which on a POINT release (Rocky 9.7, RHEL 9.4, …) # resolves to e.g. "9.7" and 404s download.docker.com — empty repodata, then # `Error: Unable to find a match: docker-ce`. get.docker.com does NOT pin it, so # it breaks on point releases (observed on Rocky 9.7). Our dnf branch pins # $releasever to the major (AI-349). docker_repo_strategy() { case "$OS_ID" in ubuntu | debian | raspbian | fedora) echo "getdocker"; return ;; centos | rhel | rocky | almalinux | ol | oracle | scientific | cloudlinux | virtuozzo) echo "dnf"; return ;; amzn) echo "amzn2023"; return ;; opensuse | opensuse-leap | opensuse-tumbleweed | opensuse-slowroll | sles | sled | suse) echo "zypper"; return ;; arch | manjaro | endeavouros | artix | cachyos | garuda | arcolinux) echo "pacman"; return ;; esac # ID not explicitly known — fall back to the family's native package manager. case "$OS_FAMILY" in debian) echo "apt" ;; rhel) echo "dnf" ;; suse) echo "zypper" ;; arch) echo "pacman" ;; *) echo "unsupported" ;; esac } # Docker's RHEL-family repos: centos repo for el9, rhel repo for el10 / RHEL. _dnf_docker_repo_url() { local id="$1" major="$2" if [ "$id" = "rhel" ] || [ "${major:-0}" -ge 10 ]; then echo "https://download.docker.com/linux/rhel/docker-ce.repo" else echo "https://download.docker.com/linux/centos/docker-ce.repo" fi } # ── Fast-path predicate ────────────────────────────────────────────────────── docker_tooling_ok() { command -v docker >/dev/null 2>&1 || return 1 # podman-docker shim presents `docker` but is not real Docker. if docker version 2>/dev/null | grep -qi podman; then return 1; fi local major major="$(docker --version 2>/dev/null | grep -oE '[0-9]+' | head -1)" [ -n "$major" ] || return 1 [ "$major" -ge 24 ] || return 1 docker compose version >/dev/null 2>&1 || return 1 docker buildx version >/dev/null 2>&1 || return 1 return 0 } # ── Native install branches (side-effecting; exercised in Layer-B matrix) ───── _curl_https() { curl --proto '=https' --tlsv1.2 -fsSL "$@"; } _docker_install_convenience() { info "Installing Docker via get.docker.com (engine + compose + buildx)..." if [ "${DRY_RUN:-false}" = "true" ]; then info "[dry] curl --proto '=https' --tlsv1.2 -fsSL https://get.docker.com | sh" return 0 fi local tmp="/tmp/aap-get-docker.sh" _curl_https https://get.docker.com -o "$tmp" || error "Failed to download get.docker.com" || return 1 sh "$tmp" || error "get.docker.com install failed on ${OS_PRETTY:-$OS_ID}" || return 1 rm -f "$tmp" } _dnf_add_docker_repo() { local url="$1" run dnf -y install dnf-plugins-core # dnf4 uses `config-manager --add-repo`; dnf5 uses `config-manager addrepo --from-repofile`. if ! run dnf config-manager --add-repo "$url" 2>/dev/null; then run dnf config-manager addrepo --from-repofile="$url" fi # Pin $releasever to the major so a point-release (e.g. 9.5) can't 404. run sed -i "s/\$releasever/${OS_VERSION_MAJOR}/g" /etc/yum.repos.d/docker-ce.repo } _docker_install_dnf() { info "Installing Docker from the native dnf repo for ${OS_PRETTY:-$OS_ID}..." _dnf_add_docker_repo "$(_dnf_docker_repo_url "$OS_ID" "$OS_VERSION_MAJOR")" _ensure_kernel_modules_extra run dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin } # el10 (Rocky/AlmaLinux/RHEL 10) minimal & cloud images ship WITHOUT # kernel-modules-extra. Without it dockerd fails to start: iptables can't load # the xt_addrtype module Docker needs for its NAT rules. Install the package # matched to the RUNNING kernel (an unversioned install could pull a different # kernel). Only attempted on el10+; a no-op on el8/el9 where it's already present. # Matched-only so it's harmless inside CI containers (host kernel ≠ repo). (AI-349) _ensure_kernel_modules_extra() { [ "${OS_VERSION_MAJOR:-0}" -ge 10 ] || return 0 info "Ensuring kernel-modules-extra for the running kernel (el10 needs xt_addrtype for Docker's NAT)..." if [ "${DRY_RUN:-false}" = "true" ]; then info '[dry] dnf -y install kernel-modules-extra-$(uname -r)' return 0 fi dnf -y install "kernel-modules-extra-$(uname -r)" >/dev/null 2>&1 || warn "kernel-modules-extra-$(uname -r) not installable from the repo; if dockerd fails to start (xt_addrtype error), install kernel-modules-extra matching your kernel and reboot." } _docker_install_amzn() { info "Installing Docker from the Amazon Linux docker-ce repo (best-effort)..." _dnf_add_docker_repo "https://download.docker.com/linux/amazonlinux/docker-ce.repo" run dnf -y install docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin } _docker_install_zypper() { info "Installing Docker on SUSE (document-only / best-effort)..." # Prefer the distro's own packages (versions lag but are integrated); fall back # to docker.com's SLES repo. if ! run zypper --non-interactive install -y docker docker-compose docker-buildx; then run zypper --non-interactive addrepo https://download.docker.com/linux/sles/docker-ce.repo run zypper --non-interactive --gpg-auto-import-keys refresh run zypper --non-interactive install -y docker-ce docker-ce-cli containerd.io docker-compose-plugin docker-buildx-plugin fi } _docker_install_pacman() { info "Installing Docker on Arch (document-only / best-effort)..." run pacman -Sy --noconfirm docker docker-compose docker-buildx } _docker_install_apt_native() { info "Installing Docker from Docker's apt repo (debian-derivative, best-effort)..." local codename="${UBUNTU_CODENAME:-${VERSION_CODENAME:-}}" if [ -z "$codename" ]; then # Last resort: try the convenience script (it may still match via os-release). _docker_install_convenience return fi run apt-get update -y run apt-get install -y ca-certificates curl run install -m 0755 -d /etc/apt/keyrings local base="https://download.docker.com/linux/ubuntu" [ "$OS_FAMILY" = "debian" ] && case "$OS_ID_LIKE" in *debian*) base="https://download.docker.com/linux/debian" ;; esac run sh -c "curl --proto '=https' --tlsv1.2 -fsSL ${base}/gpg -o /etc/apt/keyrings/docker.asc" run chmod a+r /etc/apt/keyrings/docker.asc run sh -c "echo 'deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] ${base} ${codename} stable' > /etc/apt/sources.list.d/docker.list" run apt-get update -y run apt-get install -y docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin } # ── Daemon start + tooling assertion ───────────────────────────────────────── _docker_enable_daemon() { if [ "${DRY_RUN:-false}" = "true" ]; then info "[dry] systemctl enable --now docker" return 0 fi run systemctl enable --now docker || warn "systemctl enable --now docker returned non-zero" local i=0 while [ "$i" -lt 30 ]; do docker info >/dev/null 2>&1 && return 0 sleep 1 i=$((i + 1)) done warn "Docker daemon did not become ready within 30s" } # Asserts compose + buildx + an active daemon. NEVER apt-get the plugins (the old # bug): get.docker.com / the native repo already install them, so a miss here is # a real failure, not something to paper over. verify_docker_tooling() { if [ "${DRY_RUN:-false}" = "true" ]; then return 0; fi docker compose version >/dev/null 2>&1 || error "Docker Compose v2 plugin missing after install on ${OS_PRETTY:-$OS_ID}. Install 'docker-compose-plugin' from Docker's repo and re-run." docker buildx version >/dev/null 2>&1 || error "Docker Buildx plugin missing after install on ${OS_PRETTY:-$OS_ID}. Install 'docker-buildx-plugin' from Docker's repo and re-run." systemctl is-active --quiet docker 2>/dev/null || warn "docker.service is not active" info "Docker tooling verified: $(docker compose version 2>/dev/null | head -1)" } # ── Orchestrator ───────────────────────────────────────────────────────────── install_docker() { if docker_tooling_ok; then info "Docker $(docker --version 2>/dev/null) already present with compose+buildx — skipping install" else local strat strat="$(docker_repo_strategy)" info "Installing Docker via strategy '${strat}' for ${OS_PRETTY:-$OS_ID}" case "$strat" in getdocker) _docker_install_convenience ;; dnf) _docker_install_dnf ;; amzn2023) _docker_install_amzn ;; zypper) _docker_install_zypper ;; pacman) _docker_install_pacman ;; apt) _docker_install_apt_native ;; *) error "No Docker install path for '${OS_ID}' (${OS_FAMILY}). Install Docker Engine 24+ with the compose & buildx plugins manually, then re-run." || return 1 ;; esac fi _docker_enable_daemon verify_docker_tooling } # ── NVIDIA Container Toolkit (AI-419) ───────────────────────────────────────── # A GPU-aware deploy (AI-416 P2) emits Docker DeviceRequests + NVIDIA_VISIBLE_ # DEVICES, which only work once the NVIDIA Container Toolkit is installed AND # Docker is configured with its `nvidia` runtime. The installer does this # automatically on a host with an NVIDIA GPU so the operator never touches a # terminal (Secure · Sovereign · Simple). Idempotent; a no-op on CPU-only hosts. # # Relies on platform.sh globals (OS_FAMILY/OS_ID) and preflight.sh # host_has_nvidia_gpu / host_has_nvidia_driver. Sets AAP_NVIDIA_RUNTIME_READY=true # when Docker's `nvidia` runtime ends up configured — generate_env_file consumes # that to give the panel container its own GPU visibility (AI-420 opt 1). # Set true once Docker's nvidia runtime is configured. Read by install.sh.in # generate_env_file (AI-420) to give the panel container its own GPU visibility. # Exported (not merely assigned) to document the cross-module read AND satisfy # SC2034 — the linter can't see the consumer in another bundled file. export AAP_NVIDIA_RUNTIME_READY=false NVIDIA_TOOLKIT_DOC_URL="https://docs.nvidia.com/datacenter/cloud-native/container-toolkit/latest/install-guide.html" _NVIDIA_GPGKEY="https://nvidia.github.io/libnvidia-container/gpgkey" _NVIDIA_KEYRING="/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg" _NVIDIA_DEB_LIST="https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list" _NVIDIA_RPM_REPO="https://nvidia.github.io/libnvidia-container/stable/rpm/nvidia-container-toolkit.repo" # nvidia_toolkit_strategy → install path by package-manager family. NVIDIA # publishes a libnvidia-container repo for deb (apt), rpm (dnf/yum), and SUSE # (zypper). Arch is AUR-only → unsupported (warn + manual link). Pure. nvidia_toolkit_strategy() { case "$OS_FAMILY" in debian) echo "apt" ;; rhel) echo "dnf" ;; suse) echo "zypper" ;; *) echo "unsupported" ;; esac } _nvidia_toolkit_install_apt() { if [ "${DRY_RUN:-false}" = "true" ]; then info "[dry] curl ${_NVIDIA_GPGKEY} | gpg --dearmor -o ${_NVIDIA_KEYRING}" info "[dry] write ${_NVIDIA_DEB_LIST} → /etc/apt/sources.list.d/nvidia-container-toolkit.list" info "[dry] apt-get update && apt-get install -y nvidia-container-toolkit" return 0 fi run install -m 0755 -d /usr/share/keyrings run sh -c "curl --proto '=https' --tlsv1.2 -fsSL ${_NVIDIA_GPGKEY} | gpg --dearmor -o ${_NVIDIA_KEYRING}" run chmod a+r "${_NVIDIA_KEYRING}" run sh -c "curl --proto '=https' --tlsv1.2 -fsSL ${_NVIDIA_DEB_LIST} | sed 's#deb https://#deb [signed-by=${_NVIDIA_KEYRING}] https://#g' > /etc/apt/sources.list.d/nvidia-container-toolkit.list" run apt-get update -y run apt-get install -y nvidia-container-toolkit } _nvidia_toolkit_install_dnf() { if [ "${DRY_RUN:-false}" = "true" ]; then info "[dry] curl ${_NVIDIA_RPM_REPO} → /etc/yum.repos.d/nvidia-container-toolkit.repo" info "[dry] dnf install -y nvidia-container-toolkit" return 0 fi run curl --proto '=https' --tlsv1.2 -fsSL "${_NVIDIA_RPM_REPO}" -o /etc/yum.repos.d/nvidia-container-toolkit.repo run dnf install -y nvidia-container-toolkit } _nvidia_toolkit_install_zypper() { if [ "${DRY_RUN:-false}" = "true" ]; then info "[dry] zypper addrepo ${_NVIDIA_RPM_REPO} && zypper install -y nvidia-container-toolkit" return 0 fi run zypper --non-interactive addrepo "${_NVIDIA_RPM_REPO}" nvidia-container-toolkit || true run zypper --non-interactive --gpg-auto-import-keys refresh run zypper --non-interactive install -y nvidia-container-toolkit } # Configure Docker's `nvidia` runtime + restart, then confirm it registered. # Sets AAP_NVIDIA_RUNTIME_READY=true on success (or in dry-run, so the env-file # branch is exercised). _nvidia_ctk_configure_docker() { if [ "${DRY_RUN:-false}" = "true" ]; then info "[dry] nvidia-ctk runtime configure --runtime=docker && systemctl restart docker" AAP_NVIDIA_RUNTIME_READY=true return 0 fi if ! command -v nvidia-ctk >/dev/null 2>&1; then warn "nvidia-ctk not found after install; cannot configure Docker's nvidia runtime. See ${NVIDIA_TOOLKIT_DOC_URL}" return 0 fi run nvidia-ctk runtime configure --runtime=docker run systemctl restart docker || warn "systemctl restart docker returned non-zero after nvidia runtime configure" local i=0 while [ "$i" -lt 30 ]; do docker info >/dev/null 2>&1 && break sleep 1 i=$((i + 1)) done if docker info 2>/dev/null | grep -qiE 'runtimes:.*nvidia'; then AAP_NVIDIA_RUNTIME_READY=true info "Docker nvidia runtime configured — GPU containers (and the panel's live GPU view) are ready." else warn "The nvidia runtime is not visible in 'docker info' after configure; GPU passthrough may not work. See ${NVIDIA_TOOLKIT_DOC_URL}" fi } # install_nvidia_container_toolkit — detect an NVIDIA GPU; if present, install + # configure the toolkit idempotently. A no-op (success) on CPU-only hosts. install_nvidia_container_toolkit() { if ! host_has_nvidia_gpu; then info "No NVIDIA GPU detected on the PCI bus — skipping NVIDIA Container Toolkit (CPU-only host)." return 0 fi info "NVIDIA GPU detected — ensuring the NVIDIA Container Toolkit is installed and Docker's nvidia runtime is configured." if ! host_has_nvidia_driver; then warn "An NVIDIA GPU is present but no driver is loaded (nvidia-smi unavailable). Installing the container toolkit anyway, but GPU containers need the driver too: on Ubuntu run 'ubuntu-drivers install' (or 'apt install -y nvidia-driver-'); on RHEL/Rocky enable EPEL + the CUDA repo and 'dnf install -y nvidia-driver', then reboot." fi if command -v nvidia-ctk >/dev/null 2>&1; then info "NVIDIA Container Toolkit already installed — re-asserting Docker runtime configuration (idempotent)." _nvidia_ctk_configure_docker return 0 fi local strat strat="$(nvidia_toolkit_strategy)" info "Installing NVIDIA Container Toolkit via '${strat}' for ${OS_PRETTY:-${OS_ID:-$OS_FAMILY}}..." case "$strat" in apt) _nvidia_toolkit_install_apt ;; dnf) _nvidia_toolkit_install_dnf ;; zypper) _nvidia_toolkit_install_zypper ;; *) warn "No automated NVIDIA Container Toolkit install path for '${OS_ID:-$OS_FAMILY}'. Install it manually, then run 'nvidia-ctk runtime configure --runtime=docker': ${NVIDIA_TOOLKIT_DOC_URL}" return 0 ;; esac _nvidia_ctk_configure_docker } # nvidia_panel_env_block — env-file lines that give the PANEL container its own # GPU visibility (AI-420 opt 1). Emitted by generate_env_file ONLY when Docker's # nvidia runtime was configured (AAP_NVIDIA_RUNTIME_READY=true). With the panel # container under the nvidia runtime + NVIDIA_VISIBLE_DEVICES, the toolkit injects # nvidia-smi into it, so detection shows the rich live model/VRAM/driver view # instead of the degraded PCI-only fallback. NVIDIA_DRIVER_CAPABILITIES=utility # is the minimal capability that provides nvidia-smi + NVML (no compute/graphics). # On every other host this emits nothing, so docker-compose.yml's safe defaults # (runtime runc, NVIDIA_VISIBLE_DEVICES=void) apply and the panel is unchanged. nvidia_panel_env_block() { [ "${AAP_NVIDIA_RUNTIME_READY:-false}" = "true" ] || return 0 printf '%s\n' \ "# GPU visibility for the panel container (NVIDIA runtime detected at install)." \ "PANEL_RUNTIME=nvidia" \ "NVIDIA_VISIBLE_DEVICES=all" \ "NVIDIA_DRIVER_CAPABILITIES=utility" } # ollama_gpu_override_yaml — the docker-compose overlay that hands the bundled # Ollama the host GPU (AI-428). Emitted ONLY when Docker's nvidia runtime was # configured (AAP_NVIDIA_RUNTIME_READY=true); nothing on CPU-only hosts. # # Why an overlay and not the panel's env-var trick: the panel container gets GPU # *visibility* via runtime:nvidia (above), which is enough for nvidia-smi. But # Ollama is shown in the AI Models page, whose honest per-model badge (AI-427) # reads HostConfig.DeviceRequests — a field ONLY the structured # `deploy.resources.reservations.devices` block populates (runtime:nvidia leaves # it null → the badge would read CPU even while the GPU is in use). That block # can't sit unconditionally in docker-compose.yml: on a CPU host `docker compose # up` then fails ("could not select device driver nvidia"). So on a GPU host the # installer writes this as a separate file and merges it with an extra `-f`; on a # CPU host the file is absent and the base compose stays CPU-safe. # # `count: all` → DeviceRequests:[{Driver:nvidia,Count:-1,Capabilities:[[gpu]]}], # exactly the reservation hand-verified on the OVH Quadro RTX 5000 box. ollama_gpu_override_yaml() { [ "${AAP_NVIDIA_RUNTIME_READY:-false}" = "true" ] || return 0 cat <<'YAML' # AI Admin Panel — GPU overlay. GENERATED by install.sh on an NVIDIA host (AI-428). # Gives the bundled Ollama the host GPU so Local AI runs accelerated. Merged via # `docker compose -f docker-compose.yml -f docker-compose.gpu.yml up`. Absent on # CPU-only hosts, so the base compose stays CPU-safe. Regenerated on every # install/upgrade — edit docker-compose.yml or the installer, not this file. services: ollama: deploy: resources: reservations: devices: - driver: nvidia count: all capabilities: [gpu] YAML } # ===== scripts/lib/firewall.sh ===== # scripts/lib/firewall.sh — distro-portable host firewall handling for the AI # Admin Panel installer (AI-349). Fixes audit id `ufw-rhel-portability`: the old # `apt-get install ufw` path is Debian/Ubuntu-only, so on RHEL/Fedora/SUSE # (firewalld) and Arch (nftables) the installer never opened Traefik's ports and # Let's Encrypt HTTP-01 validation failed on a box that "looked" up. # # We detect the *running* firewall by command presence + live state — never by # distro name — and idempotently allow inbound 22/80/443 for that backend. # # Pure function definitions only — nothing executes at source time; the # bootstrap (install.sh) owns `set -euo pipefail`. # # Ordering (from docs/research/universal-installer.md §5, firewalld row): # run AFTER `docker compose up`, BEFORE the Let's Encrypt HTTP-01 smoke. # Fully idempotent — safe to re-run on every install/upgrade. # # We never touch Docker's own iptables/nft chains: Docker manages its DNAT for # published ports itself. We only open the host firewall in front of it. # # Public functions: # detect_firewall → FIREWALL_BACKEND (firewalld|ufw|nftables|none) # firewall_cmds_for BK → echoes the exact shell commands open_firewall_ports # would run for backend BK, one per line (pure; no # side effects — bats asserts on this) # open_firewall_ports → idempotently allow 22/80/443 for FIREWALL_BACKEND; # honors DRY_RUN=true (prints "[would] …", runs nothing) # # Test override / mock surface: detection is driven entirely by `firewall-cmd`, # `ufw`, and `nft` resolved via PATH, so bats can shim them as fake executables. # ── tiny INFO shim ──────────────────────────────────────────────────────────── # install.sh defines info(); when this lib is sourced standalone (bats) it isn't, # so fall back to a plain stderr line. Never redefine an existing info(). _fw_info() { if command -v info >/dev/null 2>&1; then info "$@" else printf 'INFO %s\n' "$*" >&2 fi } # ── detection ───────────────────────────────────────────────────────────────── # Probe order matters: a box can have firewall-cmd AND nft installed (firewalld # is an nftables front-end), so prefer the higher-level manager that is actually # *running*. ufw likewise sits on top of iptables/nft. # 1. firewalld — `firewall-cmd` present AND `firewall-cmd --state` == running # 2. ufw — `ufw` present AND `ufw status` reports active # 3. nftables — `nft` present (rules exist or not — we still won't write any) # 4. none — nothing managing the host firewall detect_firewall() { if command -v firewall-cmd >/dev/null 2>&1 \ && firewall-cmd --state >/dev/null 2>&1; then FIREWALL_BACKEND="firewalld" return 0 fi if command -v ufw >/dev/null 2>&1 \ && ufw status 2>/dev/null | grep -qi 'Status: active'; then FIREWALL_BACKEND="ufw" return 0 fi if command -v nft >/dev/null 2>&1; then FIREWALL_BACKEND="nftables" return 0 fi FIREWALL_BACKEND="none" return 0 } # ── pure command planner ────────────────────────────────────────────────────── # Echo the exact commands open_firewall_ports would execute for backend $1, one # per line. Pure: no side effects, runs nothing. Backends with no managed # firewall (nftables/none) print nothing — the caller emits an INFO instead. firewall_cmds_for() { local backend="$1" case "$backend" in firewalld) echo "firewall-cmd --permanent --add-service=ssh --add-service=http --add-service=https" echo "firewall-cmd --reload" ;; ufw) echo "ufw allow 22/tcp" echo "ufw allow 80/tcp" echo "ufw allow 443/tcp" ;; nftables | none) : # no managed-firewall commands — handled as an INFO no-op ;; *) : # unknown backend — emit nothing, treated as no-op ;; esac } # ── apply ───────────────────────────────────────────────────────────────────── # Idempotently open 22/80/443 for the detected backend. firewalld and ufw are # both natively idempotent (re-adding a permanent service / an existing allow # rule is a no-op), so this is safe to re-run on every install and upgrade. # # DRY_RUN=true prints "[would] " for each planned command and executes # nothing — matches install.sh's dry-run so the module is unit-testable. open_firewall_ports() { # Detect on demand if the caller didn't already. [ -n "${FIREWALL_BACKEND:-}" ] || detect_firewall local backend="${FIREWALL_BACKEND}" case "$backend" in firewalld | ufw) _fw_info "Opening host firewall (${backend}) for inbound 22/80/443" local cmd while IFS= read -r cmd; do [ -n "$cmd" ] || continue if [ "${DRY_RUN:-false}" = "true" ]; then printf '[would] %s\n' "$cmd" else # word-splitting is intentional: cmd is our own fixed, trusted string. # shellcheck disable=SC2086 eval "$cmd" fi done <= 5GiB, # ports 80/443 free, DNS A-record matches box IP. # Soft gates (WARN only): outbound reachability, NTP sync, cgroup v2. # # The decision logic of each gate is split into a PURE predicate (no I/O, no # globals) so bats can exercise it directly with crafted inputs — mirrors # platform.sh's `_classify` split. Pure predicates: # arch_supported → 0 if 64-bit x86_64/aarch64 # virt_supported → 0 unless openvz / unprivileged lxc # disk_ok → 0 if >= 5 GiB # port_in_use → 0 if something LISTENs on # dns_matches → 0 if box-ip is in the list # # Honored env (flag twins owned by install.sh): DRY_RUN, VERBOSE, PANEL_DOMAIN, # TLS_MODE (letsencrypt|cloudflare), INSTALL_DIR. # Test override hooks: ARCH_OVERRIDE, VIRT_OVERRIDE, EUID_OVERRIDE — bats can't # assign the readonly $EUID, so check_root reads EUID_OVERRIDE first (see helper). PREFLIGHT_EXIT_MISSING_PREREQ=2 PREFLIGHT_MIN_DISK_BYTES=$((5 * 1024 * 1024 * 1024)) # 5 GiB # Default-route guard (AI-423): where the opt-in AAP_FIX_ROUTING fix writes its # netplan drop-in, and the metric it parks the private default route at (well # above any normal DHCP metric, so the public default always wins). ROUTE_FIX_FILE # is overridable for tests. ROUTE_FIX_FILE="${ROUTE_FIX_FILE:-/etc/netplan/99-aap-route-fix.yaml}" ROUTE_FIX_METRIC=4000 # ── minimal logging (only define if the bootstrap hasn't already) ───────────── # install.sh provides richer info/warn/error; these fallbacks keep the lib # usable standalone (and in bats) without pulling in the whole bootstrap. if ! declare -F info >/dev/null 2>&1; then info() { printf '[INFO] %s\n' "$*"; } fi if ! declare -F warn >/dev/null 2>&1; then warn() { printf '[WARN] %s\n' "$*" >&2; } fi if ! declare -F pf_fail >/dev/null 2>&1; then # pf_fail prints a blocker + fix to stderr. It does NOT exit — preflight_checks # aggregates failures and returns 2, so the bootstrap controls process exit. pf_fail() { printf '[FAIL] %s\n' "$*" >&2; } fi # ── pure predicates (no side effects — bats calls these directly) ───────────── # arch_supported → 0 for 64-bit x86_64/aarch64, 1 otherwise (32-bit). arch_supported() { local m="$1" case "$m" in x86_64 | amd64 | aarch64 | arm64) return 0 ;; *) return 1 ;; esac } # virt_supported → 1 (unsupported) for openvz and any lxc flavour # (rootful Docker can't run in OpenVZ or an unprivileged LXC container); 0 for # kvm / none / docker / amazon / etc. systemd-detect-virt prints `none` on bare # metal and a non-zero exit, which the caller maps to "none". virt_supported() { local v="$1" case "$v" in openvz | lxc | lxc-libvirt) return 1 ;; *) return 0 ;; esac } # disk_ok → 0 if >= 5 GiB. Non-numeric input is treated as a # failure (inconclusive probe shouldn't pass silently). disk_ok() { local bytes="$1" case "$bytes" in '' | *[!0-9]*) return 1 ;; esac [ "$bytes" -ge "$PREFLIGHT_MIN_DISK_BYTES" ] } # port_in_use → 0 if the table shows a LISTEN socket on # . Accepts BOTH `ss -ltn` and `netstat -ltn` output (audit ss-not-present # fallback). Both render the local endpoint as ADDR:PORT in a whitespace- # separated column, where ADDR may itself contain colons (IPv6 `[::]:443`, # `*:443`) — so we match the port as the final `:PORT` token, anchored on a # word boundary, and never on the trailing `:*` peer column. port_in_use() { local table="$1" port="$2" line addr while IFS= read -r line || [ -n "$line" ]; do case "$line" in *LISTEN*) ;; *) continue ;; esac # Scan each whitespace token; the local endpoint is the one ending in # : that is NOT the peer column (peer ends in ":*"). for addr in $line; do case "$addr" in *:\*) continue ;; # peer column (ss/netstat) — skip *:"$port") # endpoint ends exactly in : return 0 ;; esac done done < → 0 if box-ip appears as a whole # entry in the (whitespace/newline-separated) resolved list. Empty resolution # or empty box-ip is a failure (we can't prove the record points here). dns_matches() { local resolved="$1" box_ip="$2" ip [ -n "$box_ip" ] || return 1 [ -n "$resolved" ] || return 1 for ip in $resolved; do [ "$ip" = "$box_ip" ] && return 0 done return 1 } # _is_rfc1918 → 0 if the address is in a private RFC1918 range # (10/8, 172.16-31/12, 192.168/16). Pure. Tells a box's public default route # apart from a private-network (vRack/cloud-private) one. _is_rfc1918() { case "$1" in 10.* | 192.168.*) return 0 ;; 172.1[6-9].* | 172.2[0-9].* | 172.3[0-1].*) return 0 ;; *) return 1 ;; esac } # dangerous_default_routes <`ip -4 route show default` output> → detect the # multi-homed footgun (AI-423): a PRIVATE-network default route at a metric <= # the PUBLIC default's. Linux then routes forwarded Docker container replies # (private/container source) out the private NIC, where they're dropped — the # panel goes unreachable AND containers lose egress, with NO error message. # Each route is classified public/private by its `src` (fallback: `via` gateway); # a missing `metric` token means metric 0 (highest priority). Echoes # " " and returns 0 when dangerous; returns 1 when safe. # Pure (no I/O) so bats exercises it with crafted tables. dangerous_default_routes() { local table="$1" line tok prev dev gw src metric addr local pub_metrics=() priv_devs=() priv_metrics=() while IFS= read -r line || [ -n "$line" ]; do case "$line" in default*) ;; *) continue ;; esac dev=""; gw=""; src=""; metric=""; prev="" for tok in $line; do case "$prev" in via) gw="$tok" ;; dev) dev="$tok" ;; src) src="$tok" ;; metric) metric="$tok" ;; esac prev="$tok" done case "$metric" in '' | *[!0-9]*) metric=0 ;; esac # Classify by src, falling back to the gateway; skip if neither is present. if [ -n "$src" ]; then addr="$src" elif [ -n "$gw" ]; then addr="$gw" else continue; fi if _is_rfc1918 "$addr"; then priv_devs+=("$dev") priv_metrics+=("$metric") else pub_metrics+=("$metric") fi done </dev/null 2>&1; then local v v="$(systemd-detect-virt 2>/dev/null || true)" printf '%s' "${v:-none}" else printf 'none' fi } # _free_bytes → free bytes on the filesystem holding , walking up to # the nearest existing ancestor (the install dir may not exist yet pre-install). _free_bytes() { local dir="$1" while [ -n "$dir" ] && [ ! -d "$dir" ]; do dir="$(dirname "$dir")" done [ -d "$dir" ] || dir="/" # df -P -B1 → POSIX columns, bytes. Field 4 (1-indexed) of the data row is # "Available". Filesystem names can wrap to a second line, so take the last. df -P -B1 "$dir" 2>/dev/null | awk 'END { print $4 }' } # _listen_table → `ss -ltn` output, falling back to `netstat -ltn` when ss is # absent OR present-but-yields-nothing (audit id ss-not-present: minimal images # ship without iproute2, or ship a broken ss). Returns 1 only when NEITHER tool # produced any output, so the caller can treat the probe as inconclusive. _listen_table() { local out="" if command -v ss >/dev/null 2>&1; then out="$(ss -ltn 2>/dev/null)" fi if [ -z "$out" ] && command -v netstat >/dev/null 2>&1; then out="$(netstat -ltn 2>/dev/null)" fi [ -n "$out" ] || return 1 printf '%s' "$out" } # _port_holder → best-effort "name (pid)" of the process holding , # or empty. Uses `ss -ltnp` (needs root for the process column); silent on fail. _port_holder() { local port="$1" out command -v ss >/dev/null 2>&1 || return 0 out="$(ss -ltnp "sport = :$port" 2>/dev/null | grep -oE 'users:\(\("[^"]+"' | head -1)" out="${out##*\"}" out="${out%%\"*}" printf '%s' "$out" } # _box_public_ip → this box's public IPv4 via the two best-effort services in # the research (api.ipify.org, then ifconfig.co). Empty on total failure. _box_public_ip() { local ip ip="$(curl -fsS --max-time 5 https://api.ipify.org 2>/dev/null || true)" if [ -z "$ip" ]; then ip="$(curl -fsS --max-time 5 https://ifconfig.co 2>/dev/null || true)" fi # Strip stray whitespace/newlines. printf '%s' "$ip" | tr -d '[:space:]' } # _resolve_domain → newline-separated A-record IPs. Query REAL public # DNS first (dig/host bypass /etc/hosts); getent is LAST because it consults # /etc/hosts, where a box whose hostname == the panel FQDN (the cloud-init norm) # has 127.0.0.1/::1 entries that shadow the real A record. Empty if unresolved. _resolve_domain() { local domain="$1" if command -v dig >/dev/null 2>&1; then dig +short A "$domain" 2>/dev/null elif command -v host >/dev/null 2>&1; then host -t A "$domain" 2>/dev/null | awk '/has address/ { print $NF }' elif command -v getent >/dev/null 2>&1; then getent ahostsv4 "$domain" 2>/dev/null | awk '{ print $1 }' | sort -u fi } # _strip_loopback → echo only the non-loopback IPs (drop 127.x, ::1, # ::, 0.0.0.0). A box whose hostname == the panel FQDN resolves the FQDN to # loopback via /etc/hosts; that is never the public A record, so it must not # masquerade as one. Pure — bats-tested. _strip_loopback() { local ip for ip in $1; do case "$ip" in 127.* | ::1 | ::1/* | 0.0.0.0 | ::) continue ;; *) printf '%s\n' "$ip" ;; esac done } # _default_route_table → `ip -4 route show default` output. Honors # ROUTE_TABLE_OVERRIDE (tests inject crafted tables). Returns 1 when `ip` is # absent so the caller treats route assessment as inconclusive (skip, never warn). _default_route_table() { if [ -n "${ROUTE_TABLE_OVERRIDE:-}" ]; then printf '%s\n' "$ROUTE_TABLE_OVERRIDE" return 0 fi command -v ip >/dev/null 2>&1 || return 1 ip -4 route show default 2>/dev/null } # _netplan_present → 0 when this host is netplan-managed (the only network stack # the opt-in auto-fix supports). RHEL/NetworkManager and systemd-networkd-only # boxes return non-zero; the caller then warns instead of mutating. _netplan_present() { command -v netplan >/dev/null 2>&1 && [ -d /etc/netplan ] } # _apply_route_metric_fix → deprioritize the private interface 's # default route via a reversible netplan drop-in (dhcp4-overrides.route-metric), # then `netplan apply`. Idempotent (skips if the drop-in already exists). Only # ever touches the PRIVATE NIC — never the public default route. _apply_route_metric_fix() { local dev="$1" if [ -e "$ROUTE_FIX_FILE" ]; then info "Default-route fix already present at ${ROUTE_FIX_FILE} — leaving it untouched (idempotent)." return 0 fi cat >"$ROUTE_FIX_FILE" </dev/null || true if netplan apply >/dev/null 2>&1; then info "Fixed default-route priority: deprioritized ${dev} to metric ${ROUTE_FIX_METRIC} (public NIC now preferred). Wrote ${ROUTE_FIX_FILE}; revert by deleting it and running 'netplan apply'." else warn "Wrote ${ROUTE_FIX_FILE} but 'netplan apply' failed — run 'netplan apply' manually (or reboot) to finish deprioritizing ${dev}'s default route." fi } # ── GPU detection (AI-419) ──────────────────────────────────────────────────── # The installer needs to know whether THIS host has an NVIDIA GPU so it can # install + configure the NVIDIA Container Toolkit (without it, a GPU-aware # deploy silently can't start a container). This mirrors internal/gpu/detect.go's # sysfs PCI scan so the installer and the running panel agree on "GPU present?". # nvidia_pci_present <"vendor class" table> → 0 if any row is an NVIDIA (0x10de) # GPU-class device. GPU class is 0x0300xx (VGA controller) or 0x0302xx (3D # controller — how datacenter GPUs enumerate). Each row is a device's sysfs # `vendor` then `class` value, whitespace-separated (e.g. "0x10de 0x030000"). # Pure (no I/O) so bats can exercise it with crafted tables. nvidia_pci_present() { local table="$1" vendor class while read -r vendor class _; do case "$vendor" in 0x10de | 10de) ;; *) continue ;; esac case "$class" in 0x0300* | 0x0302* | 0300* | 0302*) return 0 ;; esac done </dev/null 2>&1 && nvidia-smi >/dev/null 2>&1; then return 0 fi [ -e /proc/driver/nvidia/version ] && return 0 [ -e /dev/nvidia0 ] && return 0 return 1 } # ── individual checks (each its own function; HARD ones echo a fix + return 2) ─ # check_root — EUID 0. Non-root tolerated only under DRY_RUN (mirrors # install.sh's check_root, which we preserve the intent of). check_root() { # EUID_OVERRIDE lets bats exercise the non-root path without assigning the # readonly $EUID (mirrors platform.sh's ARCH_OVERRIDE). Falls back to the real # $EUID, then `id -u`. local euid="${EUID_OVERRIDE:-${EUID:-$(id -u)}}" if [ "$euid" -eq 0 ]; then info "Running as root." return 0 fi if [ "${DRY_RUN:-false}" = "true" ]; then warn "Not running as root — dry-run mode allows non-root execution." return 0 fi pf_fail "Must run as root. Fix: re-run with 'sudo bash $0' (or as the root user)." return "$PREFLIGHT_EXIT_MISSING_PREREQ" } # check_systemd — systemctl present AND PID 1 is systemd. We need systemd- # managed Docker (`systemctl enable --now docker`), so a non-systemd init # (OpenRC/SysV/inside an unmanaged container) is a hard fail. check_systemd() { if command -v systemctl >/dev/null 2>&1 && [ -d /run/systemd/system ]; then info "systemd is PID 1 (required for systemd-managed Docker)." return 0 fi pf_fail "systemd is required (we manage Docker via systemctl) but PID 1 is not systemd. Fix: install on a systemd-based OS (Ubuntu 22.04+, Debian 12+, Rocky/AlmaLinux 9+); Alpine/OpenRC and non-systemd containers are unsupported." return "$PREFLIGHT_EXIT_MISSING_PREREQ" } # check_arch_supported — 64-bit only. Honors ARCH_OVERRIDE for tests; otherwise # reads `uname -m`. Reuses platform.sh's detect_arch when sourced alongside, but # the verdict comes from the pure arch_supported predicate either way. check_arch_supported() { local m="${ARCH_OVERRIDE:-$(uname -m)}" if arch_supported "$m"; then info "Architecture ${m} is 64-bit (supported)." return 0 fi pf_fail "Unsupported architecture '${m}'. Fix: install on a 64-bit host — x86_64/amd64 or aarch64/arm64. 32-bit (i686/armv7l) is not supported." return "$PREFLIGHT_EXIT_MISSING_PREREQ" } # check_virt — reject OpenVZ and (unprivileged) LXC: rootful Docker can't run # there. KVM/none/docker/cloud hypervisors pass. check_virt() { local v v="$(_detect_virt)" if virt_supported "$v"; then info "Virtualization '${v}' supports rootful Docker." return 0 fi pf_fail "Virtualization '${v}' cannot run rootful Docker (OpenVZ / unprivileged LXC lack the required kernel capabilities). Fix: use a KVM/Xen VM or a bare-metal/dedicated host. Ask your provider for a 'KVM' plan, not an 'OpenVZ/LXC container' plan." return "$PREFLIGHT_EXIT_MISSING_PREREQ" } # check_disk — >= 5 GiB free on the install dir (default /opt/aiadminpanel). check_disk() { local dir="${INSTALL_DIR:-/opt/aiadminpanel}" bytes gib bytes="$(_free_bytes "$dir")" if disk_ok "$bytes"; then gib=$((bytes / 1024 / 1024 / 1024)) info "Disk space OK: ${gib} GiB free at ${dir} (need >= 5 GiB)." return 0 fi pf_fail "Not enough free disk space at ${dir} (need >= 5 GiB; the Docker images alone are several GiB). Fix: free space ('docker system prune -af', remove old logs/backups) or attach a larger volume, then re-run." return "$PREFLIGHT_EXIT_MISSING_PREREQ" } # check_ports_free — 80 and 443 must be free (Traefik binds them for TLS). Probe # with ss, fall back to netstat. Name the holder when we can and print the exact # stop/disable command. Preserves the intent of install.sh's check_ports. check_ports_free() { local table port holder failed=0 if ! table="$(_listen_table)"; then warn "Neither 'ss' nor 'netstat' is available — cannot verify ports 80/443 are free; continuing (the stack will fail loudly at bind time if they're held)." return 0 fi for port in 80 443; do if port_in_use "$table" "$port"; then holder="$(_port_holder "$port")" if [ -n "$holder" ]; then pf_fail "Port ${port} is held by '${holder}'. Fix: 'systemctl stop ${holder} && systemctl disable ${holder}' (or remove the conflicting service), then re-run." else pf_fail "Port ${port} is already in use. Fix: identify the holder with 'ss -ltnp \"sport = :${port}\"', stop/disable that service, then re-run." fi failed=1 else info "Port ${port} is free." fi done [ "$failed" -eq 0 ] && return 0 return "$PREFLIGHT_EXIT_MISSING_PREREQ" } # check_outbound — best-effort reachability to the endpoints the install pulls # from. WARN only (proxied/egress-filtered networks are legitimate); never a # hard fail. check_outbound() { local host url unreachable=() for host in ghcr.io get.docker.com acme-v02.api.letsencrypt.org; do url="https://${host}" if ! curl -fsS --max-time 5 -o /dev/null "$url" 2>/dev/null; then unreachable+=("$host") fi done if [ "${#unreachable[@]}" -eq 0 ]; then info "Outbound reachability OK (ghcr.io, get.docker.com, Let's Encrypt)." return 0 fi warn "Could not reach: ${unreachable[*]}. The install pulls images and a TLS cert from these — if they stay blocked the install will fail later. Check egress firewall/proxy/DNS, then re-run if needed." return 0 } # check_dns — only when PANEL_DOMAIN is set and TLS_MODE != cloudflare. Resolve # the domain and compare to this box's public IP. A MISMATCH is a HARD stop: # letting ACME HTTP-01 hammer a wrong/missing record risks a week-long Let's # Encrypt rate-limit lockout on the domain (research risk #5). check_dns() { local domain="${PANEL_DOMAIN:-}" mode="${TLS_MODE:-letsencrypt}" if [ -z "$domain" ]; then return 0 # nothing to validate yet (interactive prompt happens later) fi if [ "$mode" = "cloudflare" ]; then info "TLS_MODE=cloudflare — skipping public A-record check (DNS-01, proxied)." return 0 fi local resolved box_ip resolved="$(_resolve_domain "$domain")" box_ip="$(_box_public_ip)" if [ -z "$box_ip" ]; then warn "Could not determine this box's public IP (api.ipify.org/ifconfig.co unreachable) — skipping the DNS match check. Make sure ${domain} points here before the TLS step or Let's Encrypt will fail." return 0 fi if [ -z "$resolved" ]; then pf_fail "${domain} does not resolve to any A record. Fix: point the A record for ${domain} at ${box_ip} (this box) and re-run. (Skipping this lets Let's Encrypt HTTP-01 fail repeatedly and can lock the domain out for a week.)" return "$PREFLIGHT_EXIT_MISSING_PREREQ" fi # Ignore loopback (127.x / ::1): a box whose hostname == ${domain} resolves it # to loopback via /etc/hosts (the cloud-init norm), which is never the public A # record. So a loopback-ONLY result is "inconclusive" → warn-and-continue (don't # break valid cloud installs); a non-loopback mismatch is a positive hard fail. local public public="$(_strip_loopback "$resolved" | tr '\n' ' ')" if [ -z "${public// /}" ]; then warn "Could not confirm the public A record for ${domain} from this host (it may be shadowed by an /etc/hosts hostname entry — common on cloud images). Ensure ${domain} points at ${box_ip} before the TLS step or Let's Encrypt will fail. Proceeding." return 0 fi if dns_matches "$public" "$box_ip"; then info "DNS OK: ${domain} resolves to ${box_ip} (this box)." return 0 fi pf_fail "${domain} resolves to '${public}' but this box is ${box_ip}. Fix: point the A record for ${domain} at ${box_ip} and re-run. (Re-running against a wrong record risks a week-long Let's Encrypt rate-limit lockout — use TLS_MODE=cloudflare for DNS-01 if you can't change the A record.)" return "$PREFLIGHT_EXIT_MISSING_PREREQ" } # tls_cert_resolver — map TLS_MODE to the Traefik certresolver the panel/keycloak # router labels must use. cloudflare → the DNS-01 resolver (letsencrypt-dns), # which works on hosts where inbound :80 is closed/proxied; anything else → the # HTTP-01 resolver (letsencrypt). The installer writes the result as CERT_RESOLVER # in .env; docker-compose.yml reads ${CERT_RESOLVER:-letsencrypt} on the routers. # Pinning AI-425: before this, cloudflare mode defined the DNS-01 resolver but the # routers still pinned letsencrypt (HTTP-01), so DNS-01 was wired yet never used. tls_cert_resolver() { case "${TLS_MODE:-letsencrypt}" in cloudflare) echo "letsencrypt-dns" ;; *) echo "letsencrypt" ;; esac } # check_time_sync — NTP synchronized? Clock skew breaks ACME JWS verification # (badNonce / signature failure → silent self-signed cert). WARN with the # chrony/timesyncd hint; the caller decides whether to auto-enable NTP (this # function never mutates). check_time_sync() { local synced="" if command -v timedatectl >/dev/null 2>&1; then synced="$(timedatectl show -p NTPSynchronized --value 2>/dev/null || true)" fi if [ "$synced" = "yes" ]; then info "Clock is NTP-synchronized." return 0 fi warn "Clock is NOT NTP-synchronized — skew breaks Let's Encrypt (badNonce/signature errors → silent self-signed cert). Fix: 'timedatectl set-ntp true' (Debian/Ubuntu: systemd-timesyncd; RHEL/AL2023: 'dnf install -y chrony && systemctl enable --now chronyd'), then re-check with 'timedatectl'." return 0 } # check_cgroup_v2 — WARN only when cgroup v2 is absent (v1 long tail: Amazon # Linux 2, CentOS 7). Docker still runs on v1, so this never hard-fails. check_cgroup_v2() { if [ -f /sys/fs/cgroup/cgroup.controllers ]; then info "cgroup v2 is active." return 0 fi warn "cgroup v1 detected (no /sys/fs/cgroup/cgroup.controllers). Docker still runs, but v1 is deprecated (Docker Engine v29 / systemd v258 drop it). Recommended: switch to cgroup v2 ('grubby --update-kernel=ALL --args=systemd.unified_cgroup_hierarchy=1 && reboot') or use a v2-default image (Amazon Linux 2023, Ubuntu 22.04+, Rocky/Alma 9+)." return 0 } # check_default_routes — WARN only (soft gate). Detect the multi-homed # dual-default-route footgun (AI-423). Default: warn with the offending NIC + the # fix + the AAP_FIX_ROUTING hint. With AAP_FIX_ROUTING=1 on a netplan host: apply # the reversible drop-in automatically. Never changes the install verdict. check_default_routes() { local table offender dev metric if ! table="$(_default_route_table)" || [ -z "$table" ]; then return 0 # no `ip` / no default routes to assess — nothing to do fi if ! offender="$(dangerous_default_routes "$table")"; then info "Default route OK (no conflicting private-network default)." return 0 fi dev="${offender%% *}" metric="${offender##* }" if [ "${AAP_FIX_ROUTING:-}" = "1" ] || [ "${AAP_FIX_ROUTING:-}" = "true" ]; then if _netplan_present; then _apply_route_metric_fix "$dev" return 0 fi warn "AAP_FIX_ROUTING is set but this host is not netplan-managed — auto-fix is netplan-only for now. Manually give '${dev}'s default route a higher metric than your public NIC so Docker container traffic stops egressing '${dev}'." return 0 fi warn "Two default routes detected: a private-network default on '${dev}' (metric ${metric}) at the same-or-better priority as your public default. Docker container return traffic can exit '${dev}' and be silently dropped — the panel becomes unreachable and containers can't reach the internet. Fix: deprioritize '${dev}'s default route, or re-run the installer with AAP_FIX_ROUTING=1 to apply it automatically (netplan hosts). Details: https://docs.aiadminpanel.com/docs/deployment/multi-homed-vps-routing" return 0 } # ── orchestrator ────────────────────────────────────────────────────────────── # preflight_checks — run every gate. Hard gates run first (cheapest, most # fundamental → DNS last since it does network I/O). Soft gates always run so # the operator sees ALL warnings in one pass. Returns 2 if any hard gate failed. preflight_checks() { info "Running pre-flight checks (fail fast before any changes)..." local hard_failed=0 rc # Hard gates — accumulate failures so the operator sees every blocker at once, # rather than fixing one and re-running to discover the next. local check for check in check_root check_systemd check_arch_supported check_virt \ check_disk check_ports_free check_dns; do "$check" rc=$? [ "$rc" -ne 0 ] && hard_failed=1 done # Soft gates — WARN only, never change the verdict. check_outbound check_default_routes check_time_sync check_cgroup_v2 if [ "$hard_failed" -ne 0 ]; then pf_fail "Pre-flight failed — fix the blocker(s) above and re-run. (No changes were made to this box.)" return "$PREFLIGHT_EXIT_MISSING_PREREQ" fi info "Pre-flight checks passed." return 0 } # ===== scripts/lib/ai.sh ===== # shellcheck disable=SC2034 # this library SETS globals (METERED_*) that are read # by install.sh.in's setup_litellm_config / pull_metered_model — shellcheck can't # see those cross-file reads when it lints this module in isolation. # scripts/lib/ai.sh — metered-AI model selection for the bundled Ollama singleton. # # The metered operator AI is served by a panel-managed Ollama singleton # (aiadminpanel_ollama) behind the LiteLLM gateway. The gateway mounts its config # read-only and has NO runtime reload path, so the model must be chosen at install # time — and it must match what Ollama actually has pulled, or the first metered # call 404s. deepseek-r1:7b (~8GB working set) also OOMs the small marketplace # boxes (DigitalOcean/Vultr) that our auto-deploy targets. # # So we RAM-gate (AI-622): deepseek-r1:7b on capable boxes, a small Llama-3B # fallback on small ones, and the installer auto-pulls whichever it picked. # # Design decision (AI-622, "stable alias"): the gateway model_name stays # `deepseek-r1` on EVERY box — it is the invoke key three Go call sites hardcode # as the VPS default (chat_handler.go, analyze_handler.go, aideploy/openai_client.go). # Renaming it would 404 those calls on exactly the small boxes we protect. On a # small box it simply routes `deepseek-r1` → the real llama3.2:3b at the honest # 3B price; the AI Models page shows the operator the model actually running, so # nobody is misled. See docs-v2 concepts/ai.md. # # Pure, sourceable functions — unit-tested directly in tests/installer/ai_metered_model.bats. # ── Model tiers ────────────────────────────────────────────────────────────── # The gateway's stable invoke name (never varies — see header). Backend defaults # to this string when no model is configured. AAP_METERED_MODEL_NAME="deepseek-r1" # Capable-box model: the showcase reasoning model. AAP_METERED_DEEPSEEK_TAG="deepseek-r1:7b" # 4.7GB download, ~8GB working set AAP_METERED_DEEPSEEK_IN="0.0000001" # operator selling price: $0.10 / 1M input tokens AAP_METERED_DEEPSEEK_OUT="0.0000004" # $0.40 / 1M output tokens # Small-box fallback: current-best small general model (verified against the live # Ollama library at wiring, 2026-07-18). Priced at half the 7B rate — honestly # cheaper for a smaller model. Local models cost $0 in LiteLLM's map, so without a # price here metered budgets never accrue spend (spec M2). AAP_METERED_FALLBACK_TAG="llama3.2:3b" # 2.0GB download, ~3-4GB working set AAP_METERED_FALLBACK_IN="0.00000005" # $0.05 / 1M input tokens AAP_METERED_FALLBACK_OUT="0.0000002" # $0.20 / 1M output tokens # Threshold (total RAM, MB). deepseek-r1:7b wants ~8GB and the box also runs # Postgres/Keycloak/Traefik/panel/litellm. 12000MB cleanly separates the real # marketplace tiers: an 8GB box reports ~7900MB MemTotal (→ fallback), a 16GB box # ~15800MB (→ deepseek). Overridable for tests. AAP_METERED_MIN_MB_DEEPSEEK="${AAP_METERED_MIN_MB_DEEPSEEK:-12000}" # _detect_total_mem_mb — echo total RAM in MB, read from MemTotal (KB) in # /proc/meminfo. Honors MEMINFO_FILE so tests can feed a fixture. Echoes 0 if # unreadable — which classifies as "small" and picks the safe fallback. _detect_total_mem_mb() { local meminfo="${MEMINFO_FILE:-/proc/meminfo}" kb kb="$(awk '/^MemTotal:/ { print $2; exit }' "$meminfo" 2>/dev/null || true)" if [ -n "$kb" ] && [ "$kb" -gt 0 ] 2>/dev/null; then echo "$(( kb / 1024 ))" else echo 0 fi } # select_metered_model — decide the bundled Ollama model from detected RAM and set: # METERED_OLLAMA_TAG the tag to pull + route to (deepseek-r1:7b | llama3.2:3b) # METERED_INPUT_COST per-token selling price (input) # METERED_OUTPUT_COST per-token selling price (output) # METERED_IS_FALLBACK "true" on a small box, "false" on a capable one # METERED_TOTAL_MB detected total RAM in MB (for the UI/log message) # The gateway model_name is always AAP_METERED_MODEL_NAME (stable alias). select_metered_model() { local mb mb="$(_detect_total_mem_mb)" METERED_TOTAL_MB="$mb" if [ "$mb" -ge "$AAP_METERED_MIN_MB_DEEPSEEK" ]; then METERED_OLLAMA_TAG="$AAP_METERED_DEEPSEEK_TAG" METERED_INPUT_COST="$AAP_METERED_DEEPSEEK_IN" METERED_OUTPUT_COST="$AAP_METERED_DEEPSEEK_OUT" METERED_IS_FALLBACK="false" else METERED_OLLAMA_TAG="$AAP_METERED_FALLBACK_TAG" METERED_INPUT_COST="$AAP_METERED_FALLBACK_IN" METERED_OUTPUT_COST="$AAP_METERED_FALLBACK_OUT" METERED_IS_FALLBACK="true" fi } # render_litellm_config — emit the LiteLLM proxy config for the selected model to # stdout. Auto-selects if select_metered_model has not run yet. Deterministic: # given the same RAM it always produces the same config. render_litellm_config() { [ -n "${METERED_OLLAMA_TAG:-}" ] || select_metered_model cat <= ${AAP_METERED_MIN_MB_DEEPSEEK}MB total RAM -> ${AAP_METERED_DEEPSEEK_TAG}, else ${AAP_METERED_FALLBACK_TAG}. # The model_name stays '${AAP_METERED_MODEL_NAME}' on every box (stable invoke key); # on a small box it routes to the real fallback model at its own price. Edit freely; # a re-run keeps an existing file. model_list: - model_name: ${AAP_METERED_MODEL_NAME} litellm_params: model: ollama_chat/${METERED_OLLAMA_TAG} api_base: http://ollama:11434 model_info: # Operator's SELLING price for the bundled local model (USD per token). # Local models are \$0 in LiteLLM's cost map — without a price here, # metered budgets never accrue spend for local usage (spec M2). # Edit to your own price list; a pricing UI is a filed follow-up. input_cost_per_token: ${METERED_INPUT_COST} output_cost_per_token: ${METERED_OUTPUT_COST} general_settings: master_key: os.environ/LITELLM_MASTER_KEY EOF } setup_colors # (Root / port / disk / DNS / systemd / virtualization checks are now provided by # lib/preflight.sh via preflight_checks().) # ── Domain Prompt ──────────────────────────────────────────────────────────── prompt_domain() { if [ -n "$PANEL_DOMAIN" ]; then info "Using domain from environment: ${PANEL_DOMAIN}" elif [ "${MANAGED_DOMAIN:-}" = "off" ]; then # Opt-out: exactly the pre-AI-544 behavior. Order matches pre-AI-544 # (unattended aborts even under --dry-run) so a dry-run of an unattended # opt-out install still surfaces the missing-PANEL_DOMAIN error instead of # masking it. off is not a claimable config, so the claim-path dry-run # short-circuit below deliberately does not apply here. if [ "$UNATTENDED" = "true" ]; then error "PANEL_DOMAIN must be set when using --unattended mode (MANAGED_DOMAIN=off)" elif [ "$DRY_RUN" = "true" ]; then PANEL_DOMAIN="dry-run.example.com" dryrun "Would prompt for domain, using placeholder: ${PANEL_DOMAIN}" elif [ ! -t 0 ]; then error "PANEL_DOMAIN is required. Download first, then run interactively: curl -fsSL https://get.aiadminpanel.com -o install.sh && bash install.sh" else echo "" read -rp "${COLOR_BOLD}Enter panel domain (e.g., panel.example.com): ${COLOR_RESET}" PANEL_DOMAIN if [ -z "$PANEL_DOMAIN" ]; then error "Domain is required. Set PANEL_DOMAIN env var or provide interactively." fi info "Domain set to: ${PANEL_DOMAIN}" fi elif [ "$DRY_RUN" = "true" ]; then # Claimable dry-run (no PANEL_DOMAIN, managed path enabled): never abort — # narrate the claim. Wins over the --unattended abort below by design. PANEL_DOMAIN="dry-run.example.com" dryrun "Would claim temporary domain from ${MANAGED_DNS_URL} (no PANEL_DOMAIN set); using placeholder: ${PANEL_DOMAIN}" elif [ "$UNATTENDED" = "true" ] || [ ! -t 0 ]; then # No domain and no human (marketplace / curl|bash): auto-claim (AI-544). md_acquire_domain || error "$MD_FAIL_REASON" else # Interactive: blank Enter = claim a temporary domain; failure re-prompts. while [ -z "$PANEL_DOMAIN" ]; do echo "" read -rp "${COLOR_BOLD}Enter panel domain (or press Enter to get a free temporary domain like aap-x7k2f9.aiadminpanel.host): ${COLOR_RESET}" PANEL_DOMAIN if [ -n "$PANEL_DOMAIN" ]; then info "Domain set to: ${PANEL_DOMAIN}" elif md_acquire_domain; then break else warn "$MD_FAIL_REASON" warn "Enter your own domain, or press Enter again to retry the claim." fi done fi # Derive ACME email from domain if not already set. Runs after every # success path (env, dry-run, interactive) so the .env file always # carries a real email. Without this, the docker-compose.yml fallback # `${ACME_EMAIL:-admin@example.com}` leaks through and Let's Encrypt # rejects example.com on registration (AI-239, demo2 2026-05-06). if [ -z "$ACME_EMAIL" ]; then ACME_EMAIL="admin@${PANEL_DOMAIN}" info "ACME email set to: ${ACME_EMAIL}" fi } # ── License Key Prompt ──────────────────────────────────────────────────────── prompt_license() { if [ -n "${LICENSE_KEY:-}" ]; then info "Using license key from environment" return 0 fi if [ "$UNATTENDED" = "true" ]; then info "No LICENSE_KEY set — panel will start a 14-day trial on first setup" return 0 fi if [ "$DRY_RUN" = "true" ]; then dryrun "Would prompt for license key (optional)" return 0 fi # When piped (curl | bash), stdin is not a terminal — skip optional prompt if [ ! -t 0 ]; then info "Non-interactive mode — skipping license key prompt (14-day trial will activate)" return 0 fi echo "" read -rp "${COLOR_BOLD}License Key (optional — press Enter for 14-day trial): ${COLOR_RESET}" LICENSE_KEY if [ -n "$LICENSE_KEY" ]; then info "License key provided — will be written to .env" else info "No license key — panel will start a 14-day full-featured trial on first setup" fi } # ── Platform Detection + Support Gate ───────────────────────────────────────── # Replaces the old Ubuntu/Debian-only detect_os(). Uses lib/platform.sh to detect # the distro family, package manager and architecture, then gates on the support # tier: certified → proceed; best-effort/document → warn and continue; reject → # abort with a specific message naming what IS supported (AI-349). detect_and_gate_platform() { detect_platform || true info "Detected: ${OS_PRETTY:-$OS_ID} (family=${OS_FAMILY}, version=${OS_VERSION_ID:-?})" case "$PLATFORM_TIER" in certified) info "Supported distribution: ${PLATFORM_REASON}" ;; best-effort) warn "Best-effort distribution (not in our CI-certified matrix): ${PLATFORM_REASON}" warn "Proceeding anyway — please report any problems." ;; document) warn "Partially-supported distribution: ${PLATFORM_REASON}" warn "Proceeding best-effort — see docs.aiadminpanel.com for manual steps if it fails." ;; reject | *) error "${PLATFORM_REASON}" ;; esac detect_pm [ "$PKG_MANAGER" != "unknown" ] || error "No supported package manager found (need one of apt/dnf/yum/zypper/pacman)." detect_arch || error "Unsupported architecture: $(uname -m) (need x86_64 or aarch64/arm64)." info "Platform: ${OS_PRETTY:-$OS_ID} | package manager: ${PKG_MANAGER} | arch: ${ARCH} (${BINARY_SUFFIX})" } # (detect_arch is now provided by lib/platform.sh.) # (Port 80/443 conflict check is now in lib/preflight.sh check_ports_free, run as # part of preflight_checks().) # ── Docker daemon storage-driver hardening ──────────────────────────────────── # The universal Docker engine + compose/buildx install is provided by # lib/docker.sh install_docker() (every distro, with tooling verification). This # helper keeps the overlay2 fix that the old inline installer applied: it works # around the containerd v2.2.x overlayfs snapshotter COPY bug by pinning Docker's # native overlay2 graphdriver. Called right after install_docker on a fresh box. configure_docker_daemon() { if [ "$DRY_RUN" = "true" ]; then dryrun "Ensure /etc/docker/daemon.json sets storage-driver overlay2" return 0 fi if [ ! -f /etc/docker/daemon.json ]; then mkdir -p /etc/docker cat >/etc/docker/daemon.json <<'DAEMONJSON' { "storage-driver": "overlay2" } DAEMONJSON systemctl restart docker 2>/dev/null || true info "Docker configured with overlay2 storage driver" fi } # ── Directory Setup ─────────────────────────────────────────────────────────── create_directories() { info "Creating panel directories..." # update/ is the host-shared channel between the panel and the systemd updater; # backups/ holds the pre-update pg_dumps the updater takes (AI-431 2B). for dir in "$PANEL_DIR" "$CONFIG_DIR" "$LOG_DIR" "$SECRETS_DIR" "$PANEL_DIR/update" "$PANEL_DIR/backups"; do if [ -d "$dir" ]; then debug "Directory exists: $dir" else run mkdir -p "$dir" info "Created: $dir" fi done } # ── Secret Generation ───────────────────────────────────────────────────────── # Thin dry-run-aware wrapper over lib/secrets.sh generate_secrets(), which is # portable (openssl/od, no GNU-only base64 -w0 / xxd), writes EVERY secret 600 # (fixes the db_password/keycloak_admin_password world-readable 644 leak), is # idempotent (never overwrites an existing secret), and additionally generates a # per-install litellm_master_key (no more shared sk-litellm-master-key default). setup_secrets() { if [ "$DRY_RUN" = "true" ]; then dryrun "Migrate any legacy /run/secrets/* into ${SECRETS_DIR}, then generate master_key, db_password, keycloak_admin_password, litellm_master_key, managed_domain_token, openbao_unseal_key in ${SECRETS_DIR} (dir 700, files 600 except openbao_unseal_key 644 — read by the openbao container's non-root server), render the OpenBao config at ${PANEL_DIR}/openbao/config/openbao.hcl (dir 755, file 644), and render the audit-log rotation script at ${PANEL_DIR}/openbao/openbao-log-rotate.sh (755, bounds audit.log for the openbao_log_rotator sidecar — AI-611)" return 0 fi # AI-397: on a box upgraded from a pre-AI-397 install that has NOT yet # rebooted, the live secrets still sit in the legacy tmpfs /run/secrets. # Carry them into the persistent dir BEFORE generating, so the existing # postgres password is preserved rather than regenerated into a mismatch. migrate_legacy_secrets "${SECRETS_DIR}" info "Generating secrets in ${SECRETS_DIR}..." generate_secrets "${SECRETS_DIR}" write_openbao_config "${PANEL_DIR}/openbao/config" # Bound the OpenBao audit log so it can never fill the disk again (AI-611). # Rendered next to config/ (a single-file :ro bind mount into the sidecar). write_openbao_log_rotate_script "${PANEL_DIR}/openbao" info "Secrets ready (master_key, db_password, keycloak_admin_password, litellm_master_key, managed_domain_token, openbao_unseal_key)" } # ── Keycloak Setup ──────────────────────────────────────────────────────────── setup_keycloak() { info "Setting up Keycloak identity server..." # Create Keycloak database in PostgreSQL if [ "$DRY_RUN" = "true" ]; then dryrun "CREATE DATABASE keycloak" return 0 fi # Run DB setup via docker exec (postgres container is already running) docker exec aiadminpanel_postgresql psql -U aiadminpanel -c \ "SELECT 1 FROM pg_database WHERE datname='keycloak'" 2>/dev/null | grep -q "1 row" || \ docker exec aiadminpanel_postgresql psql -U aiadminpanel -c \ "CREATE DATABASE keycloak;" 2>/dev/null || true info "Keycloak database configured" # Download realm export for auto-import on first boot local realm_export="${PANEL_DIR}/keycloak-realm-export.json" if [ ! -f "$realm_export" ]; then # Try downloading from GitHub Release first, then get server as fallback if curl -fsSL "${GITHUB_DOWNLOAD_BASE}/keycloak-realm-export.json" -o "$realm_export" 2>/dev/null; then chmod 644 "$realm_export" info "Realm export downloaded from GitHub Release" elif curl -fsSL "${GET_BASE}/keycloak-realm-export.json" -o "$realm_export" 2>/dev/null; then chmod 644 "$realm_export" info "Realm export downloaded from get server" else # Try local fallbacks local bundled_paths=( "$(dirname "$(readlink -f "$0")")/../keycloak-realm-export.json" "/tmp/aiadminpanel-keycloak-realm-export.json" ) for path in "${bundled_paths[@]}"; do if [ -f "$path" ]; then cp "$path" "$realm_export" chmod 644 "$realm_export" info "Realm export copied from ${path}" break fi done fi [ -f "$realm_export" ] || warn "Keycloak realm export not found — Keycloak will start with default realm. Import manually." else info "Keycloak realm export already exists at ${realm_export}" fi # Download Keycloak login theme (branded dark theme with logo) local theme_dir="${PANEL_DIR}/keycloak-theme/login/resources/css" if [ ! -d "$theme_dir" ]; then info "Downloading Keycloak login theme..." mkdir -p "$theme_dir" if curl -fsSL "${GITHUB_DOWNLOAD_BASE}/keycloak-theme.tar.gz" -o /tmp/keycloak-theme.tar.gz 2>/dev/null \ || curl -fsSL "${GET_BASE}/keycloak-theme.tar.gz" -o /tmp/keycloak-theme.tar.gz 2>/dev/null; then tar -xzf /tmp/keycloak-theme.tar.gz -C "${PANEL_DIR}/" rm -f /tmp/keycloak-theme.tar.gz info "Keycloak theme installed" else # Fallback: try individual files from get server curl -fsSL "${GET_BASE}/keycloak-theme/login/theme.properties" -o "${PANEL_DIR}/keycloak-theme/login/theme.properties" 2>/dev/null || true curl -fsSL "${GET_BASE}/keycloak-theme/login/resources/css/aiadminpanel.css" -o "${theme_dir}/aiadminpanel.css" 2>/dev/null || true curl -fsSL "${GET_BASE}/keycloak-theme/login/resources/css/logo.png" -o "${theme_dir}/logo.png" 2>/dev/null || true info "Keycloak theme installed (fallback)" fi else info "Keycloak theme already exists" fi } # 3a-2: LiteLLM's virtual-key/budget machinery needs its own database # (Prisma-managed). Same pattern as the keycloak DB: shared postgres # instance, shared aiadminpanel role, separate database. Idempotent. setup_litellm_db() { if [ "$DRY_RUN" = "true" ]; then dryrun "Create the litellm database in the bundled Postgres (idempotent)" return 0 fi docker exec aiadminpanel_postgresql psql -U aiadminpanel -c \ "SELECT 1 FROM pg_database WHERE datname='litellm'" 2>/dev/null | grep -q "1 row" || \ docker exec aiadminpanel_postgresql psql -U aiadminpanel -c \ "CREATE DATABASE litellm;" 2>/dev/null || true info "LiteLLM database ready" } create_river_tables() { info "Creating River queue tables (if not exist)..." if [ "$DRY_RUN" = "true" ]; then dryrun "CREATE river_migration, river_job, river_leader, river_queue, river_client tables" return 0 fi # Check if river tables already exist (idempotent) if docker exec aiadminpanel_postgresql psql -U aiadminpanel -d aiadminpanel -tAc \ "SELECT 1 FROM information_schema.tables WHERE table_name='river_job'" 2>/dev/null | grep -q "1"; then info "River tables already exist, skipping" return 0 fi # River v0.31.0 migration SQL (from riverdriver/riverpgxv5 migration/main/001-006) # Write to temp file to avoid stdin conflict with curl pipe local sql_file="/tmp/river-migrations.sql" cat > "$sql_file" <<'RIVERSQL' -- Migration 001: Create river_migration table CREATE TABLE IF NOT EXISTS river_migration( id bigserial PRIMARY KEY, created_at timestamptz NOT NULL DEFAULT NOW(), version bigint NOT NULL, CONSTRAINT version CHECK (version >= 1) ); CREATE UNIQUE INDEX IF NOT EXISTS river_migration_version_idx ON river_migration USING btree(version); -- Migration 002: Initial schema (river_job, river_leader) DO $$ BEGIN CREATE TYPE river_job_state AS ENUM( 'available', 'cancelled', 'completed', 'discarded', 'retryable', 'running', 'scheduled' ); EXCEPTION WHEN duplicate_object THEN NULL; END $$; CREATE TABLE IF NOT EXISTS river_job( id bigserial PRIMARY KEY, state river_job_state NOT NULL DEFAULT 'available', attempt smallint NOT NULL DEFAULT 0, max_attempts smallint NOT NULL, attempted_at timestamptz, created_at timestamptz NOT NULL DEFAULT NOW(), finalized_at timestamptz, scheduled_at timestamptz NOT NULL DEFAULT NOW(), priority smallint NOT NULL DEFAULT 1, args jsonb, attempted_by text[], errors jsonb[], kind text NOT NULL, metadata jsonb NOT NULL DEFAULT '{}', queue text NOT NULL DEFAULT 'default', tags varchar(255)[] NOT NULL DEFAULT '{}', CONSTRAINT finalized_or_finalized_at_null CHECK ( (finalized_at IS NULL AND state NOT IN ('cancelled', 'completed', 'discarded')) OR (finalized_at IS NOT NULL AND state IN ('cancelled', 'completed', 'discarded')) ), CONSTRAINT max_attempts_is_positive CHECK (max_attempts > 0), CONSTRAINT priority_in_range CHECK (priority >= 1 AND priority <= 4), CONSTRAINT queue_length CHECK (char_length(queue) > 0 AND char_length(queue) < 128), CONSTRAINT kind_length CHECK (char_length(kind) > 0 AND char_length(kind) < 128) ); CREATE INDEX IF NOT EXISTS river_job_kind ON river_job USING btree(kind); CREATE INDEX IF NOT EXISTS river_job_state_and_finalized_at_index ON river_job USING btree(state, finalized_at) WHERE finalized_at IS NOT NULL; CREATE INDEX IF NOT EXISTS river_job_prioritized_fetching_index ON river_job USING btree(state, queue, priority, scheduled_at, id); CREATE INDEX IF NOT EXISTS river_job_args_index ON river_job USING GIN(args); CREATE INDEX IF NOT EXISTS river_job_metadata_index ON river_job USING GIN(metadata); CREATE UNLOGGED TABLE IF NOT EXISTS river_leader( elected_at timestamptz NOT NULL, expires_at timestamptz NOT NULL, leader_id text NOT NULL, name text PRIMARY KEY DEFAULT 'default', CONSTRAINT name_length CHECK (name = 'default'), CONSTRAINT leader_id_length CHECK (char_length(leader_id) > 0 AND char_length(leader_id) < 128) ); -- Migration 004: Add pending state, create river_queue ALTER TYPE river_job_state ADD VALUE IF NOT EXISTS 'pending' AFTER 'discarded'; CREATE TABLE IF NOT EXISTS river_queue( name text PRIMARY KEY NOT NULL, created_at timestamptz NOT NULL DEFAULT now(), metadata jsonb NOT NULL DEFAULT '{}', paused_at timestamptz, updated_at timestamptz NOT NULL DEFAULT now() ); -- Migration 005: Rebuild river_migration with line support, add unique_key, create river_client DO $body$ BEGIN IF (SELECT to_regclass('river_migration') IS NOT NULL) THEN IF NOT EXISTS (SELECT 1 FROM information_schema.columns WHERE table_name='river_migration' AND column_name='line') THEN ALTER TABLE river_migration RENAME TO river_migration_old; CREATE TABLE river_migration( line TEXT NOT NULL, version bigint NOT NULL, created_at timestamptz NOT NULL DEFAULT NOW(), CONSTRAINT line_length CHECK (char_length(line) > 0 AND char_length(line) < 128), CONSTRAINT version_gte_1 CHECK (version >= 1), PRIMARY KEY (line, version) ); INSERT INTO river_migration (created_at, line, version) SELECT created_at, 'main', version FROM river_migration_old; DROP TABLE river_migration_old; END IF; END IF; END; $body$ LANGUAGE plpgsql; ALTER TABLE river_job ADD COLUMN IF NOT EXISTS unique_key bytea; CREATE UNIQUE INDEX IF NOT EXISTS river_job_kind_unique_key_idx ON river_job (kind, unique_key) WHERE unique_key IS NOT NULL; CREATE UNLOGGED TABLE IF NOT EXISTS river_client( id text PRIMARY KEY NOT NULL, created_at timestamptz NOT NULL DEFAULT now(), metadata jsonb NOT NULL DEFAULT '{}', paused_at timestamptz, updated_at timestamptz NOT NULL DEFAULT now(), CONSTRAINT name_length CHECK (char_length(id) > 0 AND char_length(id) < 128) ); CREATE UNLOGGED TABLE IF NOT EXISTS river_client_queue( river_client_id text NOT NULL REFERENCES river_client(id) ON DELETE CASCADE, name text NOT NULL, created_at timestamptz NOT NULL DEFAULT now(), max_workers bigint NOT NULL DEFAULT 0, metadata jsonb NOT NULL DEFAULT '{}', num_jobs_completed bigint NOT NULL DEFAULT 0, num_jobs_running bigint NOT NULL DEFAULT 0, updated_at timestamptz NOT NULL DEFAULT now(), PRIMARY KEY (river_client_id, name), CONSTRAINT name_length CHECK (char_length(name) > 0 AND char_length(name) < 128), CONSTRAINT num_jobs_completed_zero_or_positive CHECK (num_jobs_completed >= 0), CONSTRAINT num_jobs_running_zero_or_positive CHECK (num_jobs_running >= 0) ); -- Migration 006: Add river_job_state_in_bitmask function and unique_states column CREATE OR REPLACE FUNCTION river_job_state_in_bitmask(bitmask BIT(8), state river_job_state) RETURNS boolean LANGUAGE SQL IMMUTABLE AS $$ SELECT CASE state WHEN 'available' THEN get_bit(bitmask, 7) WHEN 'cancelled' THEN get_bit(bitmask, 6) WHEN 'completed' THEN get_bit(bitmask, 5) WHEN 'discarded' THEN get_bit(bitmask, 4) WHEN 'pending' THEN get_bit(bitmask, 3) WHEN 'retryable' THEN get_bit(bitmask, 2) WHEN 'running' THEN get_bit(bitmask, 1) WHEN 'scheduled' THEN get_bit(bitmask, 0) ELSE 0 END = 1; $$; ALTER TABLE river_job ADD COLUMN IF NOT EXISTS unique_states BIT(8); CREATE UNIQUE INDEX IF NOT EXISTS river_job_unique_idx ON river_job (unique_key) WHERE unique_key IS NOT NULL AND unique_states IS NOT NULL AND river_job_state_in_bitmask(unique_states, state); -- Record migration versions so River recognizes the schema INSERT INTO river_migration (line, version) VALUES ('main', 1) ON CONFLICT DO NOTHING; INSERT INTO river_migration (line, version) VALUES ('main', 2) ON CONFLICT DO NOTHING; INSERT INTO river_migration (line, version) VALUES ('main', 3) ON CONFLICT DO NOTHING; INSERT INTO river_migration (line, version) VALUES ('main', 4) ON CONFLICT DO NOTHING; INSERT INTO river_migration (line, version) VALUES ('main', 5) ON CONFLICT DO NOTHING; INSERT INTO river_migration (line, version) VALUES ('main', 6) ON CONFLICT DO NOTHING; RIVERSQL docker cp "$sql_file" aiadminpanel_postgresql:/tmp/river-migrations.sql local river_rc=0 docker exec aiadminpanel_postgresql psql -U aiadminpanel -d aiadminpanel -f /tmp/river-migrations.sql 2>&1 | tail -5 || river_rc=$? if [ "$river_rc" -eq 0 ]; then info "River queue tables created successfully" else warn "River table creation had errors (may be partially created -- panel will retry)" fi rm -f "$sql_file" } wait_for_keycloak() { local max_attempts=120 local attempt=0 info "Waiting for Keycloak to become healthy (up to 120s)..." if [ "$DRY_RUN" = "true" ]; then dryrun "poll docker inspect aiadminpanel_keycloak health status up to 120s" return 0 fi while [ "$attempt" -lt "$max_attempts" ]; do local health health=$(docker inspect --format='{{.State.Health.Status}}' aiadminpanel_keycloak 2>/dev/null || echo "unknown") if [ "$health" = "healthy" ]; then info "Keycloak is healthy" return 0 fi attempt=$((attempt + 1)) sleep 1 done warn "Keycloak did not become healthy within 120s. Check logs:" warn " docker logs aiadminpanel_keycloak" return 1 } # ── Admin User Creation ────────────────────────────────────────────────────── create_admin_user() { info "Creating admin user in Keycloak and panel database..." # Generate or use provided admin password if [ -n "${ADMIN_PASSWORD:-}" ]; then info "Using admin password from ADMIN_PASSWORD environment variable" else ADMIN_PASSWORD="$(gen_secret_alnum 20)" info "Generated random admin password" fi local admin_email="admin@${PANEL_DOMAIN}" if [ "$DRY_RUN" = "true" ]; then dryrun "kcadm.sh config credentials --server http://localhost:8180 --realm master --user admin" dryrun "kcadm.sh create users -r aiadminpanel -s username=${admin_email} -s email=${admin_email} -s enabled=true -s emailVerified=true" dryrun "kcadm.sh set-password -r aiadminpanel --username ${admin_email} --new-password ****" dryrun "INSERT INTO users (...) VALUES (...) for ${admin_email}" dryrun "Write admin password to ${SECRETS_DIR}/admin_password" return 0 fi # Authenticate kcadm.sh against Keycloak master realm local kc_admin_pw kc_admin_pw=$(cat "${SECRETS_DIR}/keycloak_admin_password" 2>/dev/null || echo "") if [ -z "$kc_admin_pw" ]; then warn "Keycloak admin password not found at ${SECRETS_DIR}/keycloak_admin_password — cannot create admin user" return 1 fi if ! docker exec aiadminpanel_keycloak /opt/keycloak/bin/kcadm.sh config credentials \ --server http://localhost:8180 --realm master --user admin --password "$kc_admin_pw" 2>/dev/null; then warn "Failed to authenticate with Keycloak admin API — cannot create admin user" return 1 fi debug "Authenticated with Keycloak admin API" # Create user in Keycloak (idempotent — ignore "already exists" errors) local kc_create_output kc_create_output=$(docker exec aiadminpanel_keycloak /opt/keycloak/bin/kcadm.sh create users \ -r aiadminpanel \ -s "username=${admin_email}" \ -s "email=${admin_email}" \ -s "enabled=true" \ -s "emailVerified=true" 2>&1) || true if echo "$kc_create_output" | grep -qi "conflict\|already exists"; then info "Keycloak user ${admin_email} already exists, updating password" else info "Created Keycloak user: ${admin_email}" fi # Set user password if docker exec aiadminpanel_keycloak /opt/keycloak/bin/kcadm.sh set-password \ -r aiadminpanel --username "${admin_email}" --new-password "${ADMIN_PASSWORD}" 2>/dev/null; then info "Admin password set in Keycloak" else warn "Failed to set admin password in Keycloak" return 1 fi # Create user in panel PostgreSQL database docker exec aiadminpanel_postgresql psql -U aiadminpanel -d aiadminpanel -c \ "INSERT INTO users (id, email, password_hash, role, totp_enabled, created_at, updated_at) \ VALUES (gen_random_uuid(), '${admin_email}', 'keycloak-managed', 'admin', false, now(), now()) \ ON CONFLICT (email) DO NOTHING;" 2>/dev/null || true info "Admin user ensured in panel database" # Save admin password to secrets echo -n "${ADMIN_PASSWORD}" > "${SECRETS_DIR}/admin_password" chmod 600 "${SECRETS_DIR}/admin_password" info "Admin password saved to ${SECRETS_DIR}/admin_password" info "Admin user created: ${admin_email}" } # ── Config Generation ───────────────────────────────────────────────────────── generate_config() { local config_file="${CONFIG_DIR}/config.yaml" if [ -f "$config_file" ] && [ "$DRY_RUN" = "false" ]; then info "Config file already exists at ${config_file}, keeping existing" return 0 fi info "Generating config file..." run bash -c "cat > '${config_file}'" << 'EOF' # AI Admin Panel Configuration # Generated by installer. Edit as needed. server: port: 8080 log_level: info database: max_conns: 25 min_conns: 5 metrics: prometheus_enabled: false collection_interval: 15s notifications: categories: service_events: true ai_ops: true system_alerts: true security_events: true updater: check_enabled: true check_interval: 24h EOF run chmod 644 "$config_file" info "Config written to ${config_file}" } # ── LiteLLM Proxy Config ───────────────────────────────────────────────────── # AI-381: docker-compose.yml bind-mounts ${PANEL_DIR}/litellm-config.yaml onto # /app/config.yaml (read-only). If that source path does not exist as a regular # file before `docker compose up`, Docker silently creates it as a *directory*, # and litellm then crash-loops forever with # IsADirectoryError: [Errno 21] Is a directory: '/app/config.yaml' # so this MUST write a regular file before deploy_stack brings litellm up. # # AI-622: the config content is RAM-gated — select_metered_model / render_litellm_config # (bundled from scripts/lib/ai.sh) pick deepseek-r1:7b on capable boxes and # llama3.2:3b on small ones. It is therefore GENERATED here, not downloaded: a # single static asset cannot be box-aware, and the gateway must agree with what # Ollama actually has pulled from first boot (no runtime config-reload path exists). setup_litellm_config() { local litellm_config="${PANEL_DIR}/litellm-config.yaml" # Decide the model now so the config and the later auto-pull use the same tag. select_metered_model if [ "$DRY_RUN" = "true" ]; then dryrun "Write ${litellm_config} (LiteLLM → Ollama ${METERED_OLLAMA_TAG}; ${METERED_TOTAL_MB}MB RAM)" return 0 fi # Self-heal a prior broken state: if Docker (or an aborted run) left a # directory here, remove it — leaving it is the exact crash-loop trigger. if [ -d "$litellm_config" ]; then warn "Removing directory at ${litellm_config} (would crash-loop litellm — AI-381)" rm -rf "$litellm_config" fi if [ -f "$litellm_config" ]; then info "LiteLLM config already exists at ${litellm_config}, keeping existing" return 0 fi if [ "$METERED_IS_FALLBACK" = "true" ]; then info "Detected ${METERED_TOTAL_MB}MB RAM (< ${AAP_METERED_MIN_MB_DEEPSEEK}MB): metered AI will run ${METERED_OLLAMA_TAG} (deepseek-r1:7b needs more RAM)." else info "Detected ${METERED_TOTAL_MB}MB RAM: metered AI will run ${METERED_OLLAMA_TAG}." fi render_litellm_config > "$litellm_config" chmod 644 "$litellm_config" info "LiteLLM config ready at ${litellm_config}" } # pull_metered_model (AI-622) — pull the RAM-gated model into the Ollama singleton # so the first metered AI call works with no manual `ollama pull` (Simple-UX / # no-terminal). select_metered_model already ran in setup_litellm_config, so # METERED_OLLAMA_TAG matches the model the gateway config routes to. # # Non-fatal by design: a failed/slow pull must never abort the install — the # panel still works and the operator can pull the model later from the AI Models # page. Idempotent: re-pulling a model that is already present is a fast no-op. pull_metered_model() { [ -n "${METERED_OLLAMA_TAG:-}" ] || select_metered_model # Escape hatch for air-gapped installs and CI (the installer matrix runs a full # unattended install and would otherwise download a multi-GB model per distro). # The RAM-gated config is already written; only the download is skipped. if [ "${AAP_SKIP_MODEL_PULL:-false}" = "true" ]; then info "AAP_SKIP_MODEL_PULL=true — skipping metered-model pull. Pull ${METERED_OLLAMA_TAG} from the AI Models page when ready." return 0 fi if [ "$DRY_RUN" = "true" ]; then dryrun "docker exec aiadminpanel_ollama ollama pull ${METERED_OLLAMA_TAG}" return 0 fi # Wait for the Ollama singleton to answer (deploy_stack started it with a 30s # healthcheck start_period). `ollama list` is the same probe compose uses. local attempt=0 while [ "$attempt" -lt 60 ]; do if docker exec aiadminpanel_ollama ollama list >/dev/null 2>&1; then break fi attempt=$((attempt + 1)) sleep 2 done if [ "$attempt" -ge 60 ]; then warn "Ollama not ready after 120s; skipping the metered-model pull. Pull ${METERED_OLLAMA_TAG} later from the AI Models page." return 0 fi info "Pulling metered model ${METERED_OLLAMA_TAG} into Ollama (one-time, may take a few minutes)..." if docker exec aiadminpanel_ollama ollama pull "${METERED_OLLAMA_TAG}"; then info "Metered model ${METERED_OLLAMA_TAG} is ready." else warn "Could not pull ${METERED_OLLAMA_TAG} now (network?). Metered AI will 404 until it is pulled from the AI Models page." fi } # ── Environment File Generation ────────────────────────────────────────────── generate_env_file() { local env_file="${PANEL_DIR}/.env" info "Generating environment file for docker-compose..." # Idempotency: don't overwrite existing .env (secrets/tokens may differ) if [ -f "$env_file" ] && [ "$DRY_RUN" = "false" ]; then info "Environment file already exists at ${env_file}, keeping existing" return 0 fi if [ "$DRY_RUN" = "true" ]; then dryrun "Write ${env_file} with PANEL_DOMAIN, ACME_EMAIL, PANEL_VERSION" dryrun "Write CERT_RESOLVER=$(tls_cert_resolver) (TLS_MODE=${TLS_MODE:-letsencrypt}) to ${env_file}" dryrun "Write Keycloak OIDC vars to ${env_file}" dryrun "Write PANEL_INTERNAL_URL=http://panel:8080 to ${env_file}" [ "${AAP_NVIDIA_RUNTIME_READY:-false}" = "true" ] && dryrun "Write panel GPU visibility (PANEL_RUNTIME=nvidia, NVIDIA_VISIBLE_DEVICES=all) to ${env_file}" # (No managed-domain dry-run narration: a --dry-run never runs md_acquire_domain, # so MD_CLAIMED is always false here — the real keys are narrated by the claim # path itself when it actually claims.) return 0 fi cat > "$env_file" <> "$env_file" fi if [ -n "${LICENSE_KEY:-}" ]; then echo "LICENSE_KEY=${LICENSE_KEY}" >> "$env_file" fi # AI-244 Phase 2: license proxy at license.aiadminpanel.com. # Customer panels never see the keygen.sh admin token — the proxy holds # it server-side. LICENSE_SERVER_URL can be overridden in advance (e.g. # for staging or a self-hosted proxy) by exporting it before install. echo "LICENSE_SERVER_URL=${LICENSE_SERVER_URL:-https://license.aiadminpanel.com}" >> "$env_file" # Managed temporary domain (AI-544 plan 2): mark the install as managed and # record which claim service issued the domain, so the panel backend (plan 3) # can renew the lease against the same service (a staging-claimed box must # renew against staging). BYO-domain installs get neither key. if [ "${MD_CLAIMED:-false}" = "true" ]; then echo "MANAGED_DOMAIN=true" >> "$env_file" echo "MANAGED_DNS_URL=${MD_BASE_URL}" >> "$env_file" fi # Public URLs (used for browser redirects and OIDC callbacks) echo "PANEL_URL=https://${PANEL_DOMAIN}" >> "$env_file" echo "FRONTEND_URL=https://${PANEL_DOMAIN}" >> "$env_file" echo "PANEL_HOSTNAME=${PANEL_DOMAIN}" >> "$env_file" echo "PANEL_SERVICE_DOMAIN=${PANEL_DOMAIN}" >> "$env_file" echo "PANEL_INTERNAL_URL=http://panel:8080" >> "$env_file" # Keycloak OIDC configuration echo "OIDC_DISCOVERY_URL=https://auth.${PANEL_DOMAIN}/realms/aiadminpanel/.well-known/openid-configuration" >> "$env_file" echo "OIDC_CLIENT_ID=panel-backend" >> "$env_file" echo "OIDC_REDIRECT_URI=https://${PANEL_DOMAIN}/api/v1/auth/oidc/callback" >> "$env_file" echo "KEYCLOAK_URL=http://keycloak:8180" >> "$env_file" echo "KEYCLOAK_REALM=aiadminpanel" >> "$env_file" echo "KEYCLOAK_ADMIN_USER=admin" >> "$env_file" echo "KEYCLOAK_HOSTNAME=auth.${PANEL_DOMAIN}" >> "$env_file" # Read secrets and write to .env (Keycloak doesn't support _FILE env vars) local db_pw kc_pw db_pw=$(cat "${SECRETS_DIR}/db_password" 2>/dev/null || echo "") kc_pw=$(cat "${SECRETS_DIR}/keycloak_admin_password" 2>/dev/null || echo "") echo "KC_DB_PASSWORD=${db_pw}" >> "$env_file" echo "KEYCLOAK_ADMIN_PASSWORD=${kc_pw}" >> "$env_file" # Per-install LiteLLM master key (lib/secrets.sh generated it — no shared # sk-litellm-master-key default reaching the litellm + panel containers). # AI-592: fail loud on empty. docker-compose.yml now requires this env var # (${LITELLM_MASTER_KEY:?...}) instead of falling back to the well-known # default, so writing nothing here would leave the AI gateway unstartable. # generate_secrets always creates the file, so an empty value here means the # secret is missing/unreadable — a real error, not a silent skip. local litellm_key litellm_key=$(cat "${SECRETS_DIR}/litellm_master_key" 2>/dev/null || echo "") if [ -z "$litellm_key" ]; then error "LITELLM_MASTER_KEY secret is missing or empty (${SECRETS_DIR}/litellm_master_key). The AI gateway will not start on a default key — re-run the installer to regenerate secrets." fi echo "LITELLM_MASTER_KEY=${litellm_key}" >> "$env_file" # 3a-2: LiteLLM DATABASE_URL — same postgres role as the panel/keycloak # (P1). db_password is alnum-only (secrets.sh gen_secret_alnum) → URL-safe. local litellm_db_password litellm_db_password=$(cat "${SECRETS_DIR}/db_password" 2>/dev/null || echo "") if [ -z "$litellm_db_password" ]; then error "db_password secret is missing or empty — cannot build LITELLM_DATABASE_URL" fi echo "LITELLM_DATABASE_URL=postgresql://aiadminpanel:${litellm_db_password}@postgresql:5432/litellm" >> "$env_file" # Install-bundled Provider AI (§6.6 / AI-447). Only written when the operator # exported a key — absent → no Provider AI seeded, panel stays sovereign by # default. The panel encrypts the key into the DB on first boot and never logs # it; we only persist it here so docker-compose can forward it to the panel # container (Rule 11: no real-key default, fail soft when unset). if [ -n "${AAP_PROVIDER_AI_API_KEY:-}" ]; then echo "AAP_PROVIDER_AI_API_KEY=${AAP_PROVIDER_AI_API_KEY}" >> "$env_file" [ -n "${AAP_PROVIDER_AI_BASE_URL:-}" ] && echo "AAP_PROVIDER_AI_BASE_URL=${AAP_PROVIDER_AI_BASE_URL}" >> "$env_file" [ -n "${AAP_PROVIDER_AI_MODEL:-}" ] && echo "AAP_PROVIDER_AI_MODEL=${AAP_PROVIDER_AI_MODEL}" >> "$env_file" [ -n "${AAP_PROVIDER_AI_PROVIDER_TYPE:-}" ] && echo "AAP_PROVIDER_AI_PROVIDER_TYPE=${AAP_PROVIDER_AI_PROVIDER_TYPE}" >> "$env_file" fi # Give the panel container its own GPU visibility when the installer configured # Docker's nvidia runtime (AI-420 opt 1). Emits nothing on CPU-only hosts, so # docker-compose.yml's runc/void defaults keep the panel container unchanged. nvidia_panel_env_block >> "$env_file" chmod 600 "$env_file" # Re-assert restrictive perms on the secrets dir + .env (self-healing). harden_perms "${SECRETS_DIR}" "$env_file" info "Environment file written to ${env_file}" } # generate_ollama_gpu_override — write (or remove) the GPU compose overlay that # gives the bundled Ollama the host GPU (AI-428). On a GPU host # (AAP_NVIDIA_RUNTIME_READY=true) ollama_gpu_override_yaml emits the reservation # and we persist it to docker-compose.gpu.yml, which deploy_stack merges with an # extra `-f`. On a CPU host it emits nothing, so we REMOVE any stale overlay — # otherwise a box that lost its GPU (driver removed, card pulled) would keep an # unconditional nvidia reservation and `docker compose up` would fail. Idempotent: # safe to re-run on every install/upgrade, so an already-installed GPU box that # re-runs the installer/updater gains the GPU (the AI-428 upgrade path). generate_ollama_gpu_override() { local gpu_file="${PANEL_DIR}/docker-compose.gpu.yml" local yaml yaml="$(ollama_gpu_override_yaml)" if [ "$DRY_RUN" = "true" ]; then if [ -n "$yaml" ]; then dryrun "Write ${gpu_file} giving the bundled Ollama the host GPU (NVIDIA runtime detected)" elif [ -f "$gpu_file" ]; then dryrun "Remove stale GPU overlay ${gpu_file} (CPU-only host)" fi return 0 fi if [ -n "$yaml" ]; then printf '%s\n' "$yaml" > "$gpu_file" chmod 644 "$gpu_file" info "Wrote GPU overlay ${gpu_file} — the bundled Ollama will use the host GPU." elif [ -f "$gpu_file" ]; then rm -f "$gpu_file" info "Removed stale GPU overlay (CPU-only host) — the bundled Ollama stays CPU." fi } # ── Docker Compose Deployment ───────────────────────────────────────────────── deploy_infra() { info "Starting infrastructure services (PostgreSQL must be healthy before Keycloak DB setup)..." local compose_file="${PANEL_DIR}/docker-compose.yml" if [ ! -f "$compose_file" ]; then # Need to download compose file first — delegate to deploy_stack's download logic deploy_stack_download_compose fi if [ "$DRY_RUN" = "true" ]; then dryrun "docker compose up -d postgresql valkey traefik" return 0 fi run docker compose -f "$compose_file" up -d postgresql valkey traefik info "Waiting for PostgreSQL to be healthy..." local attempt=0 while [ "$attempt" -lt 30 ]; do local health health=$(docker inspect --format='{{.State.Health.Status}}' aiadminpanel_postgresql 2>/dev/null || echo "unknown") if [ "$health" = "healthy" ]; then info "PostgreSQL is healthy" return 0 fi attempt=$((attempt + 1)) sleep 1 done warn "PostgreSQL did not become healthy within 30s" } deploy_stack_download_compose() { local compose_file="${PANEL_DIR}/docker-compose.yml" [ -f "$compose_file" ] && return 0 info "Downloading docker-compose.yml..." if [ "$DRY_RUN" = "true" ]; then dryrun "curl -fsSL ${GITHUB_DOWNLOAD_BASE}/docker-compose.yml -o ${compose_file}" return 0 fi if curl -fsSL "${GITHUB_DOWNLOAD_BASE}/docker-compose.yml" -o "$compose_file" 2>/dev/null; then info "Downloaded docker-compose.yml from GitHub Release" elif curl -fsSL "${GET_BASE}/docker-compose.yml" -o "$compose_file" 2>/dev/null; then info "Downloaded docker-compose.yml from get server" else error "Could not download docker-compose.yml. Check network connectivity and retry." fi } deploy_stack() { info "Deploying full panel stack..." local compose_file="${PANEL_DIR}/docker-compose.yml" # AI-428: the installer pins -f explicitly, so docker-compose.override.yml is # NOT auto-merged — the GPU overlay must be named here to take effect. # generate_ollama_gpu_override (run before us) wrote docker-compose.gpu.yml iff # this is an NVIDIA host; on a CPU host the file is absent and we deploy the # CPU-only base. This is the ONLY full `up` that (re)creates ollama — the # updater/per-session deploys use `up --no-deps panel`, so they never revert it. local -a compose_files=(-f "$compose_file") if [ -f "${PANEL_DIR}/docker-compose.gpu.yml" ]; then compose_files+=(-f "${PANEL_DIR}/docker-compose.gpu.yml") info "GPU overlay present — deploying the bundled Ollama with host-GPU access." fi # Check if existing installation (upgrade path) if [ -f "$compose_file" ] && docker compose "${compose_files[@]}" ps --status running -q 2>/dev/null | grep -q .; then info "Existing installation detected -- performing upgrade" run docker compose "${compose_files[@]}" pull run docker compose "${compose_files[@]}" up -d --remove-orphans return 0 fi # Ensure compose file exists (downloaded by deploy_infra or here) deploy_stack_download_compose run docker compose "${compose_files[@]}" up -d info "Stack deployed" } # ── Readiness Wait ──────────────────────────────────────────────────────────── wait_for_ready() { local max_attempts=60 local attempt=0 info "Waiting for panel to become ready (up to 60s)..." if [ "$DRY_RUN" = "true" ]; then dryrun "poll panel healthcheck up to ${max_attempts}s" return 0 fi while [ "$attempt" -lt "$max_attempts" ]; do # Check via docker exec (panel port is not exposed to host — only reachable via Traefik) if docker exec aiadminpanel_panel wget -qO- http://localhost:8080/healthz 2>/dev/null | grep -q '"status":"ok"'; then info "Panel is ready" return 0 fi attempt=$((attempt + 1)) sleep 1 done warn "Panel did not become ready within 60s. Check logs with: docker compose -f ${PANEL_DIR}/docker-compose.yml logs panel" return 1 } # ── Smoke Tests ─────────────────────────────────────────────────────────────── smoke_tests() { info "Running post-install smoke tests..." local all_pass=true if [ "$DRY_RUN" = "true" ]; then dryrun "curl -f http://localhost/healthz" dryrun "curl -f http://localhost/readyz" dryrun "curl -f http://localhost/" return 0 fi # Test healthz (via docker exec — panel port not exposed to host) if docker exec aiadminpanel_panel wget -qO- http://localhost:8080/healthz 2>/dev/null | grep -q '"status":"ok"'; then info "[PASS] /healthz returns 200 with status ok" else warn "[FAIL] /healthz did not return expected response" all_pass=false fi # Test frontend (via docker exec) if docker exec aiadminpanel_panel wget -qO- http://localhost:8080/ 2>/dev/null | grep -q "html"; then info "[PASS] Frontend is serving HTML" else warn "[FAIL] Frontend did not return HTML" all_pass=false fi if [ "$all_pass" = "false" ]; then warn "Some smoke tests failed. Check container logs:" warn " docker compose -f ${PANEL_DIR}/docker-compose.yml logs" return 1 fi info "All smoke tests passed" } # ── Security Baselines ──────────────────────────────────────────────────────── run_security_baselines() { local baselines_script baselines_script="$(dirname "$0")/security-baselines.sh" if [ -f "$baselines_script" ]; then info "Applying security baselines..." local dry_arg="" [ "$DRY_RUN" = "true" ] && dry_arg="--dry-run" run bash "$baselines_script" $dry_arg else warn "Security baselines script not found at ${baselines_script} -- skipping" fi } # ── Print Success ───────────────────────────────────────────────────────────── print_success() { local domain="${PANEL_DOMAIN:-localhost}" local elapsed_secs=$(( $(date +%s) - INSTALL_START )) local elapsed_min=$(( elapsed_secs / 60 )) local elapsed_sec=$(( elapsed_secs % 60 )) local ELAPSED_DISPLAY="${elapsed_min}m ${elapsed_sec}s" cat <} Health check: https://${domain}/healthz API docs: https://${domain}/api/docs Configuration: ${CONFIG_DIR}/config.yaml Credentials: ${SECRETS_DIR}/admin_password Logs: ${LOG_DIR}/install.log To check status: docker compose -f ${PANEL_DIR}/docker-compose.yml ps To view logs: docker compose -f ${PANEL_DIR}/docker-compose.yml logs -f panel Next steps: 1. Open https://${domain} in your browser 2. Log in with the admin credentials above 3. Configure notifications in Settings > Notifications Elapsed time: ${ELAPSED_DISPLAY} ${COLOR_GREEN}${COLOR_BOLD}===========================================================${COLOR_RESET} EOF if [ "${MD_CLAIMED:-false}" = "true" ]; then cat < "$dest" info "Wrote: $dest" fi chmod "$mode" "$dest" } # The updater script needs cosign (verify-or-refuse) and jq + curl (GHCR tag # query + request/result parsing). curl/flock are part of the Docker install; # cosign and jq are not, so fetch them as distro-independent static binaries. # cosign is pinned to the version the release pipeline signs with (v2.5.2). ensure_updater_deps() { local arch case "$(uname -m)" in x86_64|amd64) arch=amd64 ;; aarch64|arm64) arch=arm64 ;; *) arch=amd64 ;; esac if ! command -v cosign >/dev/null 2>&1; then info "Installing cosign (image signature verification)..." if run curl -fsSL "https://github.com/sigstore/cosign/releases/download/v2.5.2/cosign-linux-${arch}" -o /usr/local/bin/cosign; then run chmod 755 /usr/local/bin/cosign else warn "Could not download cosign — one-click updates will refuse to apply (fail-closed) until it is installed." fi fi if ! command -v jq >/dev/null 2>&1; then info "Installing jq..." if run curl -fsSL "https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-linux-${arch}" -o /usr/local/bin/jq; then run chmod 755 /usr/local/bin/jq else warn "Could not download jq — one-click updates need it to parse update requests." fi fi } install_host_updater() { info "Installing host updater (systemd path + oneshot)..." local script_dest="${PANEL_DIR}/aap-update.sh" local path_unit="/etc/systemd/system/aap-updater.path" local service_unit="/etc/systemd/system/aap-updater.service" if [ "$DRY_RUN" = "true" ]; then dryrun "ensure cosign+jq; write ${script_dest} (755) + ${path_unit} + ${service_unit}; systemctl daemon-reload; systemctl enable --now aap-updater.path" return 0 fi ensure_updater_deps if ! command -v systemctl >/dev/null 2>&1; then warn "systemd not found — skipping host updater install. One-click updates from the panel will be inert on this host." return 0 fi _write_if_changed "$script_dest" "$(_aap_update_script)" 755 _write_if_changed "$path_unit" "$(_aap_updater_path_unit)" 644 _write_if_changed "$service_unit" "$(_aap_updater_service_unit)" 644 systemctl daemon-reload if systemctl enable --now aap-updater.path >/dev/null 2>&1; then info "Host updater active — watching ${PANEL_DIR}/update/request.json" else warn "Could not enable aap-updater.path; one-click updates will be inert until it is enabled." fi } # The three functions below emit the embedded payloads. build-installer.sh # replaces each @MARKER@ line with the verbatim contents of the repo source of # truth (scripts/aap-update.sh, scripts/systemd/aap-updater.{path,service}). _aap_update_script() { cat <<'AAP_UPDATE_SCRIPT_EOF' #!/usr/bin/env bash # # aap-update.sh — AI Admin Panel host updater (AI-431 Slice 2B). # # The PANEL decides; this script EXECUTES. The unprivileged panel container # writes ${PANEL_DIR}/update/request.json (target tag derived from its own # cached status — never a client-supplied value). A host systemd path-unit # notices the file and runs THIS script as root. It then: # # validate target → pg_dump backup (retain 3) → cosign-verify the target # image → set PANEL_VERSION in .env (in place) → compose pull + recreate → # poll /healthz → success, or auto-rollback to the prior version. # # It writes the terminal state to ${PANEL_DIR}/update/result.json, which the # panel polls via GET /system/update/result to drive the UI. # # SECURITY (Product Principle #2 — secure & sovereign): # * verify-or-refuse: the target image MUST carry a valid cosign signature # produced by OUR release pipeline (keyless, GitHub Actions OIDC). An # unsigned or wrongly-signed image is refused and nothing is swapped. This # is fail-closed and has no bypass. # * No docker.sock is mounted into any container — this host unit is the only # privileged actor. # # Hermetic test: scripts/tests/aap-update.bats stubs docker/cosign/crane/curl. set -uo pipefail # ── Configuration (overridable via environment) ────────────────────────────── PANEL_DIR="${PANEL_DIR:-/opt/aiadminpanel}" UPDATE_DIR="${PANEL_DIR}/update" BACKUP_DIR="${PANEL_DIR}/backups" ENV_FILE="${PANEL_DIR}/.env" COMPOSE_FILE="${COMPOSE_FILE:-${PANEL_DIR}/docker-compose.yml}" REQUEST_FILE="${UPDATE_DIR}/request.json" RESULT_FILE="${UPDATE_DIR}/result.json" LOCK_FILE="${UPDATE_DIR}/.lock" REGISTRY_HOST="${REGISTRY_HOST:-ghcr.io}" IMAGE_PATH="${IMAGE_PATH:-aiadminpanel/ai-admin-panel}" REGISTRY="${PANEL_IMAGE:-${REGISTRY_HOST}/${IMAGE_PATH}}" PANEL_CONTAINER="${PANEL_CONTAINER:-aiadminpanel_panel}" PG_CONTAINER="${PG_CONTAINER:-aiadminpanel_postgresql}" PG_USER="${PG_USER:-aiadminpanel}" PG_DB="${PG_DB:-aiadminpanel}" # Keyless cosign identity of OUR release pipeline. Confirmed against the signed # v2.6.0 image: subject .../release.yml@refs/tags/vX.Y.Z, issuer below. The # regexp pins the workflow path (any ref) and nothing else may satisfy it. COSIGN_CERT_IDENTITY_REGEXP="${COSIGN_CERT_IDENTITY_REGEXP:-https://github.com/aiadminpanel/ai-admin-panel/\.github/workflows/release\.yml@.*}" COSIGN_CERT_OIDC_ISSUER="${COSIGN_CERT_OIDC_ISSUER:-https://token.actions.githubusercontent.com}" HEALTH_CHECK_TIMEOUT="${HEALTH_CHECK_TIMEOUT:-120}" HEALTH_CHECK_INTERVAL="${HEALTH_CHECK_INTERVAL:-3}" BACKUP_RETAIN="${BACKUP_RETAIN:-3}" # ── State (initialized so `set -u` is satisfied on every code path) ─────────── TARGET_TAG="" CHANNEL="stable" CURRENT="" PREV_VERSION="" BACKUP_PATH="" STARTED_AT="" FAIL_REASON="" HOLD_LOCK=0 RESULT_WRITTEN=0 log() { echo "[$(date -u '+%Y-%m-%dT%H:%M:%SZ')] [aap-update] $*"; } now() { date -u '+%Y-%m-%dT%H:%M:%SZ'; } # ── Terminal result + cleanup ───────────────────────────────────────────────── # write_result STATUS [REASON] — overwrite result.json with the terminal state. # Compact JSON (no spaces) so it is trivially greppable and small. write_result() { local status="$1" reason="${2:-}" new_version rolled finished tmp finished="$(now)" case "$status" in succeeded) new_version="$TARGET_TAG"; rolled="false" ;; rolled-back) new_version="$PREV_VERSION"; rolled="true" ;; *) new_version="$PREV_VERSION"; rolled="false" ;; esac mkdir -p "$UPDATE_DIR" tmp="${RESULT_FILE}.tmp" printf '{"status":"%s","targetTag":"%s","fromVersion":"%s","newVersion":"%s","rolledBack":%s,"reason":"%s","backupPath":"%s","startedAt":"%s","finishedAt":"%s"}\n' \ "$status" "$TARGET_TAG" "$PREV_VERSION" "$new_version" "$rolled" "$reason" "$BACKUP_PATH" "$STARTED_AT" "$finished" \ > "$tmp" mv -f "$tmp" "$RESULT_FILE" RESULT_WRITTEN=1 log "result: $status${reason:+ — $reason}" } finish_failed() { write_result "failed" "$1"; } finish_rollback() { write_result "rolled-back" "$1"; } finish_ok() { write_result "succeeded" ""; } # Always leave a terminal result and consume the request — but only if we are # the instance holding the lock (never disturb another run's files). cleanup() { local rc=$? if [ "$HOLD_LOCK" = "1" ]; then if [ "$RESULT_WRITTEN" != "1" ]; then write_result "failed" "updater exited unexpectedly (rc=${rc})" fi rm -f "$REQUEST_FILE" fi } trap cleanup EXIT # ── Steps ───────────────────────────────────────────────────────────────────── acquire_lock() { mkdir -p "$UPDATE_DIR" exec 9>"$LOCK_FILE" if ! flock -n 9; then log "another update is already running; leaving its state untouched" exit 0 fi HOLD_LOCK=1 } read_request() { [ -f "$REQUEST_FILE" ] || { FAIL_REASON="no update request found"; return 1; } TARGET_TAG="$(jq -r '.targetTag // ""' "$REQUEST_FILE" 2>/dev/null)" || { FAIL_REASON="malformed request.json"; return 1; } CHANNEL="$(jq -r '.channel // "stable"' "$REQUEST_FILE" 2>/dev/null)" CURRENT="$(jq -r '.currentVersion // ""' "$REQUEST_FILE" 2>/dev/null)" if [ -z "$TARGET_TAG" ] || [ "$TARGET_TAG" = "null" ]; then FAIL_REASON="request.json has no target tag" return 1 fi return 0 } read_env_version() { grep -E '^PANEL_VERSION=' "$ENV_FILE" 2>/dev/null | head -1 | cut -d= -f2- | tr -d '"' | tr -d "'" } # set_env_version V — edit PANEL_VERSION in place (NEVER truncate the file); # append the line only if it is genuinely absent. set_env_version() { local v="$1" if [ -f "$ENV_FILE" ] && grep -qE '^PANEL_VERSION=' "$ENV_FILE"; then sed -i "s/^PANEL_VERSION=.*/PANEL_VERSION=${v}/" "$ENV_FILE" else printf 'PANEL_VERSION=%s\n' "$v" >> "$ENV_FILE" fi } # List the image's tags from GHCR via the anonymous pull-token flow — the same # approach the Go detector uses (internal/updater/registry.go), so the host # needs only curl + jq (no crane binary). n=1000 returns every tag in one page # for a repo this size. list_remote_tags() { local token token="$(curl -fsS --max-time 15 \ "https://${REGISTRY_HOST}/token?scope=repository:${IMAGE_PATH}:pull&service=${REGISTRY_HOST}" \ 2>/dev/null | jq -r '.token // .access_token // empty')" [ -n "$token" ] || return 1 curl -fsS --max-time 15 -H "Authorization: Bearer ${token}" \ "https://${REGISTRY_HOST}/v2/${IMAGE_PATH}/tags/list?n=1000" \ 2>/dev/null | jq -r '.tags[]? // empty' } # Defense in depth: the host re-validates the panel-chosen tag against the # registry and refuses a pre-release on the stable channel. validate_target() { if [ "$CHANNEL" = "stable" ] && [[ "$TARGET_TAG" == *-* ]]; then FAIL_REASON="pre-release target '${TARGET_TAG}' rejected on the stable channel" return 1 fi if ! list_remote_tags | grep -Fxq "$TARGET_TAG"; then FAIL_REASON="target tag '${TARGET_TAG}' not found on the registry" return 1 fi return 0 } # pg_dump BEFORE any image change. Abort the whole apply if the dump fails, so a # bad migration on the new image can always be restored. Keep the newest N. backup_db() { local ts file ts="$(date -u '+%Y%m%d-%H%M%S')" file="${BACKUP_DIR}/backup-pre-update-${ts}.sql.gz" mkdir -p "$BACKUP_DIR" if ! docker exec "$PG_CONTAINER" pg_dump -U "$PG_USER" -d "$PG_DB" | gzip > "$file"; then rm -f "$file" return 1 fi if [ ! -s "$file" ]; then rm -f "$file" return 1 fi BACKUP_PATH="$file" prune_backups return 0 } prune_backups() { shopt -s nullglob local files=( "${BACKUP_DIR}"/backup-pre-update-*.sql.gz ) shopt -u nullglob [ "${#files[@]}" -le "$BACKUP_RETAIN" ] && return 0 local sorted=() # Timestamped names sort chronologically; newest first, drop everything past N. mapfile -t sorted < <(printf '%s\n' "${files[@]}" | sort -r) rm -f "${sorted[@]:$BACKUP_RETAIN}" } # verify-or-refuse — fail-closed. No bypass. verify_image() { cosign verify "${REGISTRY}:${TARGET_TAG}" \ --certificate-identity-regexp "$COSIGN_CERT_IDENTITY_REGEXP" \ --certificate-oidc-issuer "$COSIGN_CERT_OIDC_ISSUER" \ >/dev/null 2>&1 } apply_target() { set_env_version "$TARGET_TAG" docker compose --project-directory "$PANEL_DIR" -f "$COMPOSE_FILE" pull panel >/dev/null 2>&1 || return 1 docker compose --project-directory "$PANEL_DIR" -f "$COMPOSE_FILE" up -d --no-deps --force-recreate panel >/dev/null 2>&1 || return 1 return 0 } # Health is read from Docker's own healthcheck status. The panel image defines # `wget localhost:8080/healthz` as its healthcheck, so this is authoritative and # avoids guessing the container's IP — which is ambiguous, since the panel is # attached to several docker networks. Falls back to an in-container probe only # if the image somehow declares no healthcheck. wait_health() { local elapsed=0 status while [ "$elapsed" -lt "$HEALTH_CHECK_TIMEOUT" ]; do status="$(docker inspect -f '{{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$PANEL_CONTAINER" 2>/dev/null)" case "$status" in healthy) return 0 ;; none) if docker exec "$PANEL_CONTAINER" wget -qO- http://localhost:8080/healthz >/dev/null 2>&1; then return 0 fi ;; esac sleep "$HEALTH_CHECK_INTERVAL" elapsed=$((elapsed + HEALTH_CHECK_INTERVAL)) done return 1 } rollback() { local reason="$1" log "rolling back to ${PREV_VERSION}: ${reason}" set_env_version "$PREV_VERSION" docker compose --project-directory "$PANEL_DIR" -f "$COMPOSE_FILE" up -d --no-deps --force-recreate panel >/dev/null 2>&1 || true finish_rollback "$reason" } # ── Orchestration ───────────────────────────────────────────────────────────── main() { acquire_lock STARTED_AT="$(now)" if ! read_request; then finish_failed "$FAIL_REASON"; return 1; fi PREV_VERSION="$(read_env_version)" [ -n "$PREV_VERSION" ] || PREV_VERSION="$CURRENT" log "request: ${PREV_VERSION:-unknown} → ${TARGET_TAG} (channel ${CHANNEL})" if ! validate_target; then finish_failed "$FAIL_REASON"; return 1; fi if ! backup_db; then finish_failed "database backup failed — refusing to update"; return 1; fi if ! verify_image; then finish_failed "image signature verification failed for ${TARGET_TAG}"; return 1; fi if ! apply_target; then rollback "pulling or recreating the new image failed" return 1 fi if ! wait_health; then rollback "the new version did not become healthy in time" return 1 fi log "update to ${TARGET_TAG} succeeded" finish_ok return 0 } main "$@" AAP_UPDATE_SCRIPT_EOF } _aap_updater_path_unit() { cat <<'AAP_UPDATER_PATH_EOF' [Unit] Description=AI Admin Panel — watch for one-click update requests Documentation=https://docs.aiadminpanel.com/operations/updates [Path] # The panel container writes this file (target tag it chose itself) when an # operator clicks "Update now". Its appearance fires aap-updater.service. The # service removes the file when done, which re-arms this watcher for next time. PathExists=/opt/aiadminpanel/update/request.json Unit=aap-updater.service [Install] WantedBy=multi-user.target AAP_UPDATER_PATH_EOF } _aap_updater_service_unit() { cat <<'AAP_UPDATER_SERVICE_EOF' [Unit] Description=AI Admin Panel — apply a one-click update (oneshot) Documentation=https://docs.aiadminpanel.com/operations/updates After=docker.service Requires=docker.service [Service] Type=oneshot Environment=PANEL_DIR=/opt/aiadminpanel # Runs as root (the only privileged actor in the system — no container holds # docker.sock). The script is self-locking, takes a DB backup, cosign-verifies # the target fail-closed, swaps the panel, and auto-rolls-back on failure. ExecStart=/opt/aiadminpanel/aap-update.sh # A failed/rolled-back apply still wrote result.json for the UI; surfacing the # non-zero exit in `systemctl status` is intentional and harmless (the .path # unit re-arms on the next request, not on this unit's state). AAP_UPDATER_SERVICE_EOF } # ── Main ────────────────────────────────────────────────────────────────────── main() { if [ "$DRY_RUN" = "true" ]; then info "=== DRY RUN MODE: No changes will be made ===" fi # Ensure log directory exists early (best-effort) mkdir -p "$LOG_DIR" 2>/dev/null || true echo "" echo "${COLOR_BOLD}AI Admin Panel Installer${COLOR_RESET}" echo "Version: ${PANEL_VERSION}" echo "" info "AI Admin Panel Installer v${PANEL_VERSION}" info "Install log: ${INSTALL_LOG}" step "Detecting platform..." detect_and_gate_platform step "Prompting for configuration..." prompt_domain prompt_license step "Running pre-flight checks..." if [ "$DRY_RUN" = "true" ]; then preflight_checks || warn "Pre-flight reported issues (non-fatal in dry-run)." else preflight_checks || exit $? fi step "Creating directories..." create_directories step "Installing Docker..." install_docker configure_docker_daemon step "Configuring GPU support (NVIDIA Container Toolkit, if a GPU is present)..." install_nvidia_container_toolkit step "Generating secrets..." setup_secrets step "Generating configuration..." generate_config step "Writing LiteLLM proxy config..." setup_litellm_config step "Generating environment file..." generate_env_file # AI-428: on a GPU host, write the compose overlay that gives the bundled # Ollama the host GPU (deploy_stack merges it). No-op / stale-cleanup on CPU. generate_ollama_gpu_override step "Starting infrastructure (PostgreSQL, Valkey, Traefik)..." deploy_infra step "Setting up Keycloak..." setup_keycloak step "Setting up LiteLLM database..." setup_litellm_db step "Deploying full stack..." deploy_stack step "Installing host updater (one-click updates)..." install_host_updater step "Opening host firewall for ports 80/443..." open_firewall_ports step "Waiting for Keycloak readiness..." wait_for_keycloak step "Creating admin user..." create_admin_user step "Creating River queue tables..." create_river_tables step "Waiting for panel readiness..." wait_for_ready run_security_baselines smoke_tests step "Pulling the bundled metered-AI model into Ollama (one-time)..." pull_metered_model if [ "$DRY_RUN" = "false" ]; then print_success else info "=== DRY RUN complete: no changes were made ===" fi } main "$@"