llama-cpp: host-side backend choice, release channels, fixes from testing

- Ask for the build on the Proxmox host with a whiptail menu (auto, cpu,
  vulkan, cuda, plus rocm/sycl/openvino on amd64) instead of inside the
  container, where the installer never has a terminal. Cancel exits; no
  terminal or a preset var_backend skips the menu
- Match the Level Zero loader as libze1_*u24.04_amd64.deb: GitHub encodes the
  + in the asset URL as %2B, so the old pattern fell back to libze-dev
- SYCL: write the persistent kernel cache commented out, it stopped models
  from loading on an Intel GPU
- Add LLAMA_ARG_MODELS_DIR for router-mode model switching once
  LLAMA_ARG_HF_REPO is commented out
- Release channels: bNNNN builds are now prereleases and stable vX.Y.Z
  releases carry only nightly-tag.txt. By default install the build the
  latest stable release names; GH_INCLUDE_PRERELEASE=1 follows the bNNNN
  builds. The channel is recorded in /opt/llama-cpp_data/.prerelease and
  updates keep it unless overridden; older installs count as nightly
- Default model ggml-org/gemma-4-E2B-it-GGUF:Q4_0 (the previous default did
  not load on OpenVINO), still overridable with var_model_repo

Co-Authored-By: Claude Opus 5.5 (1M context) <noreply@anthropic.com>

Author: cosmincc

ct/llama-cpp.sh

@@ -19,6 +19,28 @@ var_gpu="${var_gpu:-yes}"
 var_unprivileged="${var_unprivileged:-1}"
 var_testurl="${var_testurl:-https://github.com/community-scripts/DevScripts/issues/2124}"
 
+# Asked here because the installer runs under lxc-attach, where it can never
+# reach a terminal. Updates run inside the container and keep the recorded build.
+if [[ -z "${var_backend:-}" ]] && command -v pveversion >/dev/null 2>&1 && [[ -t 0 ]]; then
+  LLAMA_BACKENDS=(
+    "auto" "Vulkan if a GPU is passed through, else CPU"
+    "cpu" "CPU only"
+    "vulkan" "Vulkan (Intel, AMD and NVIDIA GPUs)"
+    "cuda" "NVIDIA CUDA (12 or 13, from the host driver)"
+  )
+  if [[ "$(dpkg --print-architecture 2>/dev/null)" == "amd64" ]]; then
+    LLAMA_BACKENDS+=(
+      "rocm" "AMD ROCm"
+      "sycl-fp16" "Intel GPU via oneAPI SYCL (FP16)"
+      "sycl-fp32" "Intel GPU via oneAPI SYCL (FP32)"
+      "openvino" "Intel CPU/GPU via OpenVINO"
+    )
+  fi
+  var_backend=$(whiptail --backtitle "Proxmox VE Helper Scripts" --title "LLAMA.CPP BUILD" --default-item "auto" \
+    --menu "Choose the llama.cpp build:" $((${#LLAMA_BACKENDS[@]} / 2 + 8)) 72 $((${#LLAMA_BACKENDS[@]} / 2)) \
+    "${LLAMA_BACKENDS[@]}" 3>&1 1>&2 2>&3) || exit_script
+fi
+
 export var_backend="${var_backend:-}"
 export var_model_repo="${var_model_repo:-}"
 export var_ctx_size="${var_ctx_size:-}"
@@ -41,8 +63,24 @@ function update_script() {
     exit
   fi
 
-  # Only the bNNNN tags carry the prebuilt binaries; other tags have no assets.
-  if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp" "" "" "b"; then
+  # Follow the channel picked at install unless GH_INCLUDE_PRERELEASE is given
+  # for this run: the bNNNN prerelease builds, or the build a stable vX.Y.Z
+  # release names in its nightly-tag.txt (stable releases carry no binaries).
+  # Installs that predate the record all tracked the bNNNN builds, and reading
+  # them as stable would downgrade them.
+  GH_INCLUDE_PRERELEASE="${GH_INCLUDE_PRERELEASE:-$(cat /opt/llama-cpp_data/.prerelease 2>/dev/null || echo 1)}"
+  export GH_INCLUDE_PRERELEASE
+  LLAMA_RELEASE=""
+  if [[ "$GH_INCLUDE_PRERELEASE" == "1" ]]; then
+    if check_for_gh_release "llama-cpp" "ggml-org/llama.cpp" "" "" "b"; then
+      LLAMA_RELEASE="$CHECK_UPDATE_RELEASE"
+    fi
+  elif check_for_gh_release "llama-cpp-stable" "ggml-org/llama.cpp"; then
+    USE_ORIGINAL_FILENAME=true fetch_and_deploy_gh_release "llama-cpp-stable" "ggml-org/llama.cpp" "singlefile" "$CHECK_UPDATE_RELEASE" "/opt/llama-cpp_data" "nightly-tag.txt"
+    LLAMA_RELEASE="$(</opt/llama-cpp_data/nightly-tag.txt)"
+  fi
+
+  if [[ -n "$LLAMA_RELEASE" ]]; then
     msg_info "Stopping Service"
     systemctl stop llama-cpp
     msg_ok "Stopped Service"
@@ -68,17 +106,17 @@ function update_script() {
     *) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;;
     esac
 
-    CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "${CHECK_UPDATE_RELEASE}" "/opt/llama-cpp" "$LLAMA_ASSET" "b"
+    CLEAN_INSTALL=1 fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_ASSET"
     # The wipe above also removed the CUDA runtime, so it always goes back in.
     if [[ -n "$LLAMA_CUDART_ASSET" ]]; then
       rm -f ~/.llama-cpp-cudart
-      fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "${CHECK_UPDATE_RELEASE}" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET" "b"
+      fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET"
     fi
     chmod +x /opt/llama-cpp/llama-*
 
     case "$LLAMA_BACKEND" in
     sycl-* | openvino)
-      fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*+u24.04_amd64.deb" || true
+      fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*u24.04_amd64.deb" || true
       if [[ -d /dev/dri ]]; then
         fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true
       fi
@@ -96,6 +134,8 @@ function update_script() {
       ;;
     esac
 
+    echo "$GH_INCLUDE_PRERELEASE" >/opt/llama-cpp_data/.prerelease
+
     msg_info "Starting Service"
     systemctl start llama-cpp
     msg_ok "Started Service"

install/llama-cpp-install.sh

@@ -13,7 +13,7 @@ setting_up_container
 network_check
 update_os
 
-var_model_repo="${var_model_repo:-ggml-org/gemma-3-1b-it-GGUF}"
+var_model_repo="${var_model_repo:-ggml-org/gemma-4-E2B-it-GGUF:Q4_0}"
 var_port="${var_port:-8080}"
 var_ctx_size="${var_ctx_size:-4096}"
 var_offload_layers="${var_offload_layers:-}"
@@ -26,18 +26,6 @@ var_api_key="${var_api_key:-}"
 #   rocm      ubuntu-rocm-<ver>-x64           needs the matching ROCm (TheRock) runtime
 #   sycl      ubuntu-sycl-{fp16,fp32}-x64     Intel GPUs, needs oneAPI + Level Zero
 #   openvino  ubuntu-openvino-<ver>-x64       Intel CPU/GPU, OpenVINO runtime is bundled
-if [[ -z "${var_backend:-}" ]]; then
-  LLAMA_CHOICE=$(prompt_select "Select the llama.cpp build:" 1 60 \
-    "auto      - Vulkan when a GPU is passed through, otherwise CPU" \
-    "cpu       - CPU only" \
-    "vulkan    - Vulkan (Intel, AMD and NVIDIA GPUs)" \
-    "cuda      - NVIDIA CUDA" \
-    "rocm      - AMD ROCm" \
-    "sycl-fp16 - Intel GPU via oneAPI SYCL (FP16)" \
-    "sycl-fp32 - Intel GPU via oneAPI SYCL (FP32)" \
-    "openvino  - Intel CPU/GPU via OpenVINO")
-  var_backend="${LLAMA_CHOICE%% *}"
-fi
 var_backend="${var_backend:-auto}"
 
 if [[ "$var_backend" == "auto" ]]; then
@@ -79,7 +67,7 @@ setup_hwaccel
 # SYCL and OpenVINO reach an Intel GPU through Level Zero. release.yml builds
 # SYCL against the loader from oneapi-src/level-zero, so use the same package.
 if [[ "$var_backend" == sycl-* || "$var_backend" == "openvino" ]]; then
-  fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*+u24.04_amd64.deb"
+  fetch_and_deploy_gh_release "level-zero" "oneapi-src/level-zero" "binary" "latest" "" "libze1_*u24.04_amd64.deb"
   if [[ -d /dev/dri ]]; then
     fetch_and_deploy_gh_release "intel-level-zero-gpu" "intel/compute-runtime" "binary" "latest" "" "libze-intel-gpu1_*_amd64.deb" || true
   fi
@@ -128,14 +116,21 @@ openvino) LLAMA_ASSET="llama-*-bin-ubuntu-openvino-*x64.tar.gz" ;;
 *) LLAMA_ASSET="llama-*-bin-ubuntu-${LLAMA_ARCH}.tar.gz" ;;
 esac
 
-# Only the bNNNN tags carry the prebuilt binaries; other tags have no assets.
-fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET" "b"
-LLAMA_RELEASE="$(cat ~/.llama-cpp)"
+# Only the bNNNN builds carry binaries, and upstream marks them prerelease. A
+# stable vX.Y.Z release ships just nightly-tag.txt, naming the build it was cut
+# from, so without GH_INCLUDE_PRERELEASE=1 that build is the one deployed.
+mkdir -p /opt/llama-cpp_data/models
+if [[ "${GH_INCLUDE_PRERELEASE:-0}" == "1" ]]; then
+  fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "latest" "/opt/llama-cpp" "$LLAMA_ASSET" "b"
+  LLAMA_RELEASE="$(cat ~/.llama-cpp)"
+else
+  USE_ORIGINAL_FILENAME=true fetch_and_deploy_gh_release "llama-cpp-stable" "ggml-org/llama.cpp" "singlefile" "latest" "/opt/llama-cpp_data" "nightly-tag.txt"
+  LLAMA_RELEASE="$(</opt/llama-cpp_data/nightly-tag.txt)"
+  fetch_and_deploy_gh_release "llama-cpp" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_ASSET"
+fi
 if [[ -n "$LLAMA_CUDART_ASSET" ]]; then
-  fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET" "b"
+  fetch_and_deploy_gh_release "llama-cpp-cudart" "ggml-org/llama.cpp" "prebuild" "$LLAMA_RELEASE" "/opt/llama-cpp" "$LLAMA_CUDART_ASSET"
 fi
-
-mkdir -p /opt/llama-cpp_data/models
 LLAMA_LIBPATH="/opt/llama-cpp"
 
 # The ROCm build carries no ROCm libraries. release.yml builds it against AMD's
@@ -158,6 +153,7 @@ fi
 msg_info "Configuring llama.cpp"
 chmod +x /opt/llama-cpp/llama-*
 echo "$var_backend" >/opt/llama-cpp_data/.backend
+echo "${GH_INCLUDE_PRERELEASE:-0}" >/opt/llama-cpp_data/.prerelease
 if [[ -z "$var_offload_layers" ]]; then
   [[ "$var_backend" == "cpu" ]] && var_offload_layers=0 || var_offload_layers=999
 fi
@@ -169,6 +165,9 @@ LLAMA_ARG_PORT=${var_port}
 #LLAMA_ARG_MODEL=/opt/llama-cpp_data/models/your-model.gguf
 # ...or let llama-server pull one from Hugging Face on first start:
 LLAMA_ARG_HF_REPO=${var_model_repo}
+# Model switching: comment out LLAMA_ARG_HF_REPO (and LLAMA_ARG_MODEL) to serve
+# every model in this folder and pick one per request from the web UI or API:
+LLAMA_ARG_MODELS_DIR=/opt/llama-cpp_data/models
 
 LLAMA_CACHE=/opt/llama-cpp_data/models
 LLAMA_ARG_CTX_SIZE=${var_ctx_size}
@@ -178,9 +177,10 @@ case "$var_backend" in
 sycl-*)
   cat <<EOF >>/opt/llama-cpp.env
 
-# SYCL: cache JIT-compiled kernels across restarts; pick GPUs with the selector
-SYCL_CACHE_PERSISTENT=1
-SYCL_CACHE_DIR=/opt/llama-cpp_data/sycl-cache
+# SYCL: the persistent kernel cache speeds up restarts but can stop models
+# from loading on some Intel GPUs, so it is off by default
+#SYCL_CACHE_PERSISTENT=1
+#SYCL_CACHE_DIR=/opt/llama-cpp_data/sycl-cache
 #ONEAPI_DEVICE_SELECTOR=level_zero:0
 EOF
   ;;

json/llama-cpp.json

@@ -57,8 +57,8 @@
       "name": "var_model_repo",
       "label": "Model Repository",
       "type": "text",
-      "default": "ggml-org/gemma-3-1b-it-GGUF",
-      "help": "GGUF repo on HuggingFace, downloaded on first service start"
+      "default": "ggml-org/gemma-4-E2B-it-GGUF:Q4_0",
+      "help": "GGUF repo on HuggingFace, optionally with :QUANT, downloaded on first service start"
     },
     {
       "name": "var_ctx_size",
@@ -97,7 +97,7 @@
   ],
   "notes": [
     {
-      "text": "On first start llama-server downloads the model set in LLAMA_ARG_HF_REPO (default: ggml-org/gemma-3-1b-it-GGUF, roughly 800 MB) into /opt/llama-cpp_data/models, so the container is usable straight away. Swap it for any GGUF repo, or point LLAMA_ARG_MODEL at a local file, then restart.",
+      "text": "On first start llama-server downloads the model set in LLAMA_ARG_HF_REPO (default: ggml-org/gemma-4-E2B-it-GGUF:Q4_0, roughly 3.5 GB with its vision projector) into /opt/llama-cpp_data/models, so the container is usable straight away. Choose another at install time with var_model_repo, or later swap it for any GGUF repo (repo or repo:QUANT), or point LLAMA_ARG_MODEL at a local file, then restart.",
       "type": "info"
     },
     {
@@ -111,6 +111,10 @@
     {
       "text": "RAM is the limiting factor: the model must fit alongside the KV cache. 8 GB comfortably runs models up to roughly 7B at Q4. Raise LLAMA_ARG_CTX_SIZE only if you have headroom.",
       "type": "info"
+    },
+    {
+      "text": "By default the build named by the latest stable llama.cpp release (vX.Y.Z) is installed. Set GH_INCLUDE_PRERELEASE=1 to follow the nightly bNNNN builds instead. The choice is remembered in /opt/llama-cpp_data/.prerelease and used by updates; GH_INCLUDE_PRERELEASE=0 or 1 on an update run switches it.",
+      "type": "info"
     }
   ]
 }