Retry every engine self-download, not just the two in build.func

The previous commit covered build.func and lxc/install.func. The engine
fetches itself from a dozen more places -- api/api.func pulling its four
parts, lib/tools.func pulling its five, the incus and VM entry points, the
UI loader -- and each of those was still a single attempt. Telemetry counts
roughly 240 runs in fourteen days that failed because the engine could not
download its own parts; none of them are the fault of the script being
installed.

The retry options are repeated per file rather than shared. There is nowhere
to put a shared copy that does not itself need downloading first, which is
the problem being solved.

Two things the flags alone got wrong:

curl -S prints a line per failed attempt, so a download that recovered on
the second try now showed the user two errors and then quietly worked, mid
spinner. _cs_curl_retry holds stderr back and releases it only if every
attempt failed, so a recovery is silent and a real failure still explains
itself.

--tries and --waitretry are GNU wget options. The wget fallback exists for
the minimal systems that ship BusyBox wget, where an unrecognised option is
a hard failure rather than a slower download, so those stay single-attempt.

Also, found while auditing the fetch sites:

- The debug-MOTD branch fetched ${CORE_URL}/install.func, a path that does
  not exist -- the file is lxc/install.func. motd_ssh was never defined, so
  DEV_MODE_MOTD has silently done nothing.
- alpine-install.func sourced lib/alpine.func through a bare
  source <(curl ...), which hides the exit code: a failed download sourced
  nothing and left every helper below missing with no explanation. It now
  goes through _bootstrap_source like the rest of that file's bootstrap.
- get_header had no timeout at all. It stays without retry, since it only
  decides whether an ASCII banner appears, but an unbounded connect meant a
  black-holed network waited out the OS TCP timeout per candidate.

Verified against a local server that 503s twice then succeeds: three
requests, body returned, nothing on stderr. A 404 still fails on the first
request in under a second with curl's message intact.

Author: MickLesk

api/api.func

@@ -84,6 +84,12 @@ TELEMETRY_ERROR_MAX_BYTES=10240
 #   telemetry.func  payload, transport and the senders
 # ==============================================================================
 
+# Retry options for engine fetches. Repeated in each file that bootstraps
+# itself over the network rather than shared, because there is nowhere to put
+# a shared copy that does not itself need downloading first. core/build.func
+# carries the full reasoning.
+_CS_CURL_RETRY=(--retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10)
+
 _api_source_part() {
   local part="$1"
 
@@ -96,7 +102,7 @@ _api_source_part() {
   local base="${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}"
   if command -v curl >/dev/null 2>&1; then
     # shellcheck disable=SC1090
-    source <(curl -fsSL "${base}/api/${part}.func")
+    source <(curl -fsSL "${_CS_CURL_RETRY[@]}" "${base}/api/${part}.func")
   else
     # shellcheck disable=SC1090
     source <(wget -qO- "${base}/api/${part}.func")

core/build.func

@@ -168,8 +168,28 @@ _cs_net_log() {
 #
 # Three attempts with a one second base delay; curl doubles it, so the worst case
 # adds about three seconds to a download that was going to fail anyway.
+#
+# The wget fallbacks deliberately stay a single attempt: --tries/--waitretry are
+# GNU options, and the fallback exists for exactly the minimal systems that ship
+# BusyBox wget, where an unrecognised option is a hard failure rather than a
+# slower download.
 _CS_CURL_RETRY=(--retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10)
 
+# curl -S prints one line per failed attempt, so a download that recovers on the
+# second try would show the user two errors and then quietly work — during a
+# spinner, no less. Hold stderr back and release it only if every attempt failed.
+_cs_curl_retry() {
+  local url="${1:?url}" err rc=0
+  err="$(mktemp 2>/dev/null)" || {
+    curl -fsSL "${_CS_CURL_RETRY[@]}" "$url"
+    return
+  }
+  curl -fsSL "${_CS_CURL_RETRY[@]}" "$url" 2>"$err" || rc=$?
+  ((rc != 0)) && cat "$err" >&2
+  rm -f "$err"
+  return "$rc"
+}
+
 _cs_download() {
   local url="${1:?url}"
   local t0 code
@@ -186,9 +206,9 @@ _cs_download() {
     return 0
   fi
   if command -v curl >/dev/null 2>&1; then
-    curl -fsSL "${_CS_CURL_RETRY[@]}" "$url"
+    _cs_curl_retry "$url"
   else
-    wget -q --tries=3 --waitretry=1 -O- "$url"
+    wget -qO- "$url"
   fi
 }
 

core/core.func

@@ -1073,7 +1073,11 @@ get_header() {
     fi
 
     mkdir -p "$(dirname "$cached")"
-    if curl -fsSL "${core_url}/headers/${rel}" -o "$cached" && [[ -s "$cached" ]]; then
+    # No retry here, unlike every other engine fetch: this one only decides
+    # whether an ASCII banner appears, and the loop has several candidates to
+    # get through. It does need a bound, though — without one a black-holed
+    # network waits out the OS TCP timeout per candidate before drawing nothing.
+    if curl -fsSL --connect-timeout 5 --max-time 15 "${core_url}/headers/${rel}" -o "$cached" && [[ -s "$cached" ]]; then
       cat "$cached" 2>/dev/null || true
       return 0
     fi

incus/build.func

@@ -22,6 +22,12 @@ elif [[ -S /dev/incus/sock ]] || [[ -f /etc/profile.d/incus-motd.sh ]] || [[ -f
   _INCUS_INSIDE_CONTAINER=1
 fi
 
+# Retry options for engine fetches. Repeated in each file that bootstraps
+# itself over the network rather than shared, because there is nowhere to put
+# a shared copy that does not itself need downloading first. core/build.func
+# carries the full reasoning.
+_CS_CURL_RETRY=(--retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10)
+
 # Prefer the dispatcher's resolver; the fallback must use the ENGINE base,
 # since every path passed in here is a core file, not a script.
 _incus_source() {
@@ -33,7 +39,7 @@ _incus_source() {
   local base="${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}"
   if command -v curl >/dev/null 2>&1; then
     # shellcheck disable=SC1090
-    source <(curl -fsSL "$base/$rel")
+    source <(curl -fsSL "${_CS_CURL_RETRY[@]}" "$base/$rel")
   else
     # shellcheck disable=SC1090
     source <(wget -qO- "$base/$rel")

incus/vm-core.func

@@ -19,6 +19,12 @@ _INCUS_VM_CORE_LOADED=1
 
 _CORE_BASE="${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}"
 
+# Retry options for engine fetches. Repeated in each file that bootstraps
+# itself over the network rather than shared, because there is nowhere to put
+# a shared copy that does not itself need downloading first. core/build.func
+# carries the full reasoning.
+_CS_CURL_RETRY=(--retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10)
+
 # Prefer the dispatcher's resolver so a local checkout and forks keep working.
 _incus_vm_source() {
   local rel="$1"
@@ -32,7 +38,7 @@ _incus_vm_source() {
     return
   fi
   # shellcheck disable=SC1090
-  source <(curl -fsSL "${_CORE_BASE}/${rel}")
+  source <(curl -fsSL "${_CS_CURL_RETRY[@]}" "${_CORE_BASE}/${rel}")
 }
 
 # core/core.func first: it owns prompt_confirm/prompt_input/prompt_select,

lib/tools.func

@@ -52,6 +52,12 @@ _TOOLS_FUNC_LOADED=1
 #   hwaccel.func  GPU detection and hardware acceleration
 # ==============================================================================
 
+# Retry options for engine fetches. Repeated in each file that bootstraps
+# itself over the network rather than shared, because there is nowhere to put
+# a shared copy that does not itself need downloading first. core/build.func
+# carries the full reasoning.
+_CS_CURL_RETRY=(--retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10)
+
 _tools_source_part() {
   local part="$1"
 
@@ -66,7 +72,7 @@ _tools_source_part() {
   local base="${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}"
   if command -v curl >/dev/null 2>&1; then
     # shellcheck disable=SC1090
-    source <(curl -fsSL "${base}/lib/${part}.func")
+    source <(curl -fsSL "${_CS_CURL_RETRY[@]}" "${base}/lib/${part}.func")
   else
     # shellcheck disable=SC1090
     source <(wget -qO- "${base}/lib/${part}.func")

lxc/alpine-install.func

@@ -192,7 +192,10 @@ EOF
       exit 1
     fi
   fi
-  source <(curl -fsSL "$(_cs_core_url lib/alpine.func)")
+  # Same treatment as the bootstrap above: a bare `source <(curl ...)` hides the
+  # exit code, so a failed download used to source nothing and leave every
+  # helper below missing with no explanation.
+  _bootstrap_source "$(_cs_core_url lib/alpine.func)" fetch_and_deploy_gh_release
   msg_ok "Updated Container OS"
 }
 

lxc/install.func

@@ -45,15 +45,26 @@ _cs_core_url() {
   echo "${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}/${1:?func path}"
 }
 
-# Retry options for every engine fetch on this side. Defined again rather than
-# taken from build.func because that file stays on the host; this one is piped
-# into the container and has to stand alone.
-#
-# A container is at its most fragile right after boot — DNS may not have come up
-# yet — and exit 115 ("download failed") is one of the larger failure buckets in
-# telemetry. curl's own --retry knows to skip a 404, so a genuinely missing file
-# still fails immediately instead of costing three attempts.
-_CS_CT_CURL_RETRY=(--retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10)
+# Retry options for engine fetches. Repeated here rather than taken from
+# core/build.func, which stays on the host while this file is piped into the
+# container and has to stand alone. A container is at its most fragile right
+# after boot, before DNS is up; core/build.func carries the full reasoning.
+_CS_CURL_RETRY=(--retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10)
+
+# curl -S prints one line per failed attempt, so a download that recovers on the
+# second try would show the user two errors and then quietly work — during a
+# spinner, no less. Hold stderr back and release it only if every attempt failed.
+_cs_curl_retry() {
+  local url="${1:?url}" err rc=0
+  err="$(mktemp 2>/dev/null)" || {
+    curl -fsSL "${_CS_CURL_RETRY[@]}" "$url"
+    return
+  }
+  curl -fsSL "${_CS_CURL_RETRY[@]}" "$url" 2>"$err" || rc=$?
+  ((rc != 0)) && cat "$err" >&2
+  rm -f "$err"
+  return "$rc"
+}
 
 # The engine reaches a container in nine pieces. The host's prefetch cannot
 # help here — a container has its own filesystem — so fetch them concurrently
@@ -93,7 +104,7 @@ _cs_ct_prefetch_engine() {
   done
 
   curl -fsSL --parallel --parallel-immediate --parallel-max 16 \
-    "${_CS_CT_CURL_RETRY[@]}" --max-time 60 "${args[@]}" 2>/dev/null
+    "${_CS_CURL_RETRY[@]}" --max-time 60 "${args[@]}" 2>/dev/null
 
   local kept=0
   for rel in "${_CS_CT_ENGINE_FILES[@]}"; do
@@ -122,9 +133,9 @@ _cs_engine_read() {
     return 0
   fi
   if command -v curl >/dev/null 2>&1; then
-    curl -fsSL "${_CS_CT_CURL_RETRY[@]}" "$(_cs_core_url "$name")"
+    _cs_curl_retry "$(_cs_core_url "$name")"
   else
-    wget -q --tries=3 --waitretry=1 -O- "$(_cs_core_url "$name")"
+    wget -qO- "$(_cs_core_url "$name")"
   fi
 }
 

pve/backend.func

@@ -1424,7 +1424,7 @@ PROFILE
         if [[ "${DEV_MODE_MOTD:-false}" == "true" ]]; then
           echo -e "${TAB}${HOLD}${DGN}Setting up MOTD and SSH for debugging...${CL}"
           if pct exec "$CTID" -- bash -c "
-              source <(curl -fsSL \"${COMMUNITY_SCRIPTS_CORE_URL}/install.func\")
+              source <(curl -fsSL --retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10 \"${COMMUNITY_SCRIPTS_CORE_URL}/lxc/install.func\")
               declare -f motd_ssh >/dev/null 2>&1 && motd_ssh || true
             " >/dev/null 2>&1; then
             local ct_ip=$(pct exec "$CTID" ip a s dev eth0 2>/dev/null | awk '/inet / {print $2}' | cut -d/ -f1)

pve/vm-app.func

@@ -32,6 +32,12 @@
 #
 # ==============================================================================
 
+# Retry options for engine fetches. Repeated in each file that bootstraps
+# itself over the network rather than shared, because there is nowhere to put
+# a shared copy that does not itself need downloading first. core/build.func
+# carries the full reasoning.
+_CS_CURL_RETRY=(--retry 3 --retry-delay 1 --retry-connrefused --connect-timeout 10)
+
 # ==============================================================================
 # SECTION 1: APP DISCOVERY & SELECTION
 # ==============================================================================
@@ -374,8 +380,8 @@ customize_vm_image() {
   local tmp_install_func tmp_tools_func
   tmp_install_func=$(mktemp)
   tmp_tools_func=$(mktemp)
-  curl -fsSL "${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}/lxc/install.func" >"$tmp_install_func"
-  curl -fsSL "${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}/lib/tools.func" >"$tmp_tools_func"
+  curl -fsSL "${_CS_CURL_RETRY[@]}" "${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}/lxc/install.func" >"$tmp_install_func"
+  curl -fsSL "${_CS_CURL_RETRY[@]}" "${COMMUNITY_SCRIPTS_CORE_URL:-https://raw.githubusercontent.com/community-scripts/core/main}/lib/tools.func" >"$tmp_tools_func"
 
   virt-customize -q -a "$WORK_FILE" \
     --mkdir /opt/community-scripts \