From 214c014e3d1c67c0cb621d9a4e2f0211ed966e84 Mon Sep 17 00:00:00 2001 From: Jiwen Cai Date: Sat, 29 Aug 2026 01:55:22 +0000 Subject: [PATCH] ci: stop third-party artifact downloads reddening unrelated PRs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every build-ubuntu matrix entry fetched the DepthAI device firmware and the CloudXR SDKs from the network, so a transient failure at artifacts.luxonis.com or NGC failed a job that had nothing to do with the change under test — 22 red Build Ubuntu runs in one afternoon, and main was hit too. Cache both. depthai-core's DownloadAndChecksum skips any file already on disk, so restoring its resources/ makes the fetch a no-op; the CloudXR scripts already prefer a local tarball in deps/cloudxr/ over NGC. Restore and save are split rather than using the combined cache action, because that one also saves on a failed job and CMake's file(DOWNLOAD) leaves a truncated file behind on error — a half-written .fwp would then satisfy the "already downloaded" check forever. The runtime SDK script also retries the NGC sweep, since curl's --retry never covers the 404 actually observed. The retry wraps the whole listed/unlisted sweep, not each request: a 404 on an individual remote name is how the script discovers which name was published, so per-request retry would cost a delay on every successful run. Refs NVIDIA/IsaacTeleop#1049 Signed-off-by: Jiwen Cai --- .github/actions/setup-cloudxr-sdk/action.yml | 23 +++++++ .github/workflows/build-ubuntu.yml | 23 +++++++ scripts/download_cloudxr_runtime_sdk.sh | 68 ++++++++++++++------ 3 files changed, 93 insertions(+), 21 deletions(-) diff --git a/.github/actions/setup-cloudxr-sdk/action.yml b/.github/actions/setup-cloudxr-sdk/action.yml index 2577aeef4b..0532ec0518 100644 --- a/.github/actions/setup-cloudxr-sdk/action.yml +++ b/.github/actions/setup-cloudxr-sdk/action.yml @@ -30,6 +30,20 @@ runs: echo "CloudXR Runtime SDK version: ${CXR_RUNTIME_SDK_VERSION}" echo "CloudXR Web SDK version: ${CXR_WEB_SDK_VERSION}" + # Both download scripts prefer a tarball already sitting in deps/cloudxr/ over + # fetching from NGC, so a cache hit makes the download a no-op. NGC 404s + # transiently and reddens otherwise-good runs; this keeps the pipeline off the + # network on the common path. Restore only — the save is gated on the downloads + # succeeding, so a partial fetch is never cached. + - name: Restore CloudXR SDK tarballs + id: cache + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + deps/cloudxr/*.tar.gz + deps/cloudxr/*.tgz + key: cloudxr-sdk-${{ runner.os }}-${{ runner.arch }}-rt${{ steps.parse.outputs.runtime_version }}-web${{ steps.parse.outputs.web_version }} + - name: Download CloudXR Runtime SDK shell: bash env: @@ -44,3 +58,12 @@ runs: CXR_WEB_SDK_VERSION: ${{ steps.parse.outputs.web_version }} run: | ./scripts/download_cloudxr_sdk.sh + + - name: Save CloudXR SDK tarballs + if: steps.cache.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: | + deps/cloudxr/*.tar.gz + deps/cloudxr/*.tgz + key: ${{ steps.cache.outputs.cache-primary-key }} diff --git a/.github/workflows/build-ubuntu.yml b/.github/workflows/build-ubuntu.yml index 337889ad4c..9761e7f440 100644 --- a/.github/workflows/build-ubuntu.yml +++ b/.github/workflows/build-ubuntu.yml @@ -110,6 +110,21 @@ jobs: ccache --version ccache --show-config + # depthai-core downloads the device/bootloader *.fwp blobs into its build + # tree at configure time, once per matrix entry; artifacts.luxonis.com fails + # often enough to redden unrelated PRs. Its DownloadAndChecksum skips any + # file already on disk, so a restored resources/ makes the fetch a no-op. + # Keyed on the DEPTHAI_COMMIT pin, which selects the firmware version. + # Split restore/save on purpose: the combined action also saves on a failed + # job, and CMake's file(DOWNLOAD) leaves a truncated file behind on error — + # that would be cached and then pass the "already downloaded" check forever. + - name: Restore DepthAI device resources + id: depthai-resources + uses: actions/cache/restore@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.BUILD_DIR }}/_deps/depthai-build/resources + key: depthai-resources-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('cmake/DepthAIVcpkgManifest.cmake') }} + - name: Configure CMake run: | coverage_args=() @@ -134,6 +149,14 @@ jobs: -DCMAKE_TOOLCHAIN_FILE=${{ env.VCPKG_ROOT }}/scripts/buildsystems/vcpkg.cmake \ "${coverage_args[@]}" + # Configure succeeded, so every *.fwp downloaded and checksummed cleanly. + - name: Save DepthAI device resources + if: steps.depthai-resources.outputs.cache-hit != 'true' + uses: actions/cache/save@55cc8345863c7cc4c66a329aec7e433d2d1c52a9 # v6.1.0 + with: + path: ${{ env.BUILD_DIR }}/_deps/depthai-build/resources + key: ${{ steps.depthai-resources.outputs.cache-primary-key }} + - name: Build run: cmake --build "${BUILD_DIR}" --parallel 4 diff --git a/scripts/download_cloudxr_runtime_sdk.sh b/scripts/download_cloudxr_runtime_sdk.sh index 09e412d48c..caa643308f 100755 --- a/scripts/download_cloudxr_runtime_sdk.sh +++ b/scripts/download_cloudxr_runtime_sdk.sh @@ -180,34 +180,60 @@ install_from_public_ngc() { # Main # ----------------------------------------------------------------------------- +# One listed -> unlisted sweep over the public NGC resources. +install_from_ngc() { + local resource visibility + + if [[ "$CXR_RUNTIME_SDK_VERSION" == *-rc* ]]; then + resource="cloudxr-runtime-for-isaac-teleop" + visibility="unlisted" + else + resource="cloudxr-runtime" + visibility="listed" + fi + + echo "Trying ${visibility} public NGC..." + if install_from_public_ngc "$resource" "$visibility"; then + return 0 + fi + + if [[ "$visibility" == "listed" ]]; then + resource="cloudxr-runtime-for-isaac-teleop" + visibility="unlisted" + echo "Cannot install from listed public NGC, trying unlisted public NGC..." + # Do not combine a partial listed download with the unlisted SDK bundle. + rm -f "$CXR_DEPLOYMENT_DIR/$SDK_FILE" "$CXR_DEPLOYMENT_DIR/$EXP_SDK_FILE" + if install_from_public_ngc "$resource" "$visibility"; then + return 0 + fi + fi + + return 1 +} + # Prefer local tarball if present; otherwise use NGC if install_from_local_tarball; then exit 0 fi -if [[ "$CXR_RUNTIME_SDK_VERSION" == *-rc* ]]; then - NGC_RESOURCE="cloudxr-runtime-for-isaac-teleop" - NGC_VISIBILITY="unlisted" -else - NGC_RESOURCE="cloudxr-runtime" - NGC_VISIBILITY="listed" -fi - -echo "Cannot install from local tarball, trying ${NGC_VISIBILITY} public NGC..." -if install_from_public_ngc "$NGC_RESOURCE" "$NGC_VISIBILITY"; then - exit 0 -fi +echo "Cannot install from local tarball, trying public NGC..." -if [[ "$NGC_VISIBILITY" == "listed" ]]; then - NGC_RESOURCE="cloudxr-runtime-for-isaac-teleop" - NGC_VISIBILITY="unlisted" - echo "Cannot install from listed public NGC, trying unlisted public NGC..." - # Do not combine a partial listed download with the unlisted SDK bundle. - rm -f "$CXR_DEPLOYMENT_DIR/$SDK_FILE" "$CXR_DEPLOYMENT_DIR/$EXP_SDK_FILE" - if install_from_public_ngc "$NGC_RESOURCE" "$NGC_VISIBILITY"; then +# NGC 404s transiently, and curl's --retry never covers 4xx. Retry the whole +# sweep rather than each request: a 404 on an individual remote name is how +# download_ngc_first_match discovers the published name, so per-request retry +# would cost a delay on every successful run. +NGC_ATTEMPTS=3 +NGC_RETRY_DELAY_S=5 +for (( attempt = 1; attempt <= NGC_ATTEMPTS; attempt++ )); do + if install_from_ngc; then exit 0 fi -fi + if (( attempt < NGC_ATTEMPTS )); then + echo -e "${YELLOW}NGC download failed (attempt ${attempt}/${NGC_ATTEMPTS}); retrying in ${NGC_RETRY_DELAY_S}s...${NC}" + rm -f "$CXR_DEPLOYMENT_DIR/$SDK_FILE" "$CXR_DEPLOYMENT_DIR/$EXP_SDK_FILE" + sleep "$NGC_RETRY_DELAY_S" + fi +done -echo "Cannot install from ${NGC_VISIBILITY} public NGC, exiting..." +echo "Cannot install from public NGC after ${NGC_ATTEMPTS} attempts, exiting..." exit 1