fix: Authenticate release asset downloads with GITHUB_TOKEN

downloader/run.sh

@@ -7,8 +7,9 @@ set -e
 # or a partial version (v1) in the environment variable VERSION, and any parameters you want to
 # pass to the command in PARAMS.
 #
-# Sometimes you will hit Github API rate limits when running this command. If you have a Github token,
-# pass it in environment variable GITHUB_TOKEN.
+# Sometimes you will hit Github rate limits when running this command. If you have a Github token,
+# pass it in environment variable GITHUB_TOKEN; it is used both to list releases and to download
+# the release asset, which raises the applicable rate limit.
 #
 # This script can be used in either Linux or MacOS; it will download whichever binary is
 # appropriate for the current OS and architecture. It cannot be used in Windows. It requires
@@ -41,36 +42,48 @@ if [ "${ARCH}" = "aarch64" ]; then
 fi
 EXECUTABLE_ARCHIVE_NAME="sdk-test-harness_${OS_TYPE}_${ARCH}.${EXTENSION}"
 
-# Github rate-limits requests to its APIs. The effect is that sometimes the contract test step in CI
-# will fail spuriously when trying to find the list of releases.
-# If we provide an auth token, then we can avoid the rate limiting issues.
-if [ -n "${GITHUB_TOKEN}" ]; then
-  AUTH_HEADER="Authorization: Token ${GITHUB_TOKEN}"
-fi
-
 if [ -z "${VERSION}" -o -z "${PARAMS}" ]; then
   echo 'You must specify a version string in $VERSION and command parameters in $PARAMS' >&2
   exit 1
 fi
 
+# Github rate-limits requests, both to its APIs and to the release asset download endpoints. The
+# effect is that sometimes the contract test step in CI fails spuriously. Authenticated requests get
+# a substantially higher limit, so the token is used for every request that accepts it.
+AUTH_TOKEN="${GITHUB_TOKEN}"
+github_curl() {
+  if [ -n "${AUTH_TOKEN}" ]; then
+    curl -sS -L --retry 5 --retry-delay 2 -H "Authorization: Bearer ${AUTH_TOKEN}" "$@"
+  else
+    curl -sS -L --retry 5 --retry-delay 2 "$@"
+  fi
+}
+
 resolve_version() {
   if echo "$1" | grep -q '^v[^.][^.]*\.[^.][^.]*\..'; then
     # It's already a complete version string
     echo "$1"
-    exit
+    return
   fi
-  cmd="curl"
-  if [ -n "${AUTH_HEADER}" ]; then
-    cmd="${cmd} -H '${AUTH_HEADER}'"
-  fi
-  cmd="${cmd} -s ${RELEASES_API_URL}"
-  eval "$cmd" \
+  github_curl --fail "${RELEASES_API_URL}" \
     | grep "tag_name" \
     | sed -e 's/.*:[^"]*"\([^"]*\).*/\1/' \
     | grep "^$1\." \
     | head -n 1
 }
 
+# The unauthenticated github.com download URL drops the Authorization header when it redirects to
+# the asset host, so an authenticated download has to go through the releases API instead.
+resolve_asset_url() {
+  github_curl --fail "${RELEASES_API_URL}/tags/$1" \
+    | tr -d ' \n' \
+    | tr '{' '\n' \
+    | grep "\"name\":\"${EXECUTABLE_ARCHIVE_NAME}\"" \
+    | grep -o "\"url\":\"[^\"]*/releases/assets/[0-9]*\"" \
+    | sed -e 's/^"url":"//' -e 's/"$//' \
+    | head -n 1
+}
+
 VERSION_TO_DOWNLOAD=$(resolve_version "${VERSION}")
 if [ -z "${VERSION_TO_DOWNLOAD}" ]; then
   echo "Unable to find a release matching '${VERSION}'" >&2
@@ -79,13 +92,26 @@ fi
 
 TEMP_DIR="/tmp/sdk-test-harness_${VERSION_TO_DOWNLOAD}"
 EXECUTABLE="${TEMP_DIR}/sdk-test-harness"
-DOWNLOAD_URL="${RELEASES_SITE_URL}/download/${VERSION_TO_DOWNLOAD}/${EXECUTABLE_ARCHIVE_NAME}"
 
 if [ ! -x "${EXECUTABLE}" ]; then
+  DOWNLOAD_URL="${RELEASES_SITE_URL}/download/${VERSION_TO_DOWNLOAD}/${EXECUTABLE_ARCHIVE_NAME}"
+  DOWNLOAD_ACCEPT="*/*"
+  if [ -n "${GITHUB_TOKEN}" ]; then
+    ASSET_URL=$(resolve_asset_url "${VERSION_TO_DOWNLOAD}")
+    if [ -n "${ASSET_URL}" ]; then
+      DOWNLOAD_URL="${ASSET_URL}"
+      DOWNLOAD_ACCEPT="application/octet-stream"
+    else
+      echo "Unable to find asset '${EXECUTABLE_ARCHIVE_NAME}' in release ${VERSION_TO_DOWNLOAD}; falling back to an unauthenticated download" >&2
+      # The public download URL does not need credentials, and a rejected token would fail it.
+      AUTH_TOKEN=""
+    fi
+  fi
+
   rm -rf "${TEMP_DIR}"
   mkdir "${TEMP_DIR}"
   echo "Downloading ${DOWNLOAD_URL}"
-  curl --fail -sS -L --retry 5 --retry-delay 2 \
+  github_curl --fail -H "Accept: ${DOWNLOAD_ACCEPT}" \
     -o "${TEMP_DIR}/archive.${EXTENSION}" "${DOWNLOAD_URL}" \
     || { echo "Download failed" >&2; exit 1; }
   if [ "${EXTENSION}" = "zip" ]; then