diff --git a/.github/workflows/test-artifact-cleanup.yml b/.github/workflows/test-artifact-cleanup.yml deleted file mode 100644 index a0891c5a..00000000 --- a/.github/workflows/test-artifact-cleanup.yml +++ /dev/null @@ -1,29 +0,0 @@ -name: test-artifact-cleanup -# During this workflow, we upload a file that follows the 'telemetry-tools-*' -# After running the clean-up-artifacts action, the artifact should no longer show up -# in the web UI. - -on: - workflow_dispatch: - -jobs: - telemetry-setup: - runs-on: ubuntu-latest - permissions: - contents: read - actions: write - steps: - - name: Checkout code - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - persist-credentials: false - - name: Create dummy file - shell: bash - run: echo "Dumbo" > file.txt - - name: Upload dummy file - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: telemetry-tools-attrs-1234 - path: file.txt - - name: Clean up telemetry intermediary artifacts - uses: $/telemetry-impls/clean-up-artifacts diff --git a/README.md b/README.md index 6ead1668..3ab3f2d5 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # shared-actions -Contains all of the shared composite actions used by RAPIDS. Several of these actions, -especially the telemetry actions, use a pattern that we refer to as "dispatch actions." +Contains all of the shared composite actions used by RAPIDS. Several of these actions +use a pattern that we refer to as "dispatch actions." The general idea of a dispatch action is to make it easier to depend on other actions at a specific revision, and also to simplify using files beyond a given action .yml file. @@ -110,57 +110,6 @@ used by the `actions/checkout` action, taking priority over the default values. ## Calling in child shared workflows -Shared workflows complicate matters because environment variables do not get -passed through. If you set the `SHARED_ACTIONS_REPO` and/or `SHARED_ACTIONS_REF` -variables in the top-level parent workflow, they will not take effect in any -dispatch actions that you may call in child workflows. You can pass them as inputs -to child shared workflows, but that ends up being very verbose. - -To carry this information into child workflows, we use a scheme that writes a -file with environment variables, uploads this file as an artifact, then downloads -and loads the file at the start of the child workflow. - -The general scheme is: - -### Top-level workflow -```yaml -jobs: - setup-env-vars: - runs-on: ubuntu-latest - steps: - # implicitly picks up env vars for SHARED_ACTIONS_REPO and _REF - - uses: rapidsai/shared-actions/telemetry-dispatch-stash-base-env@main - - - - summarize-telemetry: - needs: - # private networks will affect your choice here. If your tempo server or - # forwarder/collector is only accessible on some node types, then use one of - # those instances here - runs-on: - steps: - - uses: rapidsai/shared-actions/telemetry-dispatch-summarize@main -``` - -### Child workflows -```yaml -jobs: - tests: - strategy: - matrix: ${{ fromJSON(needs.compute-matrix.outputs.MATRIX) }} - runs-on: "linux-${{ matrix.ARCH }}-gpu-${{ matrix.GPU }}-${{ matrix.DRIVER }}-1" - steps: - - name: Telemetry setup - uses: rapidsai/shared-actions/telemetry-dispatch-setup@main - continue-on-error: true - extra_attributes: "rapids.cuda=${{ matrix.CUDA_VER }},rapids.py=${{ matrix.PY_VER }}" - - -``` - -Behind the scenes, the implementation actions are: -* ./telemetry-impls/stash-base-env-vars: storing base environment variables (including setting default values): -* ./telemetry-impls/load-then-clone: Downloads base env var file, loads it, then - clones shared-actions according to env vars that were just loaded -* ./telemetry-impls/summarize: Runs Python script to parse GitHub logs and send OpenTelemetry spans to endpoint +Reusable workflows do not inherit environment variables from their callers. +Set `SHARED_ACTIONS_REPO` and `SHARED_ACTIONS_REF` in the workflow that invokes +the dispatch action when testing an alternative implementation. diff --git a/telemetry-dispatch-setup/action.yml b/telemetry-dispatch-setup/action.yml deleted file mode 100644 index 2cdaac11..00000000 --- a/telemetry-dispatch-setup/action.yml +++ /dev/null @@ -1,56 +0,0 @@ -name: telemetry-dispatch-setup -description: | - This script sets important environment variables that may be used by tools that - implement OpenTelemetry. This script also stores attributes (metadata) for the - current job, so that this metadata can be associated with spans during the final - parsing of job metadata. - - Obtains GitHub Actions job list and matches current job using runner name and attempt number. - - This action should be called at the beginning of child workflows, generally as the first - step in any job other than computing the matrix. - -inputs: - extra_attributes: - description: "comma-separated key=value attributes to associate with the current job" - -runs: - using: 'composite' - steps: - - uses: rapidsai/shared-actions/telemetry-impls/load-then-clone@main # zizmor: ignore[unpinned-uses] - id: has_base_env_var_artifact - - name: Output that setup was skipped if the base env var artifact was not found - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'false' - shell: bash - run: echo "Skipping telemetry-setup because base env var artifact was not found" - - name: Creating folder for job-created telemetry artifacts - shell: bash - run: mkdir -p telemetry-artifacts - - uses: ./shared-actions/telemetry-impls/github-actions-job-info # zizmor: ignore[self-repository] - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'true' - - shell: bash - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'true' - run: # zizmor: ignore[github-env] - printf 'JOB_ID=%s\n' "$(cat job_info.json | jq -r '.id' | tr -d '\n')" >> ${GITHUB_ENV}; - # overrides loaded value. - - name: Set OTEL_SERVICE_NAME from job - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'true' - uses: ./shared-actions/telemetry-impls/set-otel-service-name # zizmor: ignore[self-repository] - - name: Add attribute metadata beyond the stashed basic stuff - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'true' - shell: bash - env: - EXTRA_ATTRIBUTES: ${{ inputs.extra_attributes }} - run: # zizmor: ignore[github-env] - attributes="${OTEL_RESOURCE_ATTRIBUTES}"; - labels="$(jq -r '.labels | join(" ")' job_info.json)"; - if [ "${labels}" != "" ]; then - attributes="${attributes},rapids.labels=${labels}"; - fi; - if [ "${EXTRA_ATTRIBUTES}" != "" ]; then - attributes="${attributes},${EXTRA_ATTRIBUTES}"; - fi; - attributes=$(echo "${attributes}" | sed 's/^,//'); - attributes=$(echo "${attributes}" | sed 's/,$//'); - attributes=$(echo "${attributes}" | sed -r "s/(git.job_url=[^,]+)/\1\/job\/${JOB_ID}/"); - printf 'OTEL_RESOURCE_ATTRIBUTES=%s\n' "$(printf '%s' "${attributes}" | tr -d '\n')" >> ${GITHUB_ENV}; diff --git a/telemetry-dispatch-stash-base-env-vars/action.yml b/telemetry-dispatch-stash-base-env-vars/action.yml deleted file mode 100644 index c584a5e2..00000000 --- a/telemetry-dispatch-stash-base-env-vars/action.yml +++ /dev/null @@ -1,25 +0,0 @@ -name: telemetry-dispatch-stash-base-env-vars -description: | - Stores base environment variables in a file and uploads that file - as an artifact. - - This action should only be called once in a build, - at the start of the top-level workflow. All other jobs in the top - level workflow should come after this job. It is generally enough - to have only the checks and devcontainers jobs explicitly depend on - it and have everything else be downstream of them. - -runs: - using: 'composite' - steps: - # We can't use the load-then-clone action because the env vars file - # that it needs is something that we create here. - - name: Clone shared-actions repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ env.SHARED_ACTIONS_REPO || 'rapidsai/shared-actions' }} - ref: ${{ env.SHARED_ACTIONS_REF || 'main' }} - path: ./shared-actions - persist-credentials: false - - name: Stash base env vars - uses: ./shared-actions/telemetry-impls/stash-base-env-vars # zizmor: ignore[self-repository] diff --git a/telemetry-dispatch-stash-job-artifacts/action.yml b/telemetry-dispatch-stash-job-artifacts/action.yml deleted file mode 100644 index 16e7c9fa..00000000 --- a/telemetry-dispatch-stash-job-artifacts/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: dispatch-stash-job-artifacts -description: | - Clones a particular branch/ref of a shared-actions repo, then - call the stash-artifacts implementation script, which writes - some environment variables so that downstream jobs can refer to them. - - Inputs here are all assumed to be env vars set outside of this script. - Set them in your main repo's workflows (export to ${GITHUB_ENV}!!) - -runs: - using: 'composite' - steps: - - uses: rapidsai/shared-actions/telemetry-impls/load-then-clone@main # zizmor: ignore[unpinned-uses] - id: has_base_env_var_artifact - # Stash current job's OTEL_RESOURCE_ATTRIBUTES and any files in the telemetry-artifacts directory - - name: Stash job artifacts - uses: ./shared-actions/telemetry-impls/stash-job-artifacts # zizmor: ignore[self-repository] - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'true' - - name: Output that setup was skipped if the base env var artifact was not found - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'false' - shell: bash - run: echo "Skipping telemetry-stash-job-artifacts because base env var artifact was not found" diff --git a/telemetry-dispatch-summarize/action.yml b/telemetry-dispatch-summarize/action.yml deleted file mode 100644 index 695d16b5..00000000 --- a/telemetry-dispatch-summarize/action.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: telemetry-dispatch-summarize -description: | - This action is run in a final job on the top-level workflow, after all other - jobs are completed. This action downloads the JSON records of all jobs from - the current run. It then associates metadata records that were uploaded with - the telemetry-dispatch-stash-job-artifacts action with jobs. This is - effectively label metadata. Finally, this action creates OpenTelemetry spans - with the timing and label metadata, and sends it to the configured Tempo - endpoint (or forwarder). - -runs: - using: 'composite' - steps: - - uses: rapidsai/shared-actions/telemetry-impls/load-then-clone@main # zizmor: ignore[unpinned-uses] - id: has_base_env_var_artifact - - uses: ./shared-actions/telemetry-impls/summarize # zizmor: ignore[self-repository] - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'true' - - if: steps.has_base_env_var_artifact.outputs.artifact_found == 'false' - shell: bash - run: echo "Skipping telemetry-summarize because base env var artifact was not found" diff --git a/telemetry-dispatch-write-summary/action.yml b/telemetry-dispatch-write-summary/action.yml deleted file mode 100644 index cc085572..00000000 --- a/telemetry-dispatch-write-summary/action.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: dispatch-summarize -description: | - Clones a particular branch/ref of a shared-actions repo, then calls its telemetry summarize - action. The summarize action downloads and parses GitHub job metadata, and creates - OpenTelemetry spans from the job metadata. These are sent to the configured OTLP receiver/endpoint. - - DEPRECATED: this is here to ensure a smooth transition while rolling out https://github.com/rapidsai/shared-actions/pull/28 - It can/should be removed when there are no longer references to it in any repo pr.yaml files. - -inputs: - cert_concat: - description: Concatenation of certs (CA;Client;ClientKey) - extra_attributes: - description: - Additional attributes to add to OTEL_RESOURCE_ATTRIBUTES. - See https://opentelemetry.io/docs/languages/sdk-configuration/general/#otel_resource_attributes - -runs: - using: 'composite' - steps: - - name: Clone shared-actions repo - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 - with: - repository: ${{ env.SHARED_ACTIONS_REPO || 'rapidsai/shared-actions' }} - ref: ${{ env.SHARED_ACTIONS_REF || 'main' }} - path: ./shared-actions - persist-credentials: false - # This is necessary because this action will generally be run in a job separately from - # where the env vars are set - - name: Load base environment variables - uses: ./shared-actions/telemetry-impls/load-base-env-vars # zizmor: ignore[self-repository] - - name: Run summarize action - uses: ./shared-actions/telemetry-impls/summarize # zizmor: ignore[self-repository] - with: - cert_concat: ${{ inputs.cert_concat }} - extra_attributes: ${{ inputs.extra_attributes }} diff --git a/telemetry-impls/README.md b/telemetry-impls/README.md deleted file mode 100644 index 4e27f0da..00000000 --- a/telemetry-impls/README.md +++ /dev/null @@ -1,12 +0,0 @@ -These actions are meant to be called by "telemetry-dispatch-*" scripts, not directly. - -Consumers may set SHARED_ACTIONS_REPO and SHARED_ACTIONS_REF to test changes - in their customized shared-actions repo. In general, you should not add inputs - to these scripts. Instead, they should add those inputs to the variables that - get stashed in telemetry-impls/stash-base-env-vars. One exception to this rule - is secrets. Secrets shouldn't be written to a file that then gets uploaded, - and they are not directly usable in shared actions. They must be passed in - as inputs. - -Defaults are sometimes set in these implementation files. For example, -`stash-base-env-vars/action.yml` sets several default values. diff --git a/telemetry-impls/clean-up-artifacts/action.yml b/telemetry-impls/clean-up-artifacts/action.yml deleted file mode 100644 index 0a35d817..00000000 --- a/telemetry-impls/clean-up-artifacts/action.yml +++ /dev/null @@ -1,54 +0,0 @@ -name: 'clean-up-artifacts' -description: | - Artifact cleanup removes all of the temporary "carrier" files that are created - by each build job to transmit attribute values. Otherwise, these would be - noisy and make it harder to find "real," interesting artifacts. - -runs: - using: 'composite' - steps: - - name: Clean up telemetry files - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - if: runner.debug != '1' - with: - retries: 3 - script: | - const runAttempt = parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) - get_artifacts = async ({github, context, process}) => { - const opts = github.rest.actions.listWorkflowRunArtifacts.endpoint.merge( - { - attempt_number: runAttempt, - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: 100 - }); - const artifacts = await github.paginate(opts); - return artifacts; - }; - - var artifacts = undefined; - var retryCount = 0; - maxRetries = 5; - initialDelay = 2000; - while (artifacts === undefined) { - artifacts = await get_artifacts({github, context, process}); - if (artifacts === undefined && retryCount < maxRetries) { - retryCount++; - await new Promise((res) => setTimeout(res, initialDelay * Math.pow(2, retryCount - 1))); - } else { - break; - } - } - - await Promise.all( - artifacts - .filter(artifact => artifact.name.startsWith('telemetry-tools-')) - .map(artifact => { - github.rest.actions.deleteArtifact({ - owner: context.repo.owner, - repo: context.repo.repo, - artifact_id: artifact.id - }); - }) - ); diff --git a/telemetry-impls/ensure-otel-cli-available/action.yml b/telemetry-impls/ensure-otel-cli-available/action.yml deleted file mode 100644 index eefa9499..00000000 --- a/telemetry-impls/ensure-otel-cli-available/action.yml +++ /dev/null @@ -1,36 +0,0 @@ -name: 'ensure-otel-cli-available' -description: | - 'Ensure otel-cli is available for sending spans' - - DEPRECATED: this is here to ensure a smooth transition while rolling out https://github.com/rapidsai/shared-actions/pull/28 - It can/should be removed when there are no longer references to it in any repo pr.yaml files. -inputs: - otel_cli_version: - description: "version of otel-cli to download" - default: "0.4.5" -runs: - using: 'composite' - steps: - - shell: bash - id: install-otel-cli - env: - OTEL_CLI_VERSION: ${{ inputs.otel_cli_version }} - run: # zizmor: ignore[github-env] - - mkdir -p bin; - echo $(pwd)/bin >> $GITHUB_PATH; - if ! otel-cli --help 2>&1 >/dev/null; then - ARCH=$(uname -m); - if [ "$(uname -m)" = "x86_64" ]; then - ARCH="amd64"; - else - ARCH="arm64"; - fi; - curl -L -o otel-cli-${ARCH}.tar.gz https://github.com/equinix-labs/otel-cli/releases/download/v${OTEL_CLI_VERSION}/otel-cli_${OTEL_CLI_VERSION}_linux_${ARCH}.tar.gz; - tar -zxf otel-cli-${ARCH}.tar.gz; - mv otel-cli ./bin/; - rm -rf otel-cli-${ARCH}.tar.gz; - fi; - curl -LO https://github.com/rapidsai/gha-tools/releases/latest/download/tools.tar.gz; - tar -xzf tools.tar.gz -C ./bin; - echo "$(pwd)/bin" >> ${GITHUB_PATH}; diff --git a/telemetry-impls/github-actions-job-info/action.yml b/telemetry-impls/github-actions-job-info/action.yml deleted file mode 100644 index 67758a61..00000000 --- a/telemetry-impls/github-actions-job-info/action.yml +++ /dev/null @@ -1,82 +0,0 @@ -name: 'GitHub Actions Job info' -description: | - Obtains job metadata from the GitHub Actions API. - Provides one of two files on disk: - - job_info.json: metadata for the job that is calling this script. Jobs use this to know their - unique ID, which is used to associate attribute metadata with job timing information. - - all_jobs.json: metadata for all jobs under the top-level workflow. This is used by the - final job, which parses this file, associates the span information with attribute metadata, - and sends the spans to the OpenTelemetry receiver endpoint. - -inputs: - all_jobs: - description: | - When true, saves a JSON file with all jobs. Otherwise, saves a JSON file - from only the current job. - default: "false" - -runs: - using: 'composite' - steps: - - uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0 - id: get-job-info - env: - ALL_JOBS: "${{ inputs.all_jobs }}" - with: - retries: 3 - script: | - const fs = require('node:fs') - const runAttempt = parseInt(process.env.GITHUB_RUN_ATTEMPT, 10) - - get_job = async ({github, context, process}) => { - const opts = github.rest.actions.listJobsForWorkflowRunAttempt.endpoint.merge( - { - attempt_number: runAttempt, - owner: context.repo.owner, - repo: context.repo.repo, - run_id: context.runId, - per_page: 100 - }); - const jobs = await github.paginate(opts); - return jobs; - }; - - var jobs = undefined; - var retryCount = 0; - maxRetries = 5; - initialDelay = 2000; - while (jobs === undefined) { - jobs = await get_job({github, context, process}); - if (jobs === undefined && retryCount < maxRetries) { - retryCount++; - await new Promise(res => setTimeout(res, initialDelay * Math.pow(2, retryCount - 1))); - } else { - break; - } - } - - if (process.env.ALL_JOBS === 'true') { - try { - fs.writeFileSync('all_jobs.json', JSON.stringify(jobs)); - } catch(err) { - console.error(err) - throw(err) - } - } else { - // We know what the run ID is, but we don't know which specific job we're being run from. - // https://github.com/orgs/community/discussions/8945 - const this_job = jobs.find((job) => { - return job.runner_name === process.env.RUNNER_NAME && job.run_attempt === runAttempt; - }); - - - if (this_job === undefined){ - throw "Could not identify current job in workflow. Please see debug logs for more info." - } - try { - fs.writeFileSync('job_info.json', JSON.stringify(this_job)); - } catch(err) { - console.error(err) - throw(err) - } - } diff --git a/telemetry-impls/load-base-env-vars/action.yml b/telemetry-impls/load-base-env-vars/action.yml deleted file mode 100644 index 9dd7d673..00000000 --- a/telemetry-impls/load-base-env-vars/action.yml +++ /dev/null @@ -1,27 +0,0 @@ -name: load-base-env-vars -description: | - Load environment variables from a file and set them in the current env. This is used to pass - information from the base workflow into child workflows. - - This script is the counterpart to stash-base-env-vars. It is called by child shared-workflows - to load variables that are set by the top-level workflow. - -runs: - using: 'composite' - steps: - - name: Set environment variables from file into GITHUB_ENV - shell: bash - # Only set the env var if it is not already set - # the ${!VARIABLE} syntax is called "indirect expansion" and it is kind of equivalent to ${${env_var_name}} - # in other words, expand to find the variable name, then dereference that variable name - # The goofy env_var_value filtering through tr is to ensure that the strings don't include quotes. - run: | # zizmor: ignore[github-env] - while read LINE; do - env_var_name="$( cut -d '=' -f 1 <<< "$LINE" )"; - if [ "${!env_var_name}" = "" ]; then - env_var_value="$(echo ${LINE#*=} | tr -d '"')" - echo "${env_var_name}=$(echo "${env_var_value}" | sed 's/^,//')" >> ${GITHUB_ENV}; - else - echo "Load base env info: ignoring new value for "${env_var_name}" in loading base env vars. It is already set to "${!env_var_name}"." >&2; - fi - done > ${GITHUB_ENV}; - else - echo "Load base env info: ignoring new value for "${env_var_name}" in loading base env vars. It is already set to "${!env_var_name}"." >&2; - fi - done &1 >/dev/null; then - echo "otel-cli is missing"; - exit 1; - fi - - if ! type -P rapids-get-telemetry-trace-id 2>&1 >/dev/null; then - echo "rapidsai/gha-tools is missing or lacks telemetry scripts"; - exit 1; - fi diff --git a/telemetry-impls/set-otel-service-name/action.yml b/telemetry-impls/set-otel-service-name/action.yml deleted file mode 100644 index c672f6ca..00000000 --- a/telemetry-impls/set-otel-service-name/action.yml +++ /dev/null @@ -1,32 +0,0 @@ -name: set-otel-service-name -description: | - Set OTEL_SERVICE_NAME from github job info - - OTEL_SERVICE_NAME is a special environment variable that represents a job for us. - Each different OTEL_SERVICE_NAME is represented by a different color when looking at the trace - in Grafana. We have different OTEL_SERVICE_NAME for each different job, as - well as each matrix configuration of jobs that have matrices. - - Obtains GitHub Actions job list and matches current job using runner name and attempt number. - Returns name of current workflow, which will include any matrix components - Sets the OTEL_SERVICE_NAME environment variable. Output is also here if you prefer. -outputs: - name: - description: 'The name of the job matching the current runner name and attempt number' - value: ${{ steps.get-job-name.outputs.JOB_NAME }} - -runs: - using: 'composite' - steps: - - uses: ./shared-actions/telemetry-impls/github-actions-job-info # zizmor: ignore[self-repository] - id: github-job-info - - shell: bash - id: get-job-name - run: - echo JOB_NAME="$(cat job_info.json | jq -r '.name')" >> ${GITHUB_OUTPUT}; - - shell: bash - id: set-otel-service-name-env - env: - JOB_NAME: ${{ steps.get-job-name.outputs.JOB_NAME }} - run: | # zizmor: ignore[github-env] - printf 'OTEL_SERVICE_NAME=%s\n' "$(printf '%s' "${JOB_NAME}" | tr -d '\n')" >> ${GITHUB_ENV}; diff --git a/telemetry-impls/stash-base-env-vars/action.yml b/telemetry-impls/stash-base-env-vars/action.yml deleted file mode 100644 index 844c6bd9..00000000 --- a/telemetry-impls/stash-base-env-vars/action.yml +++ /dev/null @@ -1,39 +0,0 @@ -name: stash-base-env-vars -description: | - Stash base environment variables - - This is a strategy to pass environment variable values between jobs. There are - a lot of variables that need to be passed to a lot of child workflows. Because - the top-level env is not inherited by child shared-workflow calls, we either have - to have lots of inputs, or pass these environment variables another way. The - idea here is that the top-level job exports these values, then the child - shared-workflow jobs start with loading these values. - - This script should only ever be called by the top-level workflow. The child workflows should - load this, then augment the environment in whatever way they need to (e.g. matrix values) - -runs: - using: 'composite' - steps: - - name: Write base env vars to a file - shell: bash - run: | - OTEL_RESOURCE_ATTRIBUTES="${OTEL_RESOURCE_ATTRIBUTES},git.repository=${GITHUB_REPOSITORY}" - OTEL_RESOURCE_ATTRIBUTES="${OTEL_RESOURCE_ATTRIBUTES},git.ref=${GITHUB_REF}" - OTEL_RESOURCE_ATTRIBUTES="${OTEL_RESOURCE_ATTRIBUTES},git.sha=${GITHUB_SHA}" - OTEL_RESOURCE_ATTRIBUTES="${OTEL_RESOURCE_ATTRIBUTES},git.job_url=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}/actions/runs/${GITHUB_RUN_ID}" - cat < telemetry-env-vars - OTEL_EXPORTER_OTLP_ENDPOINT=${OTEL_EXPORTER_OTLP_ENDPOINT:-https://alloy.local.gha-runners.nvidia.com:4317} - OTEL_TRACES_EXPORTER=${OTEL_TRACES_EXPORTER:-otlp} - OTEL_EXPORTER_OTLP_PROTOCOL=${OTEL_EXPORTER_OTLP_PROTOCOL:-grpc} - OTEL_RESOURCE_ATTRIBUTES="$(echo "${OTEL_RESOURCE_ATTRIBUTES}" | sed 's/^,//')" - SHARED_ACTIONS_REPO=${SHARED_ACTIONS_REPO:-rapidsai/shared-actions} - SHARED_ACTIONS_REF=${SHARED_ACTIONS_REF:-main} - EOF - - name: Upload env vars file - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: telemetry-tools-env-vars - path: telemetry-env-vars - - name: Re-export env vars so that defaults take effect - uses: ./shared-actions/telemetry-impls/load-base-env-vars # zizmor: ignore[self-repository] diff --git a/telemetry-impls/stash-job-artifacts/action.yml b/telemetry-impls/stash-job-artifacts/action.yml deleted file mode 100644 index 2ce36907..00000000 --- a/telemetry-impls/stash-job-artifacts/action.yml +++ /dev/null @@ -1,22 +0,0 @@ -name: stash-job-artifacts -description: | - Saves and uploads a file with telemetry attributes that should be attached to spans from this run. - - We stash only the attributes here because we retrieve the rest of the timing - info later. We get info for all jobs at once, so we wait to retrieve that info - at the very end of the top-level job. - -runs: - using: 'composite' - steps: - - name: Write attributes to file, one per line - shell: bash - run: - IFS=, read -ra values <<< "$OTEL_RESOURCE_ATTRIBUTES"; - printf "%s\n" "${values[@]}" > telemetry-artifacts/attrs; - - - name: Upload attr file and any other files - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - with: - name: telemetry-tools-artifacts-${{ env.JOB_ID }} - path: telemetry-artifacts diff --git a/telemetry-impls/summarize/action.yml b/telemetry-impls/summarize/action.yml deleted file mode 100644 index e016fa57..00000000 --- a/telemetry-impls/summarize/action.yml +++ /dev/null @@ -1,62 +0,0 @@ -name: 'Telemetry summarize' -description: | - Consumes job info, parses into spans, and pushes spans. - -runs: - using: 'composite' - steps: - - name: Setup python - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0 - with: - python-version: 3.14 - cache: 'pip' - cache-dependency-path: './shared-actions/telemetry-impls/summarize/requirements.txt' - - name: Install dependencies - run: pip install -r './shared-actions/telemetry-impls/summarize/requirements.txt' - shell: bash - # Writes JSON file that jobs below consume - - uses: ./shared-actions/telemetry-impls/github-actions-job-info # zizmor: ignore[self-repository] - id: github-job-info - with: - all_jobs: true - - name: Upload job JSON file if debugging - uses: actions/upload-artifact@cf430e030ddbb5b0abf93d22962f4752f3646cd9 # v7.0.2 - if: runner.debug == '1' - with: - name: telemetry-tools-all_jobs.json - path: all_jobs.json - # This downloads ALL of the files that we have collected from each job. - - uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2 - with: - path: telemetry-artifacts - pattern: telemetry-tools-* - merge-multiple: false - - - name: Run parse and send trace/spans to endpoint - shell: bash - run: | # zizmor: ignore[github-env] - OTEL_SERVICE_NAME=${OTEL_SERVICE_NAME:-${GITHUB_WORKFLOW}-${GITHUB_REPOSITORY#*/}} - export OTEL_SERVICE_NAME - printf 'OTEL_SERVICE_NAME=%s\n' "$(printf '%s' "${OTEL_SERVICE_NAME}" | tr -d '\n')" >> $GITHUB_ENV - TRACEPARENT=$(./shared-actions/telemetry-impls/traceparent.sh "${OTEL_SERVICE_NAME}") - export TRACEPARENT - printf 'TRACEPARENT=%s\n' "$(printf '%s' "${TRACEPARENT}" | tr -d '\n')" >> $GITHUB_ENV - - ls -lR telemetry-artifacts - timeout 5m python3 ./shared-actions/telemetry-impls/summarize/send_trace.py - - name: Clean up attributes artifacts from all jobs - uses: ./shared-actions/telemetry-impls/clean-up-artifacts # zizmor: ignore[self-repository] - - - name: Add link in step summary - shell: bash - run: | - TRACE_ID="$(cut -d'-' -f2 <<<${TRACEPARENT})" - TRACE_URL="https://grafana.arc-nvks.gha-runners.nvidia.com/explore?schemaVersion=1&panes=%7B%22qqy%22:%7B%22datasource%22:%22tempo%22,%22queries%22:%5B%7B%22refId%22:%22A%22,%22datasource%22:%7B%22type%22:%22tempo%22,%22uid%22:%22tempo%22%7D,%22queryType%22:%22traceql%22,%22limit%22:20,%22tableType%22:%22traces%22,%22query%22:%22${TRACE_ID}%22%7D%5D,%22range%22:%7B%22from%22:%22now-15m%22,%22to%22:%22now%22%7D%7D%7D&orgId=1" - echo "**View Run Trace [in (NVIDIA Internal) Grafana](${TRACE_URL})**" >> "${GITHUB_STEP_SUMMARY}" - # Telemetry jobs should never affect the pass/fail status of the overall CI - - name: Exit success - if: always() - continue-on-error: true - shell: bash - run: | - exit 0 diff --git a/telemetry-impls/summarize/bump_time.py b/telemetry-impls/summarize/bump_time.py deleted file mode 100644 index 2d58c371..00000000 --- a/telemetry-impls/summarize/bump_time.py +++ /dev/null @@ -1,54 +0,0 @@ -#!/usr/bin/env python -# Copyright (c) 2024-2026, NVIDIA CORPORATION. - -# This script is meant to act on an 'all_jobs.json' file that comes from -# the summarize job when debug info is enabled. Bumping the time makes -# it easier to re-run the span-sending python script and check results -# in either Jaeger or Grafana - -import datetime -import json - -with open("all_jobs.json") as f: - jobs = json.load(f) - - -def _parse_time(x: str) -> int: - return int(datetime.datetime.strptime(x, "%Y-%m-%dT%H:%M:%SZ").timestamp() * 1e9) # noqa: DTZ007 - - -start_time = _parse_time(jobs[0]["created_at"]) -needed_time = _parse_time(jobs[-3]["completed_at"]) - _parse_time(jobs[0]["created_at"]) -new_start_time = datetime.datetime.now(datetime.timezone.utc) - datetime.timedelta(minutes=60) - -for idx, job in enumerate(jobs): - if job["created_at"]: - job["created_at"] = ( - new_start_time + datetime.timedelta(seconds=(_parse_time(job["created_at"]) - start_time) / 1e9) - ).strftime("%Y-%m-%dT%H:%M:%SZ") - if job["started_at"]: - job["started_at"] = ( - new_start_time + datetime.timedelta(seconds=(_parse_time(job["started_at"]) - start_time) / 1e9) - ).strftime("%Y-%m-%dT%H:%M:%SZ") - if job["completed_at"]: - job["completed_at"] = ( - new_start_time + datetime.timedelta(seconds=(_parse_time(job["completed_at"]) - start_time) / 1e9) - ).strftime("%Y-%m-%dT%H:%M:%SZ") - steps = [] - for step in job["steps"]: - if step["started_at"]: - step["started_at"] = ( - new_start_time + datetime.timedelta(seconds=(_parse_time(step["started_at"]) - start_time) / 1e9) - ).strftime("%Y-%m-%dT%H:%M:%SZ") - if step["completed_at"]: - step["completed_at"] = ( - new_start_time + datetime.timedelta(seconds=(_parse_time(step["completed_at"]) - start_time) / 1e9) - ).strftime("%Y-%m-%dT%H:%M:%SZ") - steps.append(step) - job["steps"] = steps - - jobs[idx] = job - - -with open("all_jobs.json", "w") as f: - json.dump(jobs, f) diff --git a/telemetry-impls/summarize/requirements.txt b/telemetry-impls/summarize/requirements.txt deleted file mode 100644 index a6e00743..00000000 --- a/telemetry-impls/summarize/requirements.txt +++ /dev/null @@ -1,4 +0,0 @@ -opentelemetry-api==1.30.* -opentelemetry-sdk==1.30.* -opentelemetry-exporter-otlp-proto-grpc==1.30.* -googleapis-common-protos<2.0.dev0,>=1.56.2 diff --git a/telemetry-impls/summarize/send_trace.py b/telemetry-impls/summarize/send_trace.py deleted file mode 100644 index 3ea08d2d..00000000 --- a/telemetry-impls/summarize/send_trace.py +++ /dev/null @@ -1,429 +0,0 @@ -# Copyright (c) 2019-2026, NVIDIA CORPORATION. -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -"""Process a GitHub workflow log record and output OpenTelemetry span data.""" - -from __future__ import annotations - -import hashlib -import json -import logging -import os -import re -import sys -from dataclasses import dataclass -from datetime import datetime, timezone -from pathlib import Path -from typing import TYPE_CHECKING - -from opentelemetry import trace -from opentelemetry.sdk.resources import Resource -from opentelemetry.sdk.trace import TracerProvider -from opentelemetry.sdk.trace.export import BatchSpanProcessor -from opentelemetry.sdk.trace.id_generator import IdGenerator -from opentelemetry.trace.status import StatusCode - -if TYPE_CHECKING: - from collections.abc import Mapping - from typing import Any - -match os.getenv("OTEL_EXPORTER_OTLP_PROTOCOL"): - case "http/protobuf": - from opentelemetry.exporter.otlp.proto.http.trace_exporter import ( - OTLPSpanExporter, - ) - case "grpc": - from opentelemetry.exporter.otlp.proto.grpc.trace_exporter import ( - OTLPSpanExporter, - ) - case _: - from opentelemetry.sdk.trace.export import ( - ConsoleSpanExporter as OTLPSpanExporter, - ) - - -logging.basicConfig(level=logging.DEBUG) - -SpanProcessor = BatchSpanProcessor - - -def parse_attributes(attrs: os.PathLike | str | None) -> dict[str, str]: - """Attempt to parse attributes in a given list `attrs`.""" - attrs_list: list[str] - if not attrs: - return {} - try: - with Path(attrs).open() as attribute_file: - attrs_list = attribute_file.readlines() - except FileNotFoundError: - attrs_list = str(attrs).split(",") - attributes = {} - for attr in attrs_list: - try: - key, value = attr.split("=", 1) - except ValueError: - logging.warning("Invalid attribute: %s", attr) - continue - attributes[key] = value.strip().strip('"') - logging.debug("Attribute parsed: Key: %s, Value: %s", key, value) - return attributes - - -def date_str_to_epoch(date_str: str, value_if_not_set: int | None = 0) -> int: - """GitHub logs come in RFC 3339; this converts to nanoseconds since epoch.""" - if date_str: - # replace bit is to attach the UTC timezone to our datetime object, so - # that it doesn't "help" us by adjusting our string value, which is - # already in UTC - timestamp_ns = int( - datetime.strptime(date_str, "%Y-%m-%dT%H:%M:%SZ").replace(tzinfo=timezone.utc).timestamp() * 1e9 - ) - else: - timestamp_ns = value_if_not_set or 0 - return timestamp_ns - - -def map_conclusion_to_status_code(conclusion: str) -> StatusCode: - """Map string conclusion from logs to OTel enum.""" - if conclusion == "success": - return StatusCode.OK - if conclusion == "failure": - return StatusCode.ERROR - return StatusCode.UNSET - - -class RapidsSpanIdGenerator(IdGenerator): - """ID Generator that generates span IDs. Trace IDs come from elsewhere.""" - - def __init__(self, trace_id: int, job_name: str) -> None: - """Initialize generator with trace ID and initial job_name.""" - self.trace_id = trace_id - self.job_name = job_name - self.step_name = None - - def update_job_name(self, job_name: str) -> None: - """Update the job name, which feeds into computing the span name.""" - self.job_name = job_name - logging.debug("Job name updated: %s", self.job_name) - - def update_step_name(self, step_name: str) -> None: - """Update the step name, which feeds into computing the span name.""" - self.step_name = step_name - logging.debug("Step name updated: %s", self.step_name) - - def generate_span_id(self) -> int: - """Get a new span ID. - - This is called by the OpenTelemetry SDK as-is, without arguments. - To change the generated value here, use the update methods. - """ - span_id = hashlib.sha256() - span_id.update(str(self.trace_id).encode()) - span_id.update(bytes(self.job_name.encode())) - if self.step_name: - span_id.update(bytes(self.step_name.encode())) - return int(span_id.hexdigest()[:16], 16) - - def generate_trace_id(self) -> int: - """Return the trace ID that is stored at initialization.""" - return self.trace_id - - -@dataclass -class Compiler: - hits: int = 0 - misses: int = 0 - errors: int = 0 - - @property - def requests(self) -> int: - """Calculate the total requests.""" - return self.hits + self.misses + self.errors - - @property - def hit_rate(self) -> float: - """Calculate the cache hit rate.""" - return self.hits / self.requests if self.requests > 0 else 0 - - @property - def miss_rate(self) -> float: - """Calculate the cache miss rate.""" - return self.misses / self.requests if self.requests > 0 else 0 - - @property - def error_rate(self) -> float: - """Calculate the cache error rate.""" - return self.errors / self.requests if self.requests > 0 else 0 - - -class SccacheStats: - requests: int = 0 - compilers: dict[str, Compiler] - - def __init__(self, requests: int, compilers: dict[str, Compiler]) -> None: - self.requests = requests - self.compilers = compilers - - @property - def hits(self) -> int: - """Calculate the total cache hits.""" - return sum(v.hits for v in self.compilers.values()) - - @property - def misses(self) -> int: - """Calculate the total cache misses.""" - return sum(v.misses for v in self.compilers.values()) - - @property - def errors(self) -> int: - """Calculate the total cache errors.""" - return sum(v.errors for v in self.compilers.values()) - - @property - def hit_rate(self) -> float: - """Calculate the total cache hit rate.""" - return self.hits / self.requests if self.requests > 0 else 0 - - @property - def miss_rate(self) -> float: - """Calculate the total cache miss rate.""" - return self.misses / self.requests if self.requests > 0 else 0 - - @property - def error_rate(self) -> float: - """Calculate the total cache error rate.""" - return self.errors / self.requests if self.requests > 0 else 0 - - -def get_sccache_stats(artifact_folder: Path) -> dict[str, str]: - """Get sccache stats from the artifact folder.""" - stats_files = list(artifact_folder.glob("sccache-stats*.txt")) - logging.debug("SCCache stats files: %s", stats_files) - parsed_stats = {} - lang_line_match = re.compile(r"Cache (?P\w+) \((?P\w+)[^)]*\)\s*(?P\d+)") - for file in stats_files: - with file.open() as f: - stats = SccacheStats( - requests=0, - compilers={ - "c": Compiler(), - "cHeader": Compiler(), - "c++": Compiler(), - "c++Header": Compiler(), - "c/c++": Compiler(), - "objc": Compiler(), - "objc++": Compiler(), - "cuda": Compiler(), - "ptx": Compiler(), - "cubin": Compiler(), - "rust": Compiler(), - "hip": Compiler(), - }, - ) - print("SCCache stats file: %s", file) - for line in f: - print("Line: %s", line) - if match := re.match(r"^Compile\srequests\s+(\d+).*", line, re.IGNORECASE): - stats.requests = int(match.group(1)) - elif match := lang_line_match.match(line): - compiler = stats.compilers[match.group("lang")] - setattr(compiler, match.group("result"), int(match.group("count"))) - stats_file_name = re.findall(r"sccache-stats[-]?(?P\w+).txt", file.name) - if stats_file_name: - stats_file_name = stats_file_name[0] - else: - stats_file_name = "main_process" - parsed_stats[stats_file_name] = stats - return parsed_stats - - -def process_job_blob( - trace_id: int, - job: Mapping[str, Any], - env_vars: Mapping[str, str], - first_timestamp: int, - last_timestamp: int, -) -> int: - """Transform job JSON into an OTel span.""" - # This is the top-level workflow, which we account for with the root - # trace above - if job["name"] == env_vars.get("OTEL_SERVICE_NAME"): - logging.debug("Job name is the same as the service name: %s", job["name"]) - return last_timestamp - # this cuts off matrix info from the job name, such that grafana can group - # these by name - if "/" in job["name"]: - job_name, matrix_part = job["name"].split("/", 1) - job_name = job_name.strip() - matrix_part = matrix_part.strip() - else: - job_name, matrix_part = job["name"], None - - job_id = job["id"] - logging.info("Processing job: %s", job_name) - job_create = date_str_to_epoch(job["created_at"], first_timestamp) - job_start = date_str_to_epoch(job["started_at"], first_timestamp) - # this may get later in time as we progress through the steps. It is - # common to have the job completion time be earlier than the end of - # the final cleanup steps - job_last_timestamp = date_str_to_epoch(job["completed_at"], job_start) - - if job_start == 0: - logging.info("Job is empty (no start time) - bypassing") - return last_timestamp - - if job_last_timestamp < job_start: - logging.info("Job finish is before start - setting zero length job") - job_last_timestamp = job_start - - artifact_folder = Path.cwd() / f"telemetry-artifacts/telemetry-tools-artifacts-{job_id}" - attributes = {} - if (artifact_folder / "attrs").exists(): - logging.debug("Found attribute file for job: %s", job_id) - attributes = parse_attributes(artifact_folder / "attrs") - else: - logging.debug("No attribute metadata found for job: %s", job_id) - - attributes["service.name"] = job_name - - sccache_stats = get_sccache_stats(artifact_folder) - logging.debug("SCCache stats: %s", sccache_stats) - - job_provider = TracerProvider( - resource=Resource(attributes), - id_generator=RapidsSpanIdGenerator(trace_id=trace_id, job_name=job["name"]), - ) - job_provider.add_span_processor(span_processor=SpanProcessor(OTLPSpanExporter())) - job_tracer = trace.get_tracer("GitHub Actions parser", "0.0.1", tracer_provider=job_provider) - - with job_tracer.start_as_current_span( - name=matrix_part or job["name"], - start_time=job_create, - end_on_exit=False, - ) as job_span: - logging.debug("Job span created: %s", job_span) - job_span.set_status(map_conclusion_to_status_code(job["conclusion"])) - - job_provider.id_generator.update_step_name("Start delay time") - delay_span = job_tracer.start_span(name="Start delay time", start_time=job_create) - delay_span.end(job_start) - logging.debug("Delay span created: %s", delay_span) - - for step in job["steps"]: - step_start = date_str_to_epoch(step["started_at"], job_last_timestamp) - step_end = date_str_to_epoch(step["completed_at"], step_start) - job_last_timestamp = max(step_end, job_last_timestamp) - job_provider.id_generator.update_step_name(step["name"]) - # Reset attributes for each step - span_attributes = {} - - if (step_end - step_start) / 1e9 > 1: - logging.debug("processing step: %s", step["name"]) - job_provider.id_generator.update_step_name(step["name"]) - # Only add sccache attributes if this is a build step - if re.match(r"(?:[\w+]+\sbuild$)|(?:Build\sand\srepair.*)", step["name"], re.IGNORECASE): - logging.debug("Adding sccache attributes for step: %s", step["name"]) - for file_name, stats in sccache_stats.items(): - span_attributes[f"sccache.{file_name}.hit_rate"] = stats.hit_rate - span_attributes[f"sccache.{file_name}.miss_rate"] = stats.miss_rate - span_attributes[f"sccache.{file_name}.error_rate"] = stats.error_rate - span_attributes[f"sccache.{file_name}.requests"] = stats.requests - for lang, lang_stats in stats.compilers.items(): - span_attributes[f"sccache.{file_name}.{lang}.hit_rate"] = lang_stats.hit_rate - span_attributes[f"sccache.{file_name}.{lang}.miss_rate"] = lang_stats.miss_rate - span_attributes[f"sccache.{file_name}.{lang}.error_rate"] = lang_stats.error_rate - span_attributes[f"sccache.{file_name}.{lang}.requests"] = lang_stats.requests - - # TODO: Ninja log files? - # TODO: file sizes of packages or their contents - - step_span = job_tracer.start_span( - name=step["name"], - start_time=step_start, - attributes=span_attributes, - ) - logging.debug("Step span created: %s", step_span) - if span_attributes: - logging.debug(" Step span attributes: %s", span_attributes) - # TODO: use step_span.record_exception(exc) to capture errors. exc is an exception object, - # so if we are reading from a log file, we need to read the log file into an exception object. - step_span.set_status(map_conclusion_to_status_code(step["conclusion"])) - step_span.end(step_end) - logging.debug("Step span ended: %s", step_span) - else: - logging.debug("Skipping step: %s, because its duration is < 1s", step["name"]) - job_span.end(job_last_timestamp) - logging.debug("Job span ended: %s", job_span) - return last_timestamp - - -def main() -> None: - """Run core functionality.""" - with Path("all_jobs.json").open() as f: - jobs = json.loads(f.read()) - - first_timestamp = date_str_to_epoch(jobs[0]["created_at"]) - # track the latest timestamp observed and use it for any unavailable times. - last_timestamp = date_str_to_epoch(jobs[0]["completed_at"], 0) - - attribute_folders = list(Path.cwd().glob("telemetry-artifacts/telemetry-tools-artifacts-*")) - logging.debug("Attribute folders: %s", attribute_folders) - if attribute_folders: - attribute_file = attribute_folders[0] / "attrs" - attributes = parse_attributes(attribute_file.as_posix()) - env_vars = parse_attributes(attribute_folders[0] / "telemetry-env-vars") - logging.debug("Env vars parsed from first attribute folder: %s", env_vars) - else: - attributes = {} - try: - env_vars = parse_attributes(Path.cwd() / "telemetry-artifacts/telemetry-env-vars") - except FileNotFoundError: - env_vars = os.environ - global_attrs = {k: v for k, v in attributes.items() if k.startswith("git.")} - if not (service_name := os.getenv("OTEL_SERVICE_NAME", env_vars.get("OTEL_SERVICE_NAME"))): - logging.error("OTEL_SERVICE_NAME not found in environment or attribute files") - sys.exit(1) - if not (trace_id := os.getenv("TRACEPARENT", env_vars.get("TRACEPARENT"))): - logging.error("TRACEPARENT not found in environment or attribute files") - sys.exit(1) - global_attrs["service.name"] = service_name - trace_id = int(trace_id.split("-")[1], 16) - - logging.debug("Service name: %s", global_attrs["service.name"]) - logging.debug("Trace ID: %s", trace_id) - - provider = TracerProvider( - resource=Resource(global_attrs), - id_generator=RapidsSpanIdGenerator(trace_id=trace_id, job_name=global_attrs["service.name"]), - ) - provider.add_span_processor(span_processor=SpanProcessor(OTLPSpanExporter())) - tracer = trace.get_tracer("GitHub Actions parser", "0.0.1", tracer_provider=provider) - - with tracer.start_as_current_span( - name="Top-level workflow root", start_time=first_timestamp, end_on_exit=False - ) as root_span: - logging.debug("Root span created: %s", root_span) - for job in jobs: - last_timestamp = process_job_blob( - trace_id=trace_id, - job=job, - env_vars=env_vars, - first_timestamp=first_timestamp, - last_timestamp=last_timestamp, - ) - root_span.end(last_timestamp) - logging.debug("Root span ended: %s", root_span) - - -if __name__ == "__main__": - main() diff --git a/telemetry-impls/traceparent.sh b/telemetry-impls/traceparent.sh deleted file mode 100755 index e16cc666..00000000 --- a/telemetry-impls/traceparent.sh +++ /dev/null @@ -1,42 +0,0 @@ -#!/bin/bash -# Copyright (c) 2024-2026, NVIDIA CORPORATION & AFFILIATES. All rights reserved. - -# This emits a TRACEPARENT, which follows the w3c trace context standard. -# https://www.w3.org/TR/trace-context/ -# -# This script can operate for two purposes: -# 1. The top level of a job, whether it is the job at the source repo (e.g. rmm) level, or -# the matrix job level -# 2. The steps level within a job, which uses both the job name and the step name -# -# The job name must always be provided as the first argument. -# A step name MAY be provided as the second argument. If it is specified, the output corresponds to -# the step within the context of its job. -# -# This is a shell script instead of an action because we need to call it -# in loops that iterate through job metadata. - -JOB_NAME=$1 -STEP_NAME=${2:-} - -if [ "$JOB_NAME" = "" ]; then - echo "ERROR: JOB_NAME (first parameter) is empty. This means your trace doesn't identify anything." - exit 1 -fi - -sha="$(echo "${GITHUB_REPOSITORY}+${GITHUB_RUN_ID}+${GITHUB_RUN_ATTEMPT}" | sha256sum | cut -f1 -d' ')" -TRACE_ID="${sha:0:32}" -JOB_SPAN_ID="${TRACE_ID}-${JOB_NAME}" -STEP_SPAN_ID="${JOB_SPAN_ID}-${STEP_NAME}" - -# echo "JOB_SPAN_ID pre-hash: \"$JOB_SPAN_ID\"" 1>&2 -# echo "STEP_SPAN_ID pre-hash: \"$STEP_SPAN_ID\"" 1>&2 - -JOB_TRACEPARENT=$(echo -n "${JOB_SPAN_ID}" | sha256sum | cut -f1 -d' ') -STEP_TRACEPARENT=$(echo -n "${STEP_SPAN_ID}" | sha256sum | cut -f1 -d' ') - -if [ "${STEP_NAME}" != "" ]; then - echo "00-${TRACE_ID}-${STEP_TRACEPARENT:0:16}-01" -else - echo "00-${TRACE_ID}-${JOB_TRACEPARENT:0:16}-01" -fi diff --git a/telemetry-impls/traceparent/action.yml b/telemetry-impls/traceparent/action.yml deleted file mode 100644 index 4ffd37ab..00000000 --- a/telemetry-impls/traceparent/action.yml +++ /dev/null @@ -1,47 +0,0 @@ -name: traceparent -description: | - Outputs a trace identifier computed from the GitHub runtime environment - - Trace identifier background: - https://www.w3.org/TR/trace-context/ - - Note that the TRACEPARENT env var is special to otel-cli. It will be picked up - automatically and used as a parent to whatever span otel-cli is created. - - DEPRECATED: this is here to ensure a smooth transition while rolling out https://github.com/rapidsai/shared-actions/pull/28 - It can/should be removed when there are no longer references to it in any repo pr.yaml files. -outputs: - traceparent: - description: The W3C-format traceparent, which identifies the current context - value: ${{ steps.shell.outputs.TRACEPARENT }} - -runs: - using: 'composite' - steps: - - uses: ./shared-actions/telemetry-impls/ensure-otel-cli-available # zizmor: ignore[self-repository] - # this ensures that OTEL_SERVICE_NAME is either set before this action, or sets it from the github job metadata - - uses: ./shared-actions/telemetry-impls/set-otel-service-name # zizmor: ignore[self-repository] - if: env.OTEL_SERVICE_NAME == '' - - uses: ./shared-actions/telemetry-impls/sanity-checks # zizmor: ignore[self-repository] - - - shell: bash - id: output-inputs - if: runner.debug == '1' - run: | - echo "::debug::trace ID input: '${GITHUB_REPOSITORY}+${GITHUB_RUN_ID}+${GITHUB_RUN_ATTEMPT}'" - echo "::debug::$GITHUB_REPOSITORY="${GITHUB_REPOSITORY}"" - echo "::debug::$GITHUB_RUN_ID="${GITHUB_RUN_ID}"" - echo "::debug::$GITHUB_RUN_ATTEMPT="${GITHUB_RUN_ATTEMPT}"" - echo "::debug::Evaluated trace ID input (pre-hash): "${GITHUB_REPOSITORY}+${GITHUB_RUN_ID}+${GITHUB_RUN_ATTEMPT}"" - export TRACE_ID="$(rapids-get-telemetry-trace-id)" - echo "::debug::Computed trace ID: ${TRACE_ID}" - - echo "::debug::JOB TRACEPARENT input (step empty): '00-\${TRACE_ID}-hash(\${TRACE_ID}-\$\{\OTEL_SERVICE_NAME}\})-01'" - echo "::debug::evaluated job traceparent input: "00-${TRACE_ID}-hash\(${TRACE_ID}-${OTEL_SERVICE_NAME}\)-01"" - - - shell: bash - id: shell - run: | # zizmor: ignore[github-env] - TRACEPARENT="$(rapids-get-telemetry-traceparent "${OTEL_SERVICE_NAME}")" - printf 'TRACEPARENT=%s\n' "${TRACEPARENT}" >> ${GITHUB_OUTPUT} - printf 'TRACEPARENT=%s\n' "${TRACEPARENT}" >> ${GITHUB_ENV}