deployment.yml

# ---------------------------------------------------------------------------------------------------------------------
# This job workflow does the deployments for custom-staging.yml, deploy.yml and custom-deploy.yml.
#
# The repo must contain a 'deployment/Dockerfile' and a 'deployment/build.gradle' with an installDist task that prepares
# the deployment image in the 'deployment/build' directory; if this condition is not met the deployment will fail.
# Deployments are only done for repos owned by the openremote organisation.
#
# Each deployment consists of an environment and an openremote/manager docker tag, taken from:
#
# - The ENVIRONMENT input (and MANAGER_TAG in this repo) when manually triggered on a branch
# - Otherwise the 'deploy' entries in '.ci_cd/ci_cd.json' for the triggering event (see .ci_cd/README.md)
#
# In this repo a missing manager tag builds the manager image from the checked out commit. Custom projects always deploy
# the manager image matching the openremote version in 'gradle/libs.versions.toml' and use the '.ci_cd' scripts of that
# core release; the project's own '.ci_cd/deploy.sh', '.ci_cd/host_init' and '.ci_cd/aws' files take precedence.
#
# The environment selects '.ci_cd/env/{ENVIRONMENT}.env', the '_{ENVIRONMENT}_' prefixed secrets (in uppercase) and the
# docker compose file (see .ci_cd/README.md).
#
# Secrets, inputs and environment variables are combined for consumption by the '.ci_cd/deploy.sh' bash script and
# written to the 'temp/env' file, which is copied to the host. Later entries override earlier ones:
#
# - secrets and inputs (inputs override secrets with the same name)
# - '.ci_cd/env/.env'
# - '.ci_cd/env/{ENVIRONMENT}.env'
#
# Secrets prefixed with '_TEMP_' are written to the 'temp.env' file instead, which overrides 'temp/env' and is not
# copied to the host; a secret called 'SSH_KEY' is written to the 'ssh.key' file, which is not copied to the host
# either. Other secrets starting with '_' and secrets containing '.' are ignored. Sensitive credentials should always be
# stored in github secrets so they are encrypted, do not store within repo files (even private repos as these are not
# encrypted).
#
# When the '_TEMP_GFE_PASSWORD' secret is set it is used to decrypt the encrypted files of the deployment build.
#
# See https://github.com/openremote/core/tree/main/.ci_cd for details of standard variables and handling.
# ---------------------------------------------------------------------------------------------------------------------

name: Deployment

on:
  workflow_call:
    inputs:
      INPUTS:
        type: string
      COMMIT:
        type: string
        description: "Repo branch or commit SHA to checkout"

jobs:
  deployment:
    name: Deployment
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          ref: ${{ inputs.COMMIT }}
          lfs: true # Enable Git LFS plugin to fetch *.mbtiles

      - name: Get version details
        id: get-version-details
        uses: openremote/core/.github/actions/get-version-details@accaf8ed2cab816cb081ae57be6bebed93c049f0

      - name: Check if core repo
        id: is_core_repo
        uses: openremote/core/.github/actions/check-if-core-repo@357f1387fbc99e7a59752a62064f0bdade57612d

      - name: Checkout core repo
        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        if: ${{ steps.is_core_repo.outputs.value == 'false' }}
        with:
          repository: ${{ steps.is_core_repo.outputs.repository }}
          path: core
          ref: ${{ steps.is_core_repo.outputs.version }}
          sparse-checkout: .ci_cd
          sparse-checkout-cone-mode: false

      # Compatibility with released scripts
      - name: Provide legacy core repo checkout path
        if: ${{ steps.is_core_repo.outputs.value == 'false' }}
        run: ln -s core openremote

      - name: Process ci_cd.json file
        id: ci-cd-output
        uses: openremote/core/.github/actions/process-cicd-json@357f1387fbc99e7a59752a62064f0bdade57612d
        with:
          IS_CORE_REPO: ${{ steps.is_core_repo.outputs.value }}
          MANAGER_VERSION: ${{ steps.is_core_repo.outputs.version }}
          IS_RELEASE: ${{ steps.get-version-details.outputs.is_release }}
          VERSION: ${{ steps.get-version-details.outputs.version }}

      - name: Install QEMU
        uses: docker/setup-qemu-action@99012661954931238ded8c8b007157a8430204e1 # v4.4.0
        with:
          platforms: linux/aarch64 #,linux/amd64 Only ARM is currently in use

      - name: Install Buildx
        id: buildx
        uses: docker/setup-buildx-action@594f3bf4285d9ea8dc53c9a0c9c4092420091003 # v4.4.0

      # --------------------------------------------------------------------------------------
      # Download manager build directory
      # --------------------------------------------------------------------------------------

      - name: Download manager build directory
        uses: actions/download-artifact@9000827ccba6bdab643e8b6fd33ac0654aef8333 # v8.0.2
        if: ${{ steps.is_core_repo.outputs.value == 'true' }}
        with:
          name: or-manager-build
          path: manager/build
        continue-on-error: true

      # --------------------------------------------------------------------------------------
      # Download/build deployment build directory
      # --------------------------------------------------------------------------------------

      - name: Check deployment build.gradle
        id: check_deployment_gradle
        uses: andstor/file-existence-action@558493d6c74bf472d87c84eab196434afc2fa029 # v3.1.0
        with:
          fail: true # Deployment must have a build.gradle file to prepare the deployment files in the deployment/build dir
          files: "deployment/build.gradle"

      - name: Check deployment dockerfile
        id: check_deployment_dockerfile
        uses: andstor/file-existence-action@558493d6c74bf472d87c84eab196434afc2fa029 # v3.1.0
        with:
          fail: true
          files: "deployment/Dockerfile"

      - name: Build deployment
        shell: bash
        run: |
          if [ -n "${{ secrets._TEMP_GFE_PASSWORD }}" ]; then
            export GFE_PASSWORD=${{ secrets._TEMP_GFE_PASSWORD }}
          fi
          ./gradlew -p deployment installDist # TODO: optimize the task graph

      # --------------------------------------------------------------------------------------
      # Deploy
      # --------------------------------------------------------------------------------------

      - name: Process inputs
        id: process-inputs
        shell: python
        run: |
          import os
          import json

          # Overlay all inputs and secrets onto this jobs outputs
          callerInputs = os.getenv("CALLER_INPUTS")
          inputs = os.getenv("INPUTS")
          eventName = os.getenv("GITHUB_EVENT_NAME")

          if inputs is not None and inputs != '':
            inputs = json.loads(inputs)
          if eventName == 'workflow_call' and callerInputs is not None and callerInputs != 'null':
            os.system(f"echo 'Processing caller inputs'")
            inputs = json.loads(callerInputs)

          if inputs is not None and 'INPUTS' in inputs:
            os.system("echo 'Processing inputs from caller'")
            inputs = json.loads(inputs['INPUTS'])

          # Iterate over inputs and assign them as outputs on this step

          if inputs is not None and inputs != 'null':
            for key, value in inputs.items():
              os.system(f"echo 'Outputting input: {key}'")
              lines = len(value.split("\n"))
              if lines > 1:
                  os.system(f"echo '{key}<<EEOOFF\n{value}\nEEOOFF' >> $GITHUB_OUTPUT")
              else:
                  os.system(f"echo '{key}={value}' >> $GITHUB_OUTPUT")
        env:
          CALLER_INPUTS: ${{ inputs.INPUTS }}
          INPUTS: ${{ toJSON(github.event.inputs) }}

      - name: Public IP
        id: ip-address
        shell: bash
        run: |
          OPTS="-sf -m 5"

          # Try Primary v4 -> Fallback v4 -> Default to empty
          PUBLIC_IPV4=$(curl $OPTS v4.ident.me || curl $OPTS 4.tnedi.me || true)

          # Try Primary v6 -> Fallback v6 -> Default to empty
          PUBLIC_IPV6=$(curl $OPTS v6.ident.me || curl $OPTS 6.tnedi.me || true)

          # Validation: Ensure at least one IP was found
          if [[ -z "$PUBLIC_IPV4" && -z "$PUBLIC_IPV6" ]]; then
          echo "::error::Could not determine Public IP (both v4 and v6 failed)."
          exit 1
          fi

          echo "ipv4=$PUBLIC_IPV4" >> $GITHUB_OUTPUT
          echo "ipv6=$PUBLIC_IPV6" >> $GITHUB_OUTPUT

      - name: Sanitize deployments value
        id: deployments
        shell: python
        run: |
          import os
          import sys

          repoOwner = os.getenv('GITHUB_REPOSITORY_OWNER')
          isCoreRepo = os.getenv('IS_CORE_REPO') == 'true'
          deployments = os.getenv('DEPLOYMENTS')
          eventName = os.getenv('GITHUB_EVENT_NAME')
          refType = os.getenv('GITHUB_REF_TYPE')
          inputTag = os.getenv('INPUT_MANAGER_TAG')
          inputEnv = os.getenv('INPUT_ENVIRONMENT')
          managerVersion = os.getenv('MANAGER_VERSION')

          if eventName == 'workflow_dispatch' and refType != 'tag' and repoOwner == 'openremote':
            if isCoreRepo:
              tag=inputTag
              if not inputTag:
                tag='#ref'
            else:
              tag=managerVersion
            deployments=f'{inputEnv}:{tag}'

          if not isCoreRepo:
            if "-SNAPSHOT" in deployments:
              os.system("echo 'SNAPSHOT manager docker images not currently supported for custom project deployment'")
              sys.exit(1)
          os.system(f"echo 'value={deployments}' >> $GITHUB_OUTPUT")
        env:
          IS_CORE_REPO: ${{ steps.is_core_repo.outputs.value }}
          MANAGER_VERSION: ${{ steps.is_core_repo.outputs.version }}
          DEPLOYMENTS: ${{ steps.ci-cd-output.outputs.deploys }}
          INPUT_ENVIRONMENT: ${{ steps.process-inputs.outputs.ENVIRONMENT }}
          INPUT_MANAGER_TAG: ${{ steps.process-inputs.outputs.MANAGER_TAG }}

      - name: Output image tag
        id: get-image-tag
        shell: bash
        run: |
          echo "refTag=$(git rev-parse --short HEAD)" >> $GITHUB_OUTPUT

      - name: Do deployments
        shell: python
        run: |
          import json
          import os
          import sys
          import subprocess

          deployments = os.getenv("DEPLOYMENTS")
          deployments = deployments.split(";")
          managerRef = os.getenv("MANAGER_REF")
          deploymentRef = os.getenv("DEPLOYMENT_REF")
          isCoreRepo = os.getenv("IS_CORE_REPO") == 'true'
          inputs = json.loads(os.getenv("INPUTS", "{}"))
          secrets = json.loads(os.getenv("SECRETS", "{}"))
          inputsAndSecrets = secrets | inputs
          ipv4 = os.getenv("IPV4")
          ipv6 = os.getenv("IPV6")

          failure = False

          # Clean-up AWS credentials related variable that might have been set by Configure AWS Credentials for ECR upload
          os.environ.pop("AWS_ACCESS_KEY_ID", None)
          os.environ.pop("AWS_SECRET_ACCESS_KEY", None)
          os.environ.pop("AWS_SESSION_TOKEN", None)

          # Determine deploy script to use
          deployScript = ".ci_cd/deploy.sh"

          if not os.path.exists(deployScript) and not isCoreRepo:
            deployScript = "core/.ci_cd/deploy.sh"

          if not os.path.exists(deployScript):
            os.system(f"Deploy script not found '{deployScript}'")
            sys.exit(1)

          for deployment in deployments:
            dep = deployment.split(":")
            env = dep[0]
            managerTag = dep[1]
            managerTagFound = True

            os.putenv("MANAGER_TAG", managerTag)
            os.putenv("ENVIRONMENT", env)

            # Clean stale ssh credentials and temp files
            os.system("rm temp.env 2>/dev/null")
            os.system("rm ssh.key 2>/dev/null")
            os.system("rm -r temp 2>/dev/null")
            os.system("mkdir temp")

            # ------------------------------------------------------
            # Output env variables to temp env file for POSIX shell
            # ------------------------------------------------------

            # Output inputs and secrets (spacial handling for SSH_KEY and some other variables)
            # _$ENV_ prefixed keys are output last (to override any non env specific keys)
            environment = (env if env else "").upper()
            prefix = "_" + environment + "_"

            for key, value in inputsAndSecrets.items():
              if "." in key:
                continue

              envFile = "temp/env"

              # Look for temp and env prefixed keys
              if key.startswith("_"):
                if key.startswith("_TEMP_"):
                  key = key.replace("_TEMP_", "")
                  envFile = "temp.env"
                elif key.startswith(prefix):
                  key = key.replace(prefix, "")
                else:
                  continue

              if key == "github_token":
                continue
              else:
                os.system(f"echo 'Secret found {key}...'")
                if key == "SSH_KEY":
                  os.system(f"echo \"{value}\" > ssh.key")
                else:
                  lines = len(value.split("\n"))
                  if lines > 1:
                      os.system(f"echo '{key}='\"'\"'' >> {envFile}")
                      os.system(f"echo '{value}'\"'\"'' >> {envFile}")
                  else:
                      os.system(f"echo '{key}='\"'\"'{value}'\"'\"'' >> {envFile}")

            # Output new line
            os.system(f"echo '\n' >> {envFile}")

            # Output env file if exists
            if os.path.exists(".ci_cd/env/.env"):
              os.system(f"echo 'Outputting .ci_cd/env/.env to temp/env'")
              os.system("cat .ci_cd/env/.env >> temp/env")
              # Output new line
              os.system(f"echo '\n' >> {envFile}")

            # Output environment specific env file if exists
            if env is not None and env != '' and os.path.exists(f".ci_cd/env/{env}.env"):
              os.system(f"echo 'Outputting .ci_cd/env/{env}.env to temp/env'")
              os.system(f"cat .ci_cd/env/{env}.env >> temp/env")
              # Output new line
              os.system(f"echo '\n' >> {envFile}")

            # Set CIDR environment variable
            if ipv4 is not None and ipv4 != '':
              os.putenv("CIDR", ipv4 + '/32')
            elif ipv6 is not None and ipv6 != '':
              os.putenv("CIDR", ipv6 + '/64')

            # Execute deploy script
            os.system(f"echo 'Executing deploy script for deployment: managerTag={managerTag} deploymentTag={deploymentRef} environment={env}'")
            # Uncomment this in combination with the SSH debug step afterwards to debug deployment script
            #sys.exit(0)
            result = subprocess.run(f"bash {deployScript}", shell=True)

            if result.returncode != 0:
              os.system(f"echo 'Deployment failed: managerTag={managerTag} deploymentTag={deploymentRef} environment={env}'")
              failure = True
              continue

          if failure == True:
            os.system("echo 'One or more deployments failed'")
            sys.exit(1)
        env:
          IS_CORE_REPO: ${{ steps.is_core_repo.outputs.value }}
          IS_MAIN_REPO: ${{ steps.is_core_repo.outputs.value }} # Compatibility with released scripts
          DEPLOYMENTS: ${{ steps.deployments.outputs.value }}
          BUILDX_BUILDER: ${{ steps.buildx.outputs.name }}
          MANAGER_DOCKER_BUILD_PATH: manager/build/install/manager
          DEPLOYMENT_DOCKER_BUILD_PATH: deployment/build
          MANAGER_REF: ${{ steps.get-image-tag.outputs.refTag }}
          DEPLOYMENT_REF: ${{ steps.get-image-tag.outputs.refTag }}
          INPUTS: ${{ toJSON(steps.process-inputs.outputs) }}
          SECRETS: ${{ toJSON(secrets) }}
          IPV4: ${{ steps.ip-address.outputs.ipv4 }}
          IPV6: ${{ steps.ip-address.outputs.ipv6 }}