diff --git a/.github/workflows/app_ci_pipeline.yml b/.github/workflows/app_ci_pipeline.yml index 6e82bd19..bfdb5f2c 100644 --- a/.github/workflows/app_ci_pipeline.yml +++ b/.github/workflows/app_ci_pipeline.yml @@ -27,7 +27,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Setup Node.js @@ -35,8 +35,10 @@ jobs: with: cache: "npm" cache-dependency-path: "app/package-lock.json" + - name: Install safe-chain + run: curl -fsSL https://github.com/AikidoSec/safe-chain/releases/download/1.5.12/install-safe-chain.sh | sh -s -- --ci - name: Install Node.js dependencies - run: npm ci + run: npm ci --safe-chain-minimum-package-age-hours=168 - name: Sandworm Audit run: npm run sandworm @@ -77,7 +79,7 @@ jobs: node-version: [22.x] steps: - name: Check out Git repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Setup Node.js @@ -96,7 +98,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out Git repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Setup Node.js @@ -120,7 +122,7 @@ jobs: node-version: [22.x] steps: - name: Check out Git repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Setup Node.js @@ -137,7 +139,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Check out Git repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Setup Node.js @@ -162,7 +164,7 @@ jobs: steps: - name: Check out Git repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: persist-credentials: false - name: Setup Node.js diff --git a/.github/workflows/executor_ci_pipeline.yml b/.github/workflows/executor_ci_pipeline.yml new file mode 100644 index 00000000..f701efe4 --- /dev/null +++ b/.github/workflows/executor_ci_pipeline.yml @@ -0,0 +1,36 @@ +name: executor-ci-pipeline + +on: + pull_request: + branches: [main, development] + paths: + - "executor/**" + +permissions: + contents: read + +defaults: + run: + working-directory: ./executor + +jobs: + audit: + runs-on: ubuntu-latest + + strategy: + matrix: + python-version: ["3.12"] + + steps: + - name: Check out Git repository + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + with: + python-version: ${{ matrix.python-version }} + - name: Install safe-chain + run: curl -fsSL https://github.com/AikidoSec/safe-chain/releases/download/1.5.12/install-safe-chain.sh | sh -s -- --ci + - name: Install Python dependencies + run: pip install -r requirements.txt --safe-chain-minimum-package-age-hours=168 diff --git a/.github/workflows/linting.yml b/.github/workflows/linting.yml index 162269f3..1912b132 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/linting.yml @@ -17,12 +17,12 @@ jobs: python-version: ["3.12"] steps: - name: Checkout Repository - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{ github.head_ref }} persist-credentials: false - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0 + uses: actions/setup-python@ece7cb06caefa5fff74198d8649806c4678c61a1 # v6.3.0 with: python-version: ${{ matrix.python-version }} - name: Install dependencies diff --git a/.github/workflows/semantic_version_compare.yml b/.github/workflows/semantic_version_compare.yml index a41d3496..a4cbbc6e 100644 --- a/.github/workflows/semantic_version_compare.yml +++ b/.github/workflows/semantic_version_compare.yml @@ -15,7 +15,7 @@ jobs: steps: - name: Checkout Compare Branch - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{github.ref}} persist-credentials: false @@ -31,7 +31,7 @@ jobs: echo "${new_version}" - name: Checkout Base Branch - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: ref: ${{github.event.pull_request.base.ref}} persist-credentials: false diff --git a/app/Dockerfile b/app/Dockerfile index a4c2340c..c9b1c36e 100644 --- a/app/Dockerfile +++ b/app/Dockerfile @@ -1,28 +1,24 @@ FROM public.ecr.aws/docker/library/node:22-alpine -WORKDIR /usr/src/app +RUN adduser -D -s /bin/sh app && mkdir -p /usr/src/app && chown app:app /usr/src/app +# symlink for prisma/openssl issue on alpine: https://github.com/nodejs/docker-node/issues/2175#issuecomment-2530130523 +RUN ln -s /usr/lib/libssl.so.3 /lib/libssl.so.3 -COPY . . +USER app +WORKDIR /usr/src/app -# RUN apk add --no-cache --virtual .build-deps python3 make g++ \ -# && npm ci \ -# && apk del .build-deps +COPY --chown=app:app package.json package-lock.json ./ RUN npm ci +COPY --chown=app:app . . RUN wget https://truststore.pki.rds.amazonaws.com/global/global-bundle.pem ARG GIT_HASH=${GIT_HASH:-undefined_hash} -# symlink for prisma/openssl issue on alpine: https://github.com/nodejs/docker-node/issues/2175#issuecomment-2530130523 -RUN ln -s /usr/lib/libssl.so.3 /lib/libssl.so.3 RUN npm run prisma:generate-client RUN npm run api:build -RUN adduser -H -D -s /bin/sh app && \ - chown -R app:app /usr/src/app -USER app - EXPOSE 5000 ENTRYPOINT [ "npm" ] CMD ["run", "api:start-deployed"] diff --git a/app/api/.env.copyme b/app/api/.env.copyme index 973033c0..78dd5406 100644 --- a/app/api/.env.copyme +++ b/app/api/.env.copyme @@ -42,3 +42,11 @@ EDU_SNOWFLAKE_ROLE= OAUTH2_ISSUER=http://localhost:8080/realms/example OAUTH2_AUDIENCE=runway-local + + # for tenant sync +UM_SYNC_CRON="0 0 * * *" +UM_URL=https://api.admin.internal-dev.edanalytics.app +# UM_AUTH0_DOMAIN=ea-apps-internal-dev.us.auth0.com +# UM_CLIENT_ID= +# UM_CLIENT_SECRET= +# UM_AUDIENCE=https://admin.internal-dev.edanalytics.app \ No newline at end of file diff --git a/app/api/buildspec.yml b/app/api/buildspec.yml index 75699109..793a4d40 100644 --- a/app/api/buildspec.yml +++ b/app/api/buildspec.yml @@ -1,5 +1,8 @@ version: 0.2 phases: + install: + commands: + - docker buildx create --name containerd --driver=docker-container --driver-opt default-load=true --use pre_build: commands: - pwd @@ -9,16 +12,23 @@ phases: - export ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - export ECR_REPO=$ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPO_NAME - export DOCKER_TAG=$ECR_REPO:$CODEBUILD_RESOLVED_SOURCE_VERSION - - export DOCKER_CACHE=$ECR_REPO:latest + - export DOCKER_LATEST=$ECR_REPO:latest - aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $ECR_REPO - env build: commands: - - docker pull $DOCKER_CACHE || true - - docker build --cache-from $DOCKER_CACHE --build-arg GIT_HASH=${CODEBUILD_RESOLVED_SOURCE_VERSION} -t $DOCKER_TAG . - - docker tag $DOCKER_TAG $DOCKER_CACHE + - >- + docker buildx build + --cache-to type=inline,ref=${DOCKER_LATEST},mode=min + --cache-from type=registry,ref=${DOCKER_LATEST} + --tag ${DOCKER_TAG} + --builder=containerd + --build-arg GIT_HASH=${CODEBUILD_RESOLVED_SOURCE_VERSION} + --progress=plain + . + - docker tag $DOCKER_TAG $DOCKER_LATEST - docker push $DOCKER_TAG - - docker push $DOCKER_CACHE + - docker push $DOCKER_LATEST post_build: commands: - envsubst < Dockerrun.aws.template.json > Dockerrun.aws.json @@ -27,4 +37,4 @@ artifacts: files: - .ebextensions/**/* - .platform/**/* - - Dockerrun.aws.json \ No newline at end of file + - Dockerrun.aws.json diff --git a/app/api/integration/factories/partner-user-tenant.ts b/app/api/integration/factories/partner-user-tenant.ts index c6da0d86..470a833f 100644 --- a/app/api/integration/factories/partner-user-tenant.ts +++ b/app/api/integration/factories/partner-user-tenant.ts @@ -14,11 +14,15 @@ export const makePartnerUserTenantContext = (tag: string) => { idpId: idp.id, descriptorNamespace: null, crossYearMatchingEnabled: false, + managedBy: null, + deletedOn: null, }; const tenant: WithoutAudit = { code: `tenant-${tag}`, partnerId: partner.id, + isGlobal: false, + deletedOn: null, }; const user: WithoutAudit> = { @@ -74,4 +78,3 @@ export const seedContext = async ( user, }; }; - diff --git a/app/api/integration/fixtures/context-fixtures/partner-fixtures.ts b/app/api/integration/fixtures/context-fixtures/partner-fixtures.ts index 652604b8..15cb779a 100644 --- a/app/api/integration/fixtures/context-fixtures/partner-fixtures.ts +++ b/app/api/integration/fixtures/context-fixtures/partner-fixtures.ts @@ -8,6 +8,8 @@ export const partnerA: WithoutAudit = { idpId: idpA.id, descriptorNamespace: 'partner-a', crossYearMatchingEnabled: false, + managedBy: null, + deletedOn: null, }; export const partnerC: WithoutAudit = { @@ -16,6 +18,8 @@ export const partnerC: WithoutAudit = { idpId: idpA.id, // shares idp with partner A descriptorNamespace: 'partner-c', crossYearMatchingEnabled: false, + managedBy: null, + deletedOn: null, }; export const partnerX: WithoutAudit = { @@ -24,4 +28,6 @@ export const partnerX: WithoutAudit = { idpId: idpX.id, descriptorNamespace: null, crossYearMatchingEnabled: false, + managedBy: null, + deletedOn: null, }; diff --git a/app/api/integration/fixtures/context-fixtures/partner-sync-fixtures.ts b/app/api/integration/fixtures/context-fixtures/partner-sync-fixtures.ts new file mode 100644 index 00000000..8fd9ee33 --- /dev/null +++ b/app/api/integration/fixtures/context-fixtures/partner-sync-fixtures.ts @@ -0,0 +1,139 @@ +import { Partner, Tenant } from '@prisma/client'; + +export const syncPartner: Pick = { + id: 'sync-partner', + name: 'Sync Partner', + managedBy: 'user_management_sync', +}; + +export const gonePartner: Pick = { + id: 'gone-partner', + name: 'Gone Partner', + managedBy: 'user_management_sync', +}; + +export const returningPartner: Pick = { + id: 'returning-partner', + name: 'Returning Partner', + managedBy: 'user_management_sync', + deletedOn: new Date('2024-01-01'), +}; + +export const doomedPartner: Pick = { + id: 'doomed-partner', + name: 'Doomed Partner', + managedBy: 'user_management_sync', +}; + +export const unmanagedDeletedPartner: Pick = { + id: 'unmanaged-deleted-partner', + name: 'Unmanaged Deleted Partner', + managedBy: null, + deletedOn: new Date('2024-01-01'), +}; + +export const syncPartnerOldTenant: Pick = { + code: 'old-tenant', + partnerId: syncPartner.id, +}; + +export const syncPartnerReturningTenant: Pick< + Tenant, + 'code' | 'partnerId' | 'deletedOn' | 'isGlobal' +> = { + code: 'returning-tenant', + partnerId: syncPartner.id, + deletedOn: new Date('2024-01-01'), + isGlobal: true +}; + +export const syncPartnerReturningTenantWithChangedGlobal: Pick< + Tenant, + 'code' | 'partnerId' | 'deletedOn' | 'isGlobal' +> = { + code: 'stale-returning-tenant', + partnerId: syncPartner.id, + deletedOn: new Date('2024-01-01'), + isGlobal: false, +}; + +export const syncPartnerUpdateableTenant: Pick< + Tenant, + 'code' | 'partnerId' | 'isGlobal' +> = { + code: 'updateable-tenant', + partnerId: syncPartner.id, + isGlobal: false, +}; + +export const doomedTenant1: Pick = { + code: 'doomed-tenant-1', + partnerId: doomedPartner.id, +}; + +export const doomedTenant2: Pick = { + code: 'doomed-tenant-2', + partnerId: doomedPartner.id, +}; + +export const allActionsPartner: Pick = { + id: 'all-actions-partner', + name: 'Kitchen Sink Partner', + managedBy: 'user_management_sync', +}; + +export const allActionsTenantToDelete: Pick = { + code: 'all-actions-to-delete', + partnerId: allActionsPartner.id, +}; + +export const allActionsTenantToUndelete: Pick< + Tenant, + 'code' | 'partnerId' | 'deletedOn' | 'isGlobal' +> = { + code: 'all-actions-to-undelete', + partnerId: allActionsPartner.id, + deletedOn: new Date('2024-01-01'), + isGlobal: false, +}; + +export const allActionsTenantToUpdate: Pick = { + code: 'all-actions-to-update', + partnerId: allActionsPartner.id, + isGlobal: false, +}; + +export const allActionsTenantUnchanged: Pick = { + code: 'all-actions-unchanged', + partnerId: allActionsPartner.id, + isGlobal: true, +}; + +export const concurrentPartnerOne: Pick = { + id: 'concurrent-partner-one', + name: 'Concurrent Partner One', + managedBy: 'user_management_sync', +}; + +export const concurrentPartnerTwo: Pick = { + id: 'concurrent-partner-two', + name: 'Concurrent Partner Two', + managedBy: 'user_management_sync', +}; + +export const partialFailurePartnerOk: Pick = { + id: 'partial-failure-partner-ok', + name: 'Partial Failure Partner Ok', + managedBy: 'user_management_sync', +}; + +export const partialFailurePartnerFail: Pick = { + id: 'partial-failure-partner-fail', + name: 'Partial Failure Partner Fail', + managedBy: 'user_management_sync', +}; + +export const partialFailureExistingTenant: Pick = { + code: 'partial-failure-existing-tenant', + partnerId: partialFailurePartnerFail.id, +}; diff --git a/app/api/integration/fixtures/context-fixtures/tenant-fixtures.ts b/app/api/integration/fixtures/context-fixtures/tenant-fixtures.ts index 8b2f21b3..a3ef873e 100644 --- a/app/api/integration/fixtures/context-fixtures/tenant-fixtures.ts +++ b/app/api/integration/fixtures/context-fixtures/tenant-fixtures.ts @@ -5,19 +5,27 @@ import { partnerA, partnerC, partnerX } from './partner-fixtures'; export const tenantA: WithoutAudit = { code: 'tenant-a', partnerId: partnerA.id, + isGlobal: false, + deletedOn: null, }; export const tenantB: WithoutAudit = { code: 'tenant-b', partnerId: partnerA.id, + isGlobal: false, + deletedOn: null, }; export const tenantC: WithoutAudit = { code: 'tenant-c', partnerId: partnerC.id, // shares idp with partner A + isGlobal: false, + deletedOn: null, }; export const tenantX: WithoutAudit = { code: 'tenant-x', partnerId: partnerX.id, + isGlobal: false, + deletedOn: null, }; diff --git a/app/api/integration/helpers/mocks/pg-boss.mock.ts b/app/api/integration/helpers/mocks/pg-boss.mock.ts new file mode 100644 index 00000000..4418d2c9 --- /dev/null +++ b/app/api/integration/helpers/mocks/pg-boss.mock.ts @@ -0,0 +1,8 @@ +export class PgBoss { + async start() {} + async stop() {} + async unschedule(_name: string) {} + async createQueue(_name: string) {} + async schedule(_name: string, _cron: string, _data: unknown, _opts?: unknown) {} + async work(_name: string, _handler: unknown) {} +} diff --git a/app/api/integration/tests/partner-sync.spec.ts b/app/api/integration/tests/partner-sync.spec.ts new file mode 100644 index 00000000..6df523c5 --- /dev/null +++ b/app/api/integration/tests/partner-sync.spec.ts @@ -0,0 +1,544 @@ +import { + UserManagementTenant, + UserManagementPartner, +} from 'api/src/partner-sync/user-management/um-sync.types'; +import { + syncPartner, + gonePartner, + returningPartner, + doomedPartner, + syncPartnerOldTenant, + syncPartnerReturningTenant, + syncPartnerReturningTenantWithChangedGlobal, + syncPartnerUpdateableTenant, + doomedTenant1, + doomedTenant2, + unmanagedDeletedPartner, + allActionsPartner, + allActionsTenantToDelete, + allActionsTenantToUndelete, + allActionsTenantToUpdate, + allActionsTenantUnchanged, + concurrentPartnerOne, + concurrentPartnerTwo, + partialFailurePartnerOk, + partialFailurePartnerFail, + partialFailureExistingTenant, +} from '../fixtures/context-fixtures/partner-sync-fixtures'; +import { UmSyncHandler } from 'api/src/partner-sync/user-management/um-sync.handler'; +import { AppConfigService } from 'api/src/config/app-config.service'; + +const UM_CONFIG = { + syncCron: '*/5 * * * *', + url: 'https://um.example.com', + auth0Domain: 'auth.example.com', + clientId: 'test-client-id', + clientSecret: 'test-client-secret', + audience: 'https://um.example.com', +}; + +function makeUmTenant( + partnerCode: string, + tenantCode: string, + overrides: Partial = {} +): UserManagementTenant { + return { + partnerCode, + tenantCode, + displayName: tenantCode, + isGlobal: false, + ...overrides, + }; +} + +function mockUmFetch({ + partners = [] as UserManagementPartner[], + tenants = {} as Record, + tokenFails = false, + partnersFail = false, + tenantsFail = [] as string[], +} = {}): jest.SpyInstance { + return jest.spyOn(global, 'fetch').mockImplementation((input) => { + const url = (input as RequestInfo | URL).toString(); + + if (url.includes('/oauth/token')) { + if (tokenFails) { + return Promise.resolve(new Response(null, { status: 401 })); + } + return Promise.resolve( + new Response(JSON.stringify({ access_token: 'test-token', expires_in: 3600 }), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + } + + if (url.includes('/api/v1/partners')) { + if (partnersFail) { + return Promise.resolve(new Response(null, { status: 500 })); + } + return Promise.resolve( + new Response(JSON.stringify(partners), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + } + + if (url.includes('/api/v1/tenants')) { + const partnerCode = new URL(url).searchParams.get('partnerCode') ?? ''; + if (tenantsFail.includes(partnerCode)) { + return Promise.resolve(new Response(null, { status: 500 })); + } + return Promise.resolve( + new Response(JSON.stringify(tenants[partnerCode] ?? []), { + status: 200, + headers: { 'Content-Type': 'application/json' }, + }) + ); + } + + return Promise.reject(new Error(`Unexpected fetch call: ${url}`)); + }); +} + +describe('PartnerSyncService.sync', () => { + let service: UmSyncHandler; + let fetchSpy: jest.SpyInstance; + + beforeEach(() => { + service = app.get(UmSyncHandler); + // Reset cached token so each test starts with a fresh auth state + (service as any).umToken = null; + (service as any).umTokenExpiration = null; + // UmSyncHandler's AppConfigService is injected via the global ServicesModule, + // a separate instance from app.get(AppConfigService) — spy on the actual instance it holds. + jest.spyOn((service as any).appConfig as AppConfigService, 'umConfig').mockResolvedValue(UM_CONFIG); + }); + + afterEach(() => { + jest.restoreAllMocks(); + }); + + const sync = () => service.sync(); + + describe('partner sync', () => { + it('creates new partners returned by UM', async () => { + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'new-partner' }], + tenants: { 'new-partner': [] }, + }); + + await sync(); + + const created = await global.prisma.partner.findUnique({ where: { id: 'new-partner' } }); + expect(created).not.toBeNull(); + expect(created?.managedBy).not.toBeNull(); + expect(created?.deletedOn).toBeNull(); + }); + + it('soft-deletes sync-managed partners absent from UM', async () => { + await global.prisma.partner.create({ data: gonePartner }); + + fetchSpy = mockUmFetch({ partners: [], tenants: {} }); + + await sync(); + + const partner = await global.prisma.partner.findUnique({ where: { id: 'gone-partner' } }); + expect(partner?.deletedOn).not.toBeNull(); + }); + + it('does not delete non-sync-managed partners absent from UM', async () => { + // seeded partnerA has managedBy: null — must not be touched + fetchSpy = mockUmFetch({ partners: [], tenants: {} }); + + await sync(); + + const partner = await global.prisma.partner.findUnique({ where: { id: 'partner-a' } }); + expect(partner?.deletedOn).toBeNull(); + }); + + it('does not touch a non-sync-managed partner returned by UM', async () => { + // seeded partnerA has managedBy: null — UM returning its code must not create/modify it + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'partner-a' }], + tenants: {}, + }); + + await sync(); + + const partner = await global.prisma.partner.findUnique({ where: { id: 'partner-a' } }); + expect(partner?.managedBy).toBeNull(); + expect(partner?.deletedOn).toBeNull(); + }); + + it('does not undelete a soft-deleted non-sync-managed partner returned by UM', async () => { + await global.prisma.partner.create({ data: unmanagedDeletedPartner }); + + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'unmanaged-deleted-partner' }], + tenants: {}, + }); + + await sync(); + + const partner = await global.prisma.partner.findUnique({ + where: { id: 'unmanaged-deleted-partner' }, + }); + expect(partner?.managedBy).toBeNull(); + expect(partner?.deletedOn).not.toBeNull(); + }); + + it('undeletes a sync-managed partner that reappears in UM', async () => { + await global.prisma.partner.create({ data: returningPartner }); + + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'returning-partner' }], + tenants: { 'returning-partner': [] }, + }); + + await sync(); + + const partner = await global.prisma.partner.findUnique({ + where: { id: 'returning-partner' }, + }); + expect(partner?.deletedOn).toBeNull(); + }); + }); + + describe('tenant sync', () => { + it('creates new tenants returned by UM', async () => { + await global.prisma.partner.create({ data: syncPartner }); + + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'sync-partner' }], + tenants: { + 'sync-partner': [ + makeUmTenant('sync-partner', 'new-tenant', { + isGlobal: true, + }), + ], + }, + }); + + await sync(); + + const tenant = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'new-tenant', partnerId: 'sync-partner' } }, + }); + expect(tenant).not.toBeNull(); + expect(tenant?.isGlobal).toBe(true); + }); + + it('creates tenants for a partner that UM returns as new in the same sync', async () => { + // 'brand-new-partner' is not seeded — it's created by this same sync run + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'brand-new-partner' }], + tenants: { + 'brand-new-partner': [ + makeUmTenant('brand-new-partner', 'brand-new-tenant', { isGlobal: true }), + ], + }, + }); + + await sync(); + + const partner = await global.prisma.partner.findUnique({ + where: { id: 'brand-new-partner' }, + }); + expect(partner).not.toBeNull(); + + const tenant = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'brand-new-tenant', partnerId: 'brand-new-partner' } }, + }); + expect(tenant).not.toBeNull(); + expect(tenant?.isGlobal).toBe(true); + }); + + it('soft-deletes sync-managed tenants absent from UM', async () => { + await global.prisma.partner.create({ data: syncPartner }); + await global.prisma.tenant.create({ data: syncPartnerOldTenant }); + + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'sync-partner' }], + tenants: { 'sync-partner': [] }, + }); + + await sync(); + + const tenant = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'old-tenant', partnerId: 'sync-partner' } }, + }); + expect(tenant?.deletedOn).not.toBeNull(); + }); + + it('does not delete non-sync-managed tenants absent from UM', async () => { + fetchSpy = mockUmFetch({ partners: [], tenants: {} }); + + await sync(); + + const tenant = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'tenant-a', partnerId: 'partner-a' } }, + }); + expect(tenant?.deletedOn).toBeNull(); + }); + + it('does not request or create tenants for a partner UM does not manage', async () => { + // seeded partnerA has managedBy: null — its tenants must be untouched even if + // UM's tenants endpoint would return some for it + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'partner-a' }], + tenants: { + 'partner-a': [makeUmTenant('partner-a', 'unwanted-tenant')], + }, + }); + + await sync(); + + const tenantRequests = fetchSpy.mock.calls.filter(([input]: [RequestInfo | URL]) => + input.toString().includes('/api/v1/tenants?partnerCode=partner-a') + ); + expect(tenantRequests).toHaveLength(0); + + const tenant = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'unwanted-tenant', partnerId: 'partner-a' } }, + }); + expect(tenant).toBeNull(); + }); + + it('undeletes tenants that reappear in UM', async () => { + await global.prisma.partner.create({ data: syncPartner }); + await global.prisma.tenant.create({ data: syncPartnerReturningTenant }); + + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'sync-partner' }], + tenants: { + 'sync-partner': [makeUmTenant('sync-partner', 'returning-tenant', { isGlobal: true })], + }, + }); + + await sync(); + + const tenant = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'returning-tenant', partnerId: 'sync-partner' } }, + }); + expect(tenant?.deletedOn).toBeNull(); + expect(tenant?.isGlobal).toBe(true); + }); + + it('updates isGlobal when undeleting a tenant whose isGlobal changed while deleted', async () => { + await global.prisma.partner.create({ data: syncPartner }); + await global.prisma.tenant.create({ data: syncPartnerReturningTenantWithChangedGlobal }); + + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'sync-partner' }], + tenants: { + 'sync-partner': [ + makeUmTenant('sync-partner', 'stale-returning-tenant', { isGlobal: true }), + ], + }, + }); + + await sync(); + + const tenant = await global.prisma.tenant.findUnique({ + where: { + code_partnerId: { code: 'stale-returning-tenant', partnerId: 'sync-partner' }, + }, + }); + expect(tenant?.deletedOn).toBeNull(); + expect(tenant?.isGlobal).toBe(true); + }); + + it('updates isGlobal when it changes', async () => { + await global.prisma.partner.create({ data: syncPartner }); + await global.prisma.tenant.create({ data: syncPartnerUpdateableTenant }); + + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'sync-partner' }], + tenants: { + 'sync-partner': [ + makeUmTenant('sync-partner', 'updateable-tenant', { + isGlobal: true, + }), + ], + }, + }); + + await sync(); + + const tenant = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'updateable-tenant', partnerId: 'sync-partner' } }, + }); + expect(tenant?.isGlobal).toBe(true); + }); + + it('soft-deletes all tenants when their sync-managed partner is deleted', async () => { + await global.prisma.partner.create({ data: doomedPartner }); + await global.prisma.tenant.createMany({ data: [doomedTenant1, doomedTenant2] }); + + // UM no longer knows about this partner + fetchSpy = mockUmFetch({ partners: [], tenants: {} }); + + await sync(); + + const tenants = await global.prisma.tenant.findMany({ + where: { partnerId: 'doomed-partner' }, + }); + expect(tenants).toHaveLength(2); + expect(tenants.every((t) => t.deletedOn !== null)).toBe(true); + }); + + it('handles creation, deletion, undeletion, and isGlobal updates together for one partner', async () => { + await global.prisma.partner.create({ data: allActionsPartner }); + await global.prisma.tenant.createMany({ + data: [ + allActionsTenantToDelete, + allActionsTenantToUndelete, + allActionsTenantToUpdate, + allActionsTenantUnchanged, + ], + }); + + fetchSpy = mockUmFetch({ + partners: [{ partnerCode: 'all-actions-partner' }], + tenants: { + // 'all-actions-to-delete' is intentionally absent from the UM response + 'all-actions-partner': [ + makeUmTenant('all-actions-partner', 'all-actions-to-undelete', { isGlobal: true }), + makeUmTenant('all-actions-partner', 'all-actions-to-update', { isGlobal: true }), + makeUmTenant('all-actions-partner', 'all-actions-unchanged', { isGlobal: true }), + makeUmTenant('all-actions-partner', 'all-actions-new', { isGlobal: false }), + ], + }, + }); + + await sync(); + + const tenants = await global.prisma.tenant.findMany({ + where: { partnerId: 'all-actions-partner' }, + }); + const byCode = new Map(tenants.map((t) => [t.code, t])); + + expect(byCode.get('all-actions-to-delete')?.deletedOn).not.toBeNull(); + + expect(byCode.get('all-actions-to-undelete')?.deletedOn).toBeNull(); + expect(byCode.get('all-actions-to-undelete')?.isGlobal).toBe(true); + + expect(byCode.get('all-actions-to-update')?.deletedOn).toBeNull(); + expect(byCode.get('all-actions-to-update')?.isGlobal).toBe(true); + + expect(byCode.get('all-actions-unchanged')?.deletedOn).toBeNull(); + expect(byCode.get('all-actions-unchanged')?.isGlobal).toBe(true); + + expect(byCode.get('all-actions-new')).not.toBeUndefined(); + expect(byCode.get('all-actions-new')?.deletedOn).toBeNull(); + expect(byCode.get('all-actions-new')?.isGlobal).toBe(false); + }); + + it('syncs tenants independently for multiple partners in a single run', async () => { + await global.prisma.partner.createMany({ data: [concurrentPartnerOne, concurrentPartnerTwo] }); + + fetchSpy = mockUmFetch({ + partners: [ + { partnerCode: 'concurrent-partner-one' }, + { partnerCode: 'concurrent-partner-two' }, + ], + tenants: { + 'concurrent-partner-one': [ + makeUmTenant('concurrent-partner-one', 'tenant-one', { isGlobal: true }), + ], + 'concurrent-partner-two': [ + makeUmTenant('concurrent-partner-two', 'tenant-two', { isGlobal: false }), + ], + }, + }); + + await sync(); + + const tenantOne = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'tenant-one', partnerId: 'concurrent-partner-one' } }, + }); + const tenantTwo = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'tenant-two', partnerId: 'concurrent-partner-two' } }, + }); + const crossAssigned = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'tenant-one', partnerId: 'concurrent-partner-two' } }, + }); + + expect(tenantOne).not.toBeNull(); + expect(tenantOne?.isGlobal).toBe(true); + expect(tenantTwo).not.toBeNull(); + expect(tenantTwo?.isGlobal).toBe(false); + expect(crossAssigned).toBeNull(); + }); + }); + + describe('error handling', () => { + it('makes no DB changes when the token fetch fails', async () => { + await global.prisma.partner.create({ data: syncPartner }); + + fetchSpy = mockUmFetch({ tokenFails: true }); + + await sync(); + + // sync-managed partner must not be soft-deleted when we cannot reach UM + const partner = await global.prisma.partner.findUnique({ where: { id: 'sync-partner' } }); + expect(partner?.deletedOn).toBeNull(); + }); + + it('makes no DB changes when the partners fetch fails', async () => { + await global.prisma.partner.create({ data: syncPartner }); + + fetchSpy = mockUmFetch({ partnersFail: true }); + + await sync(); + + // sync-managed partner must not be soft-deleted when we cannot reach UM + const partner = await global.prisma.partner.findUnique({ where: { id: 'sync-partner' } }); + expect(partner?.deletedOn).toBeNull(); + }); + + it('continues syncing other partners when one partner\'s tenant fetch fails', async () => { + await global.prisma.partner.createMany({ + data: [partialFailurePartnerOk, partialFailurePartnerFail], + }); + await global.prisma.tenant.create({ data: partialFailureExistingTenant }); + + fetchSpy = mockUmFetch({ + partners: [ + { partnerCode: 'partial-failure-partner-ok' }, + { partnerCode: 'partial-failure-partner-fail' }, + ], + tenants: { + 'partial-failure-partner-ok': [ + makeUmTenant('partial-failure-partner-ok', 'ok-tenant', { isGlobal: true }), + ], + // UM no longer lists this tenant — if the fetch succeeded, it would be soft-deleted + 'partial-failure-partner-fail': [], + }, + tenantsFail: ['partial-failure-partner-fail'], + }); + + await sync(); + + const okTenant = await global.prisma.tenant.findUnique({ + where: { code_partnerId: { code: 'ok-tenant', partnerId: 'partial-failure-partner-ok' } }, + }); + expect(okTenant).not.toBeNull(); + expect(okTenant?.isGlobal).toBe(true); + + // the failing partner's tenant fetch never succeeded, so its existing tenant + // must be left untouched rather than treated as absent-from-UM and deleted + const untouchedTenant = await global.prisma.tenant.findUnique({ + where: { + code_partnerId: { + code: 'partial-failure-existing-tenant', + partnerId: 'partial-failure-partner-fail', + }, + }, + }); + expect(untouchedTenant?.deletedOn).toBeNull(); + }); + }); +}); diff --git a/app/api/jest.config.integration.ts b/app/api/jest.config.integration.ts index ef8d5a3a..6cefa351 100644 --- a/app/api/jest.config.integration.ts +++ b/app/api/jest.config.integration.ts @@ -7,6 +7,9 @@ export default { '^.+\\.[tj]s$': ['ts-jest', { tsconfig: '/tsconfig.spec.json' }], }, moduleFileExtensions: ['ts', 'js', 'html'], + moduleNameMapper: { + '^pg-boss$': '/integration/helpers/mocks/pg-boss.mock.ts', + }, // global setup is run in a separate process from setupFiles and setupFilesAfterEnv // and so cannot share global context with tests, though it does share global context diff --git a/app/api/src/app/app.module.ts b/app/api/src/app/app.module.ts index 02e5949c..3453261e 100644 --- a/app/api/src/app/app.module.ts +++ b/app/api/src/app/app.module.ts @@ -17,6 +17,7 @@ import { ExternalApiV1Module } from '../external-api/v1/external-api.v1.module'; import { AuthorizedGuard } from '../auth/login/authorized.guard'; import { PartnersModule } from '../partners/partners.module'; import { SchoolYearConfigModule } from '../school-year-config/school-year-config.module'; +import { PartnerSyncModule } from '../partner-sync/partner-sync.module'; const resourceModules = [ UsersModule, @@ -36,6 +37,7 @@ const resourceModules = [ RouterModule.register(routes), AuthModule, ...resourceModules, + PartnerSyncModule, ExternalApiV1Module, ], controllers: [AppController], diff --git a/app/api/src/config/app-config.service.ts b/app/api/src/config/app-config.service.ts index 94bfa8ed..95ff5f28 100644 --- a/app/api/src/config/app-config.service.ts +++ b/app/api/src/config/app-config.service.ts @@ -8,6 +8,14 @@ import { SSMClient, GetParametersCommand, Parameter } from '@aws-sdk/client-ssm' type ParameterWithNameAndValue = Required>; +export type UmConfig = { + url: string; + auth0Domain: string; + clientId: string; + clientSecret: string; + audience: string; +}; + /** * AppConfigService is a wrapper on the @nestjs/config package's * ConfigService. It allows us to define custom getters, including @@ -139,9 +147,7 @@ export class AppConfigService { const envLabel = this.get('ENVLABEL'); if (!envLabel) { - throw new Error( - 'ENVLABEL must be set in order to retrieve EDU connection info' - ); + throw new Error('ENVLABEL must be set in order to retrieve EDU connection info'); } const secretName = `${envLabel}-edu-connection-info-${partnerId}`; let secret: string | Record; @@ -164,8 +170,7 @@ export class AppConfigService { if (typeof secret !== 'object') { return null; } - const { username, account, database, schema, privateKey, warehouse, role } = - secret; + const { username, account, database, schema, privateKey, warehouse, role } = secret; if (!username || !account || !database || !schema || !privateKey) { return null; } @@ -210,6 +215,35 @@ export class AppConfigService { return { issuerUrl, audience }; } + async umConfig(): Promise { + const configSecret = this.get('UM_CONFIG_SECRET'); + if (configSecret) { + const secret = await this.fetchAWSSecret(configSecret); + if (typeof secret !== 'object') { + throw new Error(`Value for AWS secret ${configSecret} must be an object`); + } + return { + url: secret['url'], + auth0Domain: secret['auth0Domain'], + clientId: secret['clientId'], + clientSecret: secret['clientSecret'], + audience: secret['audience'], + }; + } + + const url = this.get('UM_URL'); + const auth0Domain = this.get('UM_AUTH0_DOMAIN'); + const clientId = this.get('UM_CLIENT_ID'); + const clientSecret = this.get('UM_CLIENT_SECRET'); + const audience = this.get('UM_AUDIENCE'); + + if (!url || !auth0Domain || !clientId || !clientSecret || !audience) { + return null; + } + + return { url, auth0Domain, clientId, clientSecret, audience }; + } + // Jobs whose input files total at least this many bytes run on the large // ECS task instead of medium. Null when unset or unparseable; the caller // decides the default. diff --git a/app/api/src/config/env-vars.interface.ts b/app/api/src/config/env-vars.interface.ts index 1902e2ee..03bbd421 100644 --- a/app/api/src/config/env-vars.interface.ts +++ b/app/api/src/config/env-vars.interface.ts @@ -43,4 +43,13 @@ export interface IEnvironmentVariables { OAUTH2_ISSUER?: string; // token issuer for external API OAUTH2_AUDIENCE?: string; // token audience for external API + + // for tenant sync + UM_SYNC_CRON?: string; // e.g. "0 0 * * *" — unset = sync disabled + UM_URL?: string; + UM_AUTH0_DOMAIN?: string; + UM_CLIENT_ID?: string; + UM_CLIENT_SECRET?: string; + UM_AUDIENCE?: string; + UM_CONFIG_SECRET?: string; // deployed envs: name of AWS secret containing UM credentials } diff --git a/app/api/src/database/postgrator/migrations/032.do.partner-tenant-sync.sql b/app/api/src/database/postgrator/migrations/032.do.partner-tenant-sync.sql new file mode 100644 index 00000000..13bbb060 --- /dev/null +++ b/app/api/src/database/postgrator/migrations/032.do.partner-tenant-sync.sql @@ -0,0 +1,11 @@ +CREATE TYPE sync_manager AS ENUM ('user_management_sync', 'tx_sync'); + +-- Add soft-delete and management columns to partner table +ALTER TABLE public.partner + ADD COLUMN deleted_on TIMESTAMP, + ADD COLUMN managed_by sync_manager; + +-- Add soft-delete, management, and is_global to tenant table +ALTER TABLE public.tenant + ADD COLUMN deleted_on TIMESTAMP, + ADD COLUMN is_global BOOLEAN NOT NULL DEFAULT false; diff --git a/app/api/src/database/postgrator/migrations/032.undo.partner-tenant-sync.sql b/app/api/src/database/postgrator/migrations/032.undo.partner-tenant-sync.sql new file mode 100644 index 00000000..7c3c523d --- /dev/null +++ b/app/api/src/database/postgrator/migrations/032.undo.partner-tenant-sync.sql @@ -0,0 +1,9 @@ +ALTER TABLE public.partner + DROP COLUMN managed_by, + DROP COLUMN deleted_on; + +ALTER TABLE public.tenant + DROP COLUMN is_global, + DROP COLUMN deleted_on; + +DROP TYPE sync_manager; diff --git a/app/api/src/database/prisma/schema.prisma b/app/api/src/database/prisma/schema.prisma index 0cabb457..e7cb8697 100644 --- a/app/api/src/database/prisma/schema.prisma +++ b/app/api/src/database/prisma/schema.prisma @@ -31,6 +31,8 @@ model Partner { descriptorNamespace String? @map("descriptor_namespace") @db.VarChar idpId String? @map("idp_id") @db.VarChar crossYearMatchingEnabled Boolean @default(false) @map("cross_year_matching_enabled") + deletedOn DateTime? @map("deleted_on") @db.Timestamp(6) + managedBy SyncManager? @map("managed_by") customDescriptorMapping CustomDescriptorMapping[] userPartnerCreatedByIdTouser User? @relation("partner_created_by_idTouser", fields: [createdById], references: [id], onUpdate: NoAction) identityProvider IdentityProvider? @relation(fields: [idpId], references: [id], onUpdate: NoAction) @@ -49,6 +51,8 @@ model Tenant { createdOn DateTime @default(now()) @map("created_on") @db.Timestamp(6) modifiedById Int? @map("modified_by_id") modifiedOn DateTime @default(now()) @map("modified_on") @db.Timestamp(6) + deletedOn DateTime? @map("deleted_on") @db.Timestamp(6) + isGlobal Boolean @default(false) @map("is_global") job Job[] odsConfig OdsConfig[] userTenantCreatedByIdTouser User? @relation("tenant_created_by_idTouser", fields: [createdById], references: [id], onUpdate: NoAction) @@ -489,3 +493,10 @@ enum FileStorageProtocol { @@map("file_storage_protocol") } + +enum SyncManager { + user_management_sync + tx_sync + + @@map("sync_manager") +} diff --git a/app/api/src/partner-sync/partner-sync.module.ts b/app/api/src/partner-sync/partner-sync.module.ts new file mode 100644 index 00000000..b738adc6 --- /dev/null +++ b/app/api/src/partner-sync/partner-sync.module.ts @@ -0,0 +1,11 @@ +import { Module } from '@nestjs/common'; +import { PgBossService } from '../pg-boss/pg-boss.service'; +import { UmSyncHandler } from './user-management/um-sync.handler'; + +@Module({ + providers: [ + PgBossService, + UmSyncHandler, + ], +}) +export class PartnerSyncModule {} diff --git a/app/api/src/partner-sync/user-management/um-sync.handler.ts b/app/api/src/partner-sync/user-management/um-sync.handler.ts new file mode 100644 index 00000000..0a868460 --- /dev/null +++ b/app/api/src/partner-sync/user-management/um-sync.handler.ts @@ -0,0 +1,313 @@ +import { Inject, Injectable, Logger, OnModuleInit } from '@nestjs/common'; +import { PrismaClient } from '@prisma/client'; +import { AppConfigService, UmConfig } from '../../config/app-config.service'; +import { PRISMA_ANONYMOUS } from '../../database/database.service'; +import { PgBossService } from '../../pg-boss/pg-boss.service'; +import { TenantUpsert, UserManagementPartner, UserManagementTenant } from './um-sync.types'; + +@Injectable() +export class UmSyncHandler implements OnModuleInit { + readonly sourceKey = 'user_management_sync'; + + private readonly logger = new Logger(UmSyncHandler.name); + + private umToken: string | null = null; + private umTokenExpiration: Date | null = null; + + constructor( + private readonly appConfig: AppConfigService, + private readonly pgBoss: PgBossService, + @Inject(PRISMA_ANONYMOUS) private readonly prisma: PrismaClient + ) {} + + async onModuleInit() { + try { + const config = await this.appConfig.umConfig(); + const syncCron = this.appConfig.get('UM_SYNC_CRON') ?? '0 2 * * *'; + + const boss = await this.pgBoss.boss; + + if (!config) { + this.logger.warn(`${this.sourceKey} config not set — unscheduling any existing job`); + await boss.unschedule(this.sourceKey); + return; + } + + await boss.createQueue(this.sourceKey); + await boss.schedule(this.sourceKey, syncCron, null, { + singletonKey: this.sourceKey, + }); + await boss.work(this.sourceKey, () => this.sync()); + this.logger.log(`${this.sourceKey} sync scheduled: ${syncCron}`); + } catch (err) { + this.logger.error(`Failed to schedule ${this.sourceKey} sync`, err); + } + } + + async sync(): Promise { + const umConfig = await this.appConfig.umConfig(); + if (!umConfig) { + this.logger.warn('UM sync config not set — skipping sync'); + return; + } + + this.logger.log('Starting UM sync'); + + const allExistingParnters = await this.prisma.partner.findMany({ + include: { tenant: true }, + }); + const existingById = new Map(allExistingParnters.map((p) => [p.id, p])); + + // --- Partner sync --- + const partnersResult = await this.umRequest(umConfig, 'partners'); + + if (partnersResult.status !== 'success') { + this.logger.error('Failed to fetch partners from UM — aborting sync'); + return; + } + const apiPartnerCodes = new Set(partnersResult.data.map((p) => p.partnerCode)); + + const partnerIdsToCreate = partnersResult.data + .filter((p) => !existingById.has(p.partnerCode)) + .map((p) => p.partnerCode); + + const partnerIdsToDelete: string[] = allExistingParnters + .filter((p) => !apiPartnerCodes.has(p.id)) + .filter((p) => p.managedBy === this.sourceKey && !p.deletedOn) + .map((p) => p.id); + + const partnerIdsToUndelete: string[] = partnersResult.data + .filter((p) => existingById.has(p.partnerCode)) + .filter( + (p) => + existingById.get(p.partnerCode)?.deletedOn && + existingById.get(p.partnerCode)?.managedBy === this.sourceKey + ) + .map((p) => p.partnerCode); + + const deletingPartnerIds = new Set(partnerIdsToDelete); + + // --- Tenant sync --- + const tenantsToCreate: TenantUpsert[] = []; + const tenantsToUndelete: TenantUpsert[] = []; + const tenantsToUpdate: TenantUpsert[] = []; + + const tenantsToDelete: { code: string; partnerId: string }[] = partnerIdsToDelete.flatMap( + (partnerId) => + (existingById.get(partnerId)?.tenant ?? []) + .filter((tenant) => !tenant.deletedOn) + .map((tenant) => ({ code: tenant.code, partnerId: tenant.partnerId })) + ); + + const partnerIdsForTenantSync = [ + ...allExistingParnters + .filter((p) => p.managedBy === this.sourceKey && !deletingPartnerIds.has(p.id)) + .map((p) => p.id), + ...partnerIdsToCreate, + ]; + + const tenantFetchResults = await Promise.all( + partnerIdsForTenantSync.map(async (partnerId) => ({ + partnerId, + result: await this.umRequest(umConfig, 'tenants', { + partnerCode: partnerId, + }), + })) + ); + + for (const { partnerId, result } of tenantFetchResults) { + if (result.status !== 'success') { + this.logger.error(`Failed to fetch tenants for partner ${partnerId} from UM`); + continue; + } + + const tenantMap = new Map( + (existingById.get(partnerId)?.tenant ?? []).map((t) => [t.code, t]) + ); + const umTenantCodes = new Set(result.data.map((t) => t.tenantCode)); + + for (const apiTenant of result.data) { + const existing = tenantMap.get(apiTenant.tenantCode); + const isGlobal = apiTenant.isGlobal; + if (!existing) { + tenantsToCreate.push({ code: apiTenant.tenantCode, partnerId, isGlobal }); + } else if (existing.deletedOn) { + tenantsToUndelete.push({ + code: existing.code, + partnerId: existing.partnerId, + isGlobal, + }); + } else if (existing.isGlobal !== isGlobal) { + tenantsToUpdate.push({ + code: existing.code, + partnerId: existing.partnerId, + isGlobal, + }); + } + } + + for (const [code, tenant] of tenantMap) { + if (!umTenantCodes.has(code) && !tenant.deletedOn) { + tenantsToDelete.push({ code: tenant.code, partnerId: tenant.partnerId }); + } + } + } + + await this.prisma.$transaction(async (tx) => { + let partnersCreated = 0; + let partnersDeleted = 0; + let partnersUndeleted = 0; + let tenantsCreated = 0; + let tenantsDeleted = 0; + let tenantsUndeleted = 0; + + if (partnerIdsToCreate.length) { + const r = await tx.partner.createMany({ + data: partnerIdsToCreate.map((id) => ({ + id, + name: id, + managedBy: 'user_management_sync', + })), + }); + partnersCreated = r.count; + } + + if (partnerIdsToDelete.length) { + const r = await tx.partner.updateMany({ + where: { id: { in: partnerIdsToDelete }, managedBy: this.sourceKey }, + data: { deletedOn: new Date() }, + }); + partnersDeleted = r.count; + } + + if (partnerIdsToUndelete.length) { + const r = await tx.partner.updateMany({ + where: { id: { in: partnerIdsToUndelete }, managedBy: this.sourceKey }, + data: { deletedOn: null }, + }); + partnersUndeleted = r.count; + } + + if (tenantsToCreate.length) { + const r = await tx.tenant.createMany({ + data: tenantsToCreate.map((t) => ({ ...t })), + }); + tenantsCreated = r.count; + } + + if (tenantsToDelete.length) { + for (const { partnerId, code } of tenantsToDelete) { + await tx.tenant.update({ + where: { code_partnerId: { code, partnerId }, partner: {managedBy: this.sourceKey} }, + data: { deletedOn: new Date() }, + }); + tenantsDeleted++; + } + } + + for (const { code, partnerId, isGlobal } of tenantsToUndelete) { + await tx.tenant.update({ + where: { code_partnerId: { code, partnerId }, partner: {managedBy: this.sourceKey} }, + data: { deletedOn: null, isGlobal }, + }); + tenantsUndeleted++; + } + + for (const { code, partnerId, isGlobal } of tenantsToUpdate) { + await tx.tenant.update({ + where: { code_partnerId: { code, partnerId }, partner: {managedBy: this.sourceKey} }, + data: { isGlobal }, + }); + } + + this.logger.log( + `UM sync: partners +${partnersCreated} -${partnersDeleted} ↑${partnersUndeleted} | ` + + `tenants +${tenantsCreated} -${tenantsDeleted} ↑${tenantsUndeleted}` + ); + }); + + this.logger.log('UM sync complete'); + } + + private async getToken( + umConfig: UmConfig + ): Promise<{ status: 'success' } | { status: 'failure' }> { + try { + const response = await fetch(`https://${umConfig.auth0Domain}/oauth/token`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + grant_type: 'client_credentials', + client_id: umConfig.clientId, + client_secret: umConfig.clientSecret, + audience: umConfig.audience, + }), + }); + + if (!response.ok) { + this.logger.error(`Failed to get UM token: ${response.status} ${response.statusText}`); + return { status: 'failure' }; + } + + const data = (await response.json()) as { access_token: string; expires_in: number }; + this.umToken = data.access_token; + this.umTokenExpiration = new Date(Date.now() + data.expires_in * 1000); + return { status: 'success' }; + } catch (error) { + this.logger.error('Error fetching UM token', error); + return { status: 'failure' }; + } + } + + private async ensureToken( + umConfig: UmConfig + ): Promise<{ status: 'success' } | { status: 'failure' }> { + // add 30 second buffer to avoid token expiration during request + const expirationBufferMs = 30000; + if ( + !this.umToken || + !this.umTokenExpiration || + this.umTokenExpiration.getTime() - expirationBufferMs < Date.now() + ) { + return this.getToken(umConfig); + } + return { status: 'success' }; + } + + private async umRequest( + umConfig: UmConfig, + path: string, + searchParams?: Record + ): Promise<{ status: 'success'; data: T } | { status: 'failure' }> { + const tokenResult = await this.ensureToken(umConfig); + if (tokenResult.status === 'failure') { + return { status: 'failure' }; + } + + const url = new URL(`${umConfig.url}/api/v1/${path}`); + if (searchParams) { + Object.entries(searchParams).forEach(([key, value]) => url.searchParams.append(key, value)); + } + + let response = await fetch(url.toString(), { + headers: { Authorization: `Bearer ${this.umToken}` }, + }); + + if (response.status === 401) { + const refreshResult = await this.getToken(umConfig); + if (refreshResult.status === 'failure') { + return { status: 'failure' }; + } + response = await fetch(url.toString(), { + headers: { Authorization: `Bearer ${this.umToken}` }, + }); + } + + if (response.ok) { + return { status: 'success', data: (await response.json()) as T }; + } + + this.logger.warn(`UM request to ${url} failed with ${response.status}: ${response.statusText}`); + return { status: 'failure' }; + } +} diff --git a/app/api/src/partner-sync/user-management/um-sync.types.ts b/app/api/src/partner-sync/user-management/um-sync.types.ts new file mode 100644 index 00000000..c3b6ff6a --- /dev/null +++ b/app/api/src/partner-sync/user-management/um-sync.types.ts @@ -0,0 +1,16 @@ +export type UserManagementTenant = { + partnerCode: string; + tenantCode: string; + displayName: string; + isGlobal: boolean; +}; + +export type UserManagementPartner = { + partnerCode: string; +}; + +export type TenantUpsert = { + code: string; + partnerId: string; + isGlobal: boolean; +}; diff --git a/app/api/src/pg-boss/pg-boss.service.ts b/app/api/src/pg-boss/pg-boss.service.ts new file mode 100644 index 00000000..732799e5 --- /dev/null +++ b/app/api/src/pg-boss/pg-boss.service.ts @@ -0,0 +1,32 @@ +import { Injectable, OnModuleDestroy, OnModuleInit } from '@nestjs/common'; +import { PgBoss } from 'pg-boss'; +import { AppConfigService } from '../config/app-config.service'; + +@Injectable() +export class PgBossService implements OnModuleInit, OnModuleDestroy { + private _boss: PgBoss | undefined; + private _ready: Promise | undefined; + + constructor(private readonly appConfig: AppConfigService) {} + + // Resolves once the underlying PgBoss instance has started (migrations run, ready for DB access). + get boss(): Promise { + this._ready ??= this.init(); + return this._ready; + } + + private async init(): Promise { + const pgConfig = await this.appConfig.postgresPoolConfig(); + this._boss = new PgBoss(pgConfig); + return this._boss.start(); + } + + async onModuleInit() { + this._ready ??= this.init(); + await this._ready; + } + + async onModuleDestroy() { + await this._boss?.stop(); + } +} diff --git a/app/fe/vite.config.mts b/app/fe/vite.config.mts index e5ce0881..c3c9729d 100644 --- a/app/fe/vite.config.mts +++ b/app/fe/vite.config.mts @@ -1,4 +1,5 @@ /// +import { resolve } from 'node:path'; import { nxViteTsPaths } from '@nx/vite/plugins/nx-tsconfig-paths.plugin'; import { TanStackRouterVite } from '@tanstack/router-plugin/vite'; import react from '@vitejs/plugin-react'; @@ -21,8 +22,8 @@ export default defineConfig({ plugins: [ TanStackRouterVite({ - routesDirectory: './src/app/routes/', - generatedRouteTree: './src/app/routeTree.gen.ts', + routesDirectory: resolve(__dirname, './src/app/routes/'), // absolute paths are a workaround for the issue fixed in https://github.com/TanStack/router/pull/5963 + generatedRouteTree: resolve(__dirname, './src/app/routeTree.gen.ts'), }), react(), nxViteTsPaths(), diff --git a/app/models/src/dtos/tenant.dto.ts b/app/models/src/dtos/tenant.dto.ts index c86c71bf..bce8e5c2 100644 --- a/app/models/src/dtos/tenant.dto.ts +++ b/app/models/src/dtos/tenant.dto.ts @@ -9,6 +9,12 @@ export class GetTenantDto extends DtoGetBase implements Tenant { @Expose() partnerId: string; + + @Expose() + deletedOn: Date | null; + + @Expose() + isGlobal: boolean; } export const toGetTenantDto = makeSerializer(GetTenantDto); diff --git a/app/package-lock.json b/app/package-lock.json index 53c765da..eb11866c 100644 --- a/app/package-lock.json +++ b/app/package-lock.json @@ -59,6 +59,7 @@ "openid-client": "^5.6.5", "passport": "^0.7.0", "pg": "^8.12.0", + "pg-boss": "^12.18.2", "re-resizable": "^6.9.17", "react": "18.3.1", "react-dom": "18.3.1", @@ -3991,12 +3992,14 @@ "version": "2.2.2", "resolved": "https://registry.npmjs.org/@chakra-ui/anatomy/-/anatomy-2.2.2.tgz", "integrity": "sha512-MV6D4VLRIHr4PkW4zMyqfrNS1mPlCTiCXwvYGtDFQYr+xHFfonhAuf9WjsSc0nyp2m0OdkSLnzmVKkZFLo25Tg==", + "license": "MIT", "peer": true }, "node_modules/@chakra-ui/breakpoint-utils": { "version": "2.0.8", "resolved": "https://registry.npmjs.org/@chakra-ui/breakpoint-utils/-/breakpoint-utils-2.0.8.tgz", "integrity": "sha512-Pq32MlEX9fwb5j5xx8s18zJMARNHlQZH2VH1RZgfgRDpp7DcEgtRW5AInfN5CfqdHLO1dGxA7I3MqEuL5JnIsA==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/shared-utils": "2.0.5" @@ -4006,6 +4009,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/clickable/-/clickable-2.1.0.tgz", "integrity": "sha512-flRA/ClPUGPYabu+/GLREZVZr9j2uyyazCAUHAdrTUEdDYCr31SVGhgh7dgKdtq23bOvAQJpIJjw/0Bs0WvbXw==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-use-merge-refs": "2.1.0", @@ -4019,6 +4023,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/@chakra-ui/color-mode/-/color-mode-2.2.0.tgz", "integrity": "sha512-niTEA8PALtMWRI9wJ4LL0CSBDo8NBfLNp4GD6/0hstcm3IlbBHTVKxN6HwSaoNYfphDQLxCjT4yG+0BJA5tFpg==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-use-safe-layout-effect": "2.1.0" @@ -4031,6 +4036,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/descendant/-/descendant-3.1.0.tgz", "integrity": "sha512-VxCIAir08g5w27klLyi7PVo8BxhW4tgU/lxQyujkmi4zx7hT9ZdrcQLAted/dAa+aSIZ14S1oV0Q9lGjsAdxUQ==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-context": "2.1.0", @@ -4044,12 +4050,14 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/dom-utils/-/dom-utils-2.1.0.tgz", "integrity": "sha512-ZmF2qRa1QZ0CMLU8M1zCfmw29DmPNtfjR9iTo74U5FPr3i1aoAh7fbJ4qAlZ197Xw9eAW28tvzQuoVWeL5C7fQ==", + "license": "MIT", "peer": true }, "node_modules/@chakra-ui/form-control": { "version": "2.2.0", "resolved": "https://registry.npmjs.org/@chakra-ui/form-control/-/form-control-2.2.0.tgz", "integrity": "sha512-wehLC1t4fafCVJ2RvJQT2jyqsAwX7KymmiGqBu7nQoQz8ApTkGABWpo/QwDh3F/dBLrouHDoOvGmYTqft3Mirw==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/icon": "3.2.0", @@ -4063,15 +4071,6 @@ "react": ">=18" } }, - "node_modules/@chakra-ui/form-control/node_modules/@chakra-ui/react-types": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/@chakra-ui/react-types/-/react-types-2.0.7.tgz", - "integrity": "sha512-12zv2qIZ8EHwiytggtGvo4iLT0APris7T0qaAWqzpUGS0cdUtR8W+V1BJ5Ocq+7tA6dzQ/7+w5hmXih61TuhWQ==", - "peer": true, - "peerDependencies": { - "react": ">=18" - } - }, "node_modules/@chakra-ui/hooks": { "version": "2.4.5", "resolved": "https://registry.npmjs.org/@chakra-ui/hooks/-/hooks-2.4.5.tgz", @@ -4137,6 +4136,7 @@ "version": "2.3.1", "resolved": "https://registry.npmjs.org/@chakra-ui/layout/-/layout-2.3.1.tgz", "integrity": "sha512-nXuZ6WRbq0WdgnRgLw+QuxWAHuhDtVX8ElWqcTK+cSMFg/52eVP47czYBE5F35YhnoW2XBwfNoNgZ7+e8Z01Rg==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/breakpoint-utils": "2.0.8", @@ -4155,12 +4155,14 @@ "version": "2.0.5", "resolved": "https://registry.npmjs.org/@chakra-ui/lazy-utils/-/lazy-utils-2.0.5.tgz", "integrity": "sha512-UULqw7FBvcckQk2n3iPO56TMJvDsNv0FKZI6PlUNJVaGsPbsYxK/8IQ60vZgaTVPtVcjY6BE+y6zg8u9HOqpyg==", + "license": "MIT", "peer": true }, "node_modules/@chakra-ui/media-query": { "version": "3.3.0", "resolved": "https://registry.npmjs.org/@chakra-ui/media-query/-/media-query-3.3.0.tgz", "integrity": "sha512-IsTGgFLoICVoPRp9ykOgqmdMotJG0CnPsKvGQeSFOB/dZfIujdVb14TYxDU4+MURXry1MhJ7LzZhv+Ml7cr8/g==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/breakpoint-utils": "2.0.8", @@ -4176,6 +4178,7 @@ "version": "2.2.1", "resolved": "https://registry.npmjs.org/@chakra-ui/menu/-/menu-2.2.1.tgz", "integrity": "sha512-lJS7XEObzJxsOwWQh7yfG4H8FzFPRP5hVPN/CL+JzytEINCSBvsCDHrYPQGp7jzpCi8vnTqQQGQe0f8dwnXd2g==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/clickable": "2.1.0", @@ -4204,12 +4207,14 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/object-utils/-/object-utils-2.1.0.tgz", "integrity": "sha512-tgIZOgLHaoti5PYGPTwK3t/cqtcycW0owaiOXoZOcpwwX/vlVb+H1jFsQyWiiwQVPt9RkoSLtxzXamx+aHH+bQ==", + "license": "MIT", "peer": true }, "node_modules/@chakra-ui/popper": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/popper/-/popper-3.1.0.tgz", "integrity": "sha512-ciDdpdYbeFG7og6/6J8lkTFxsSvwTdMLFkpVylAF6VNC22jssiWfquj2eyD4rJnzkRFPvIWJq8hvbfhsm+AjSg==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-types": "2.0.7", @@ -4220,15 +4225,6 @@ "react": ">=18" } }, - "node_modules/@chakra-ui/popper/node_modules/@chakra-ui/react-types": { - "version": "2.0.7", - "resolved": "https://registry.npmjs.org/@chakra-ui/react-types/-/react-types-2.0.7.tgz", - "integrity": "sha512-12zv2qIZ8EHwiytggtGvo4iLT0APris7T0qaAWqzpUGS0cdUtR8W+V1BJ5Ocq+7tA6dzQ/7+w5hmXih61TuhWQ==", - "peer": true, - "peerDependencies": { - "react": ">=18" - } - }, "node_modules/@chakra-ui/react": { "version": "2.10.9", "resolved": "https://registry.npmjs.org/@chakra-ui/react/-/react-2.10.9.tgz", @@ -4258,6 +4254,7 @@ "version": "2.0.6", "resolved": "https://registry.npmjs.org/@chakra-ui/react-children-utils/-/react-children-utils-2.0.6.tgz", "integrity": "sha512-QVR2RC7QsOsbWwEnq9YduhpqSFnZGvjjGREV8ygKi8ADhXh93C8azLECCUVgRJF2Wc+So1fgxmjLcbZfY2VmBA==", + "license": "MIT", "peer": true, "peerDependencies": { "react": ">=18" @@ -4275,6 +4272,7 @@ "version": "3.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-env/-/react-env-3.1.0.tgz", "integrity": "sha512-Vr96GV2LNBth3+IKzr/rq1IcnkXv+MLmwjQH6C8BRtn3sNskgDFD5vLkVXcEhagzZMCh8FR3V/bzZPojBOyNhw==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-use-safe-layout-effect": "2.1.0" @@ -4284,9 +4282,10 @@ } }, "node_modules/@chakra-ui/react-types": { - "version": "2.0.6", - "resolved": "https://registry.npmjs.org/@chakra-ui/react-types/-/react-types-2.0.6.tgz", - "integrity": "sha512-aAq/nl//PneEfeaDb94zwfXor4OP/d5kc6dEXOZB2HJgCt3hu2+F/1u1QpPLPPTys5xexkQojuZQLnnD9lmQFw==", + "version": "2.0.7", + "resolved": "https://registry.npmjs.org/@chakra-ui/react-types/-/react-types-2.0.7.tgz", + "integrity": "sha512-12zv2qIZ8EHwiytggtGvo4iLT0APris7T0qaAWqzpUGS0cdUtR8W+V1BJ5Ocq+7tA6dzQ/7+w5hmXih61TuhWQ==", + "license": "MIT", "peerDependencies": { "react": ">=18" } @@ -4295,6 +4294,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-animation-state/-/react-use-animation-state-2.1.0.tgz", "integrity": "sha512-CFZkQU3gmDBwhqy0vC1ryf90BVHxVN8cTLpSyCpdmExUEtSEInSCGMydj2fvn7QXsz/za8JNdO2xxgJwxpLMtg==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/dom-utils": "2.1.0", @@ -4308,6 +4308,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-callback-ref/-/react-use-callback-ref-2.1.0.tgz", "integrity": "sha512-efnJrBtGDa4YaxDzDE90EnKD3Vkh5a1t3w7PhnRQmsphLy3g2UieasoKTlT2Hn118TwDjIv5ZjHJW6HbzXA9wQ==", + "license": "MIT", "peer": true, "peerDependencies": { "react": ">=18" @@ -4317,6 +4318,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-controllable-state/-/react-use-controllable-state-2.1.0.tgz", "integrity": "sha512-QR/8fKNokxZUs4PfxjXuwl0fj/d71WPrmLJvEpCTkHjnzu7LnYvzoe2wB867IdooQJL0G1zBxl0Dq+6W1P3jpg==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-use-callback-ref": "2.1.0" @@ -4329,6 +4331,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-disclosure/-/react-use-disclosure-2.1.0.tgz", "integrity": "sha512-Ax4pmxA9LBGMyEZJhhUZobg9C0t3qFE4jVF1tGBsrLDcdBeLR9fwOogIPY9Hf0/wqSlAryAimICbr5hkpa5GSw==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-use-callback-ref": "2.1.0" @@ -4341,6 +4344,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-event-listener/-/react-use-event-listener-2.1.0.tgz", "integrity": "sha512-U5greryDLS8ISP69DKDsYcsXRtAdnTQT+jjIlRYZ49K/XhUR/AqVZCK5BkR1spTDmO9H8SPhgeNKI70ODuDU/Q==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-use-callback-ref": "2.1.0" @@ -4353,6 +4357,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-focus-effect/-/react-use-focus-effect-2.1.0.tgz", "integrity": "sha512-xzVboNy7J64xveLcxTIJ3jv+lUJKDwRM7Szwn9tNzUIPD94O3qwjV7DDCUzN2490nSYDF4OBMt/wuDBtaR3kUQ==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/dom-utils": "2.1.0", @@ -4368,6 +4373,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-merge-refs/-/react-use-merge-refs-2.1.0.tgz", "integrity": "sha512-lERa6AWF1cjEtWSGjxWTaSMvneccnAVH4V4ozh8SYiN9fSPZLlSG3kNxfNzdFvMEhM7dnP60vynF7WjGdTgQbQ==", + "license": "MIT", "peer": true, "peerDependencies": { "react": ">=18" @@ -4377,6 +4383,7 @@ "version": "2.2.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-outside-click/-/react-use-outside-click-2.2.0.tgz", "integrity": "sha512-PNX+s/JEaMneijbgAM4iFL+f3m1ga9+6QK0E5Yh4s8KZJQ/bLwZzdhMz8J/+mL+XEXQ5J0N8ivZN28B82N1kNw==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/react-use-callback-ref": "2.1.0" @@ -4389,6 +4396,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-safe-layout-effect/-/react-use-safe-layout-effect-2.1.0.tgz", "integrity": "sha512-Knbrrx/bcPwVS1TorFdzrK/zWA8yuU/eaXDkNj24IrKoRlQrSBFarcgAEzlCHtzuhufP3OULPkELTzz91b0tCw==", + "license": "MIT", "peer": true, "peerDependencies": { "react": ">=18" @@ -4398,6 +4406,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/react-use-update-effect/-/react-use-update-effect-2.1.0.tgz", "integrity": "sha512-ND4Q23tETaR2Qd3zwCKYOOS1dfssojPLJMLvUtUbW5M9uW1ejYWgGUobeAiOVfSplownG8QYMmHTP86p/v0lbA==", + "license": "MIT", "peer": true, "peerDependencies": { "react": ">=18" @@ -4407,6 +4416,7 @@ "version": "2.0.12", "resolved": "https://registry.npmjs.org/@chakra-ui/react-utils/-/react-utils-2.0.12.tgz", "integrity": "sha512-GbSfVb283+YA3kA8w8xWmzbjNWk14uhNpntnipHCftBibl0lxtQ9YqMFQLwuFOO0U2gYVocszqqDWX+XNKq9hw==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/utils": "2.0.15" @@ -4490,6 +4500,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/spinner/-/spinner-2.1.0.tgz", "integrity": "sha512-hczbnoXt+MMv/d3gE+hjQhmkzLiKuoTo42YhUG7Bs9OSv2lg1fZHW1fGNRFP3wTi6OIbD044U1P9HK+AOgFH3g==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/shared-utils": "2.0.5" @@ -4525,6 +4536,7 @@ "version": "2.9.2", "resolved": "https://registry.npmjs.org/@chakra-ui/styled-system/-/styled-system-2.9.2.tgz", "integrity": "sha512-To/Z92oHpIE+4nk11uVMWqo2GGRS86coeMmjxtpnErmWRdLcp1WVCVRAvn+ZwpLiNR+reWFr2FFqJRsREuZdAg==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/shared-utils": "2.0.5", @@ -4536,6 +4548,7 @@ "version": "2.6.2", "resolved": "https://registry.npmjs.org/@chakra-ui/system/-/system-2.6.2.tgz", "integrity": "sha512-EGtpoEjLrUu4W1fHD+a62XR+hzC5YfsWm+6lO0Kybcga3yYEij9beegO0jZgug27V+Rf7vns95VPVP6mFd/DEQ==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/color-mode": "2.2.0", @@ -4556,6 +4569,7 @@ "version": "3.3.1", "resolved": "https://registry.npmjs.org/@chakra-ui/theme/-/theme-3.3.1.tgz", "integrity": "sha512-Hft/VaT8GYnItGCBbgWd75ICrIrIFrR7lVOhV/dQnqtfGqsVDlrztbSErvMkoPKt0UgAkd9/o44jmZ6X4U2nZQ==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/anatomy": "2.2.2", @@ -4570,6 +4584,7 @@ "version": "2.1.2", "resolved": "https://registry.npmjs.org/@chakra-ui/theme-tools/-/theme-tools-2.1.2.tgz", "integrity": "sha512-Qdj8ajF9kxY4gLrq7gA+Azp8CtFHGO9tWMN2wfF9aQNgG9AuMhPrUzMq9AMQ0MXiYcgNq/FD3eegB43nHVmXVA==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/anatomy": "2.2.2", @@ -4584,6 +4599,7 @@ "version": "2.0.21", "resolved": "https://registry.npmjs.org/@chakra-ui/theme-utils/-/theme-utils-2.0.21.tgz", "integrity": "sha512-FjH5LJbT794r0+VSCXB3lT4aubI24bLLRWB+CuRKHijRvsOg717bRdUN/N1fEmEpFnRVrbewttWh/OQs0EWpWw==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/shared-utils": "2.0.5", @@ -4596,6 +4612,7 @@ "version": "2.1.0", "resolved": "https://registry.npmjs.org/@chakra-ui/transition/-/transition-2.1.0.tgz", "integrity": "sha512-orkT6T/Dt+/+kVwJNy7zwJ+U2xAZ3EU7M3XCs45RBvUnZDr/u9vdmaM/3D/rOpmQJWgQBwKPJleUXrYWUagEDQ==", + "license": "MIT", "peer": true, "dependencies": { "@chakra-ui/shared-utils": "2.0.5" @@ -4609,6 +4626,7 @@ "version": "2.0.15", "resolved": "https://registry.npmjs.org/@chakra-ui/utils/-/utils-2.0.15.tgz", "integrity": "sha512-El4+jL0WSaYYs+rJbuYFDbjmfCcfGDmRY95GO4xwzit6YAPZBLcR65rOEwLps+XWluZTy1xdMrusg/hW0c1aAA==", + "license": "MIT", "peer": true, "dependencies": { "@types/lodash.mergewith": "4.6.7", @@ -14866,6 +14884,7 @@ "version": "4.6.7", "resolved": "https://registry.npmjs.org/@types/lodash.mergewith/-/lodash.mergewith-4.6.7.tgz", "integrity": "sha512-3m+lkO5CLRRYU0fhGRp7zbsGi6+BZj0uTVSwvcKU+nSlhjA9/QRNfuSGnD2mX6hQA7ZbmcCkzk5h4ZYGOtk14A==", + "license": "MIT", "peer": true, "dependencies": { "@types/lodash": "*" @@ -18821,6 +18840,17 @@ "resolved": "https://registry.npmjs.org/create-require/-/create-require-1.1.1.tgz", "integrity": "sha512-dcKFX3jn0MpIaXjisoRvexIJVEKzaq7z2rZKxf+MSr9TkdmHmsU4m2lcLojrj/FHl8mk5VxMmYA+ftRkP/3oKQ==" }, + "node_modules/cron-parser": { + "version": "5.5.0", + "resolved": "https://registry.npmjs.org/cron-parser/-/cron-parser-5.5.0.tgz", + "integrity": "sha512-oML4lKUXxizYswqmxuOCpgFS8BNUJpIu6k/2HVHyaL8Ynnf3wdf9tkns0yRdJLSIjkJ+b0DXHMZEHGpMwjnPww==", + "dependencies": { + "luxon": "^3.7.1" + }, + "engines": { + "node": ">=18" + } + }, "node_modules/cross-spawn": { "version": "7.0.3", "resolved": "https://registry.npmjs.org/cross-spawn/-/cross-spawn-7.0.3.tgz", @@ -18847,6 +18877,7 @@ "version": "1.2.1", "resolved": "https://registry.npmjs.org/css-box-model/-/css-box-model-1.2.1.tgz", "integrity": "sha512-a7Vr4Q/kd/aw96bnJG332W9V9LkJO69JRcaCYDUqjp6/z0w6VcZjgAcTbgFxEPfBgdnAwlh3iwu+hLopa+flJw==", + "license": "MIT", "peer": true, "dependencies": { "tiny-invariant": "^1.0.6" @@ -26395,6 +26426,14 @@ "yallist": "^3.0.2" } }, + "node_modules/luxon": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/luxon/-/luxon-3.7.2.tgz", + "integrity": "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew==", + "engines": { + "node": ">=12" + } + }, "node_modules/magic-string": { "version": "0.30.10", "resolved": "https://registry.npmjs.org/magic-string/-/magic-string-0.30.10.tgz", @@ -27206,6 +27245,17 @@ "resolved": "https://registry.npmjs.org/node-releases/-/node-releases-2.0.14.tgz", "integrity": "sha512-y10wOWt8yZpqXmOgRo77WaHEmhYQYGNA6y421PKsKYWEK8aW+cqAphborZDhqfyKrbZEN92CN1X2KbafY2s7Yw==" }, + "node_modules/non-error": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/non-error/-/non-error-0.1.0.tgz", + "integrity": "sha512-TMB1uHiGsHRGv1uYclfhivcnf0/PdFp2pNqRxXjncaAsjYMoisaQJI+SSZCqRq+VliwRTC8tsMQfmrWjDMhkPQ==", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/normalize-package-data": { "version": "2.5.0", "resolved": "https://registry.npmjs.org/normalize-package-data/-/normalize-package-data-2.5.0.tgz", @@ -28349,21 +28399,21 @@ "dev": true }, "node_modules/pg": { - "version": "8.12.0", - "resolved": "https://registry.npmjs.org/pg/-/pg-8.12.0.tgz", - "integrity": "sha512-A+LHUSnwnxrnL/tZ+OLfqR1SxLN3c/pgDztZ47Rpbsd4jUytsTtwQo/TLPRzPJMp/1pbhYVhH9cuSZLAajNfjQ==", + "version": "8.21.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.21.0.tgz", + "integrity": "sha512-AUP1EYJuHraQGsVoCQVIcM7TEJVGtDzxWtGFZd8rds9d+CCXlU5Js1rYgfLNvxy9iJrpHjGrRjoi/3BT9fRyiA==", "dependencies": { - "pg-connection-string": "^2.6.4", - "pg-pool": "^3.6.2", - "pg-protocol": "^1.6.1", - "pg-types": "^2.1.0", - "pgpass": "1.x" + "pg-connection-string": "^2.13.0", + "pg-pool": "^3.14.0", + "pg-protocol": "^1.14.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" }, "engines": { - "node": ">= 8.0.0" + "node": ">= 16.0.0" }, "optionalDependencies": { - "pg-cloudflare": "^1.1.1" + "pg-cloudflare": "^1.4.0" }, "peerDependencies": { "pg-native": ">=3.0.1" @@ -28374,16 +28424,33 @@ } } }, + "node_modules/pg-boss": { + "version": "12.18.2", + "resolved": "https://registry.npmjs.org/pg-boss/-/pg-boss-12.18.2.tgz", + "integrity": "sha512-06kXeWvVWY+BUNsOt2me1okg6NXx2DBnAQHTurA9jtrvbAO9qUOSE3/0ERERQDrokI+FREFM2Twha+JbrFT/8Q==", + "license": "MIT", + "dependencies": { + "cron-parser": "^5.5.0", + "pg": "^8.20.0", + "serialize-error": "^13.0.1" + }, + "bin": { + "pg-boss": "dist/cli.js" + }, + "engines": { + "node": ">=22.12.0" + } + }, "node_modules/pg-cloudflare": { - "version": "1.1.1", - "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.1.1.tgz", - "integrity": "sha512-xWPagP/4B6BgFO+EKz3JONXv3YDgvkbVrGw2mTo3D6tVDQRh1e7cqVGvyR3BE+eQgAvx1XhW/iEASj4/jCWl3Q==", + "version": "1.4.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.4.0.tgz", + "integrity": "sha512-Vo7z/6rrQYxpNRylp4Tlob2elzbh+N/MOQbxFVWCxS7oEx6jF53GTJFxK2WWpKuBRkmiin4Mt+xofFDjx09R0A==", "optional": true }, "node_modules/pg-connection-string": { - "version": "2.6.4", - "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.6.4.tgz", - "integrity": "sha512-v+Z7W/0EO707aNMaAEfiGnGL9sxxumwLl2fJvCQtMn9Fxsg+lPpPkdcyBSv/KFgpGdYkMfn+EI1Or2EHjpgLCA==" + "version": "2.13.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.13.0.tgz", + "integrity": "sha512-EMnU9E2fSULdsbErBbMaXJvFeD9B4+nPcM3f+4lsiCR0BHLPrLVjv3DbyM2hgQQviKJaTWIRRTjKjWlHg3p2ig==" }, "node_modules/pg-int8": { "version": "1.0.1", @@ -28403,17 +28470,17 @@ } }, "node_modules/pg-pool": { - "version": "3.6.2", - "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.6.2.tgz", - "integrity": "sha512-Htjbg8BlwXqSBQ9V8Vjtc+vzf/6fVUuak/3/XXKA9oxZprwW3IMDQTGHP+KDmVL7rtd+R1QjbnCFPuTHm3G4hg==", + "version": "3.14.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.14.0.tgz", + "integrity": "sha512-gKtPkFdQPU3DksooVLi9LsjZxrsBUZIpa+7aVx+LV5pNh0KzP4Zleud2po+ConrxbuXGBJ6Hfer6hdgpIBpBaw==", "peerDependencies": { "pg": ">=8.0" } }, "node_modules/pg-protocol": { - "version": "1.6.1", - "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.6.1.tgz", - "integrity": "sha512-jPIlvgoD63hrEuihvIg+tJhoGjUsLPn6poJY9N5CnlPd91c2T18T/9zBtLxZSb1EhYxBRoZJtzScCaWlYLtktg==" + "version": "1.14.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.14.0.tgz", + "integrity": "sha512-n5taZ1kO3s9ngDTVxsEznOqCyToTgz0FLuPq0B33COy5pPpuWJpY3/2oRBVETuOgzdqRXfWpM9HIhp2LBBT1BA==" }, "node_modules/pg-types": { "version": "2.2.0", @@ -30832,6 +30899,35 @@ "upper-case-first": "^2.0.2" } }, + "node_modules/serialize-error": { + "version": "13.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-13.0.1.tgz", + "integrity": "sha512-bBZaRwLH9PN5HbLCjPId4dP5bNGEtumcErgOX952IsvOhVPrm3/AeK1y0UHA/QaPG701eg0yEnOKsCOC6X/kaA==", + "dependencies": { + "non-error": "^0.1.0", + "type-fest": "^5.4.1" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/serialize-error/node_modules/type-fest": { + "version": "5.7.0", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-5.7.0.tgz", + "integrity": "sha512-1URUxUqfHFM1c+zfSPsa3gnkO7Aq21qyH75SIduNYz4SzY964rn1X2vCMQaHSHhktiw+0kPa2iyb6PUpXqB6Vg==", + "dependencies": { + "tagged-tag": "^1.0.0" + }, + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/serialize-javascript": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/serialize-javascript/-/serialize-javascript-6.0.2.tgz", @@ -32040,6 +32136,17 @@ "integrity": "sha512-AsS729u2RHUfEra9xJrE39peJcc2stq2+poBXX8bcM08Y6g9j/i/PUzwNQqkaJde7Ntg1TO7bSREbR5sdosQ+g==", "dev": true }, + "node_modules/tagged-tag": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/tagged-tag/-/tagged-tag-1.0.0.tgz", + "integrity": "sha512-yEFYrVhod+hdNyx7g5Bnkkb0G6si8HJurOoOEgC8B/O0uXLHlaey/65KRv6cuWBNhBgHKAROVpc7QyYqE5gFng==", + "engines": { + "node": ">=20" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, "node_modules/tapable": { "version": "2.2.1", "resolved": "https://registry.npmjs.org/tapable/-/tapable-2.2.1.tgz", @@ -34717,3 +34824,4 @@ } } } + diff --git a/app/package.json b/app/package.json index efbf7ed6..20c9de79 100644 --- a/app/package.json +++ b/app/package.json @@ -78,6 +78,7 @@ "openid-client": "^5.6.5", "passport": "^0.7.0", "pg": "^8.12.0", + "pg-boss": "^12.18.2", "re-resizable": "^6.9.17", "react": "18.3.1", "react-dom": "18.3.1", diff --git a/cloudformation/templates/0-waf.yml b/cloudformation/templates/0-waf.yml index 6b7dc560..b46edd7c 100644 --- a/cloudformation/templates/0-waf.yml +++ b/cloudformation/templates/0-waf.yml @@ -389,6 +389,11 @@ Resources: ResourceArn: !GetAtt RegionalWebAcl.Arn LogDestinationConfigs: - !GetAtt WebAclLogging.Arn + RedactedFields: + - SingleHeader: + Name: cookie + - SingleHeader: + Name: authorization WAFDashboard: Type: AWS::CloudWatch::Dashboard @@ -929,7 +934,7 @@ Resources: Outputs: BaseVersion: Description: The version of the cloudformation template - Value: '1.0.0' + Value: '1.0.2' RegionalWebAclArn: Value: !GetAtt RegionalWebAcl.Arn Description: ARN of the Web ACL used to create rules to inspect web requests. diff --git a/cloudformation/templates/1-main.yml b/cloudformation/templates/1-main.yml index 597d9575..abc057ad 100644 --- a/cloudformation/templates/1-main.yml +++ b/cloudformation/templates/1-main.yml @@ -42,6 +42,7 @@ Metadata: - DeploymentStrategy - BeanstalkPlatformUpdateTime - BundleBranch + - OAuth2Issuer - Label: default: S3 Cloudfront Information Parameters: @@ -66,9 +67,10 @@ Metadata: - ViteAlternateMatomoUrl - ViteAlternateMatomoSiteId - ViteAlternateMatomoSubdomain - - Label: - default: ECS Information - Parameters: + - Label: + default: Job Executor and ECS Information + Parameters: + - CreateExecutorS3Bucket - CreateEcsServiceLinkedRole - Label: default: VPC Network Configuration @@ -213,6 +215,13 @@ Parameters: AllowedValues: - 'Create the S3 Bucket' - 'The S3 Bucket already exists' + CreateExecutorS3Bucket: + Type: String + Description: Create the S3 bucket named 'job-executor-codepipeline-artifact-store-{AWS::Region}-{AWS::AccountId}'? Do not change this value if updating an existing stack. + Default: 'The S3 Bucket already exists' + AllowedValues: + - 'Create the S3 Bucket' + - 'The S3 Bucket already exists' S3FrontEndBucket: Type: String Description: Name of the S3 bucket containing the front end web files @@ -482,7 +491,7 @@ Resources: GitHubBranch: !Ref GitHubBranch GitHubConnectionArn: !Ref GitHubConnectionArn GitHubRepo: !Ref GitHubRepo - CreateS3Bucket: 'The S3 Bucket already exists' + CreateS3Bucket: !Ref CreateExecutorS3Bucket EcsStack: Type: AWS::CloudFormation::Stack DeletionPolicy: Delete @@ -496,6 +505,7 @@ Resources: DataBucketName: !Sub '${EnvLabel}-${AWS::AccountId}-data-integration' DomainName: !Ref DomainName S3AccessLoggingBucket: !If [EnableS3AccessLogging, !Ref S3AccessLoggingBucket, ''] + SetCloudWatchRetentionLambdaFunctionArn: !GetAtt 'LambdaFunctionsStack.Outputs.SetCloudWatchRetentionLambdaFunctionArn' CloudWatchDashboardStack: Type: AWS::CloudFormation::Stack DeletionPolicy: Delete @@ -512,4 +522,4 @@ Resources: Outputs: TemplateVersion: Description: The version of the cloudformation template - Value: '2.0.4' + Value: '2.0.9' diff --git a/cloudformation/templates/beanstalk.yml b/cloudformation/templates/beanstalk.yml index 9ac13cfa..cdc93b97 100644 --- a/cloudformation/templates/beanstalk.yml +++ b/cloudformation/templates/beanstalk.yml @@ -82,7 +82,7 @@ Parameters: OAuth2Issuer: Type: String Description: URL of the identity provider that issues access tokens for external API requests. If not set, API access is disabled; the web application continues to function. - + Conditions: EnableHealthCheck: !Not [!Equals [!Ref SNSTopicArn, '']] AttachWAF: !Not [!Equals [!Ref WebACLArn, '']] @@ -581,6 +581,9 @@ Resources: - Namespace: aws:elasticbeanstalk:application:environment OptionName: OAUTH2_ISSUER Value: !Ref OAuth2Issuer + - Namespace: aws:elasticbeanstalk:application:environment + OptionName: UM_CONFIG_SECRET + Value: !Sub '${EnvLabel}-UMConfigSecret' - Namespace: aws:elasticbeanstalk:environment:process:https OptionName: Port Value: 443 diff --git a/cloudformation/templates/cloudwatch-dashboard.yml b/cloudformation/templates/cloudwatch-dashboard.yml index b15ef74a..1a4b4d27 100644 --- a/cloudformation/templates/cloudwatch-dashboard.yml +++ b/cloudformation/templates/cloudwatch-dashboard.yml @@ -569,9 +569,10 @@ Resources: "x": 0, "type": "log", "properties": { - "query": "SOURCE '/ecs/JobExecutorSmall-${EnvLabel}' | parse @message \"{'appDataBasePath': '*'\" as @s3path\n | filter !isblank(@s3path)\n | parse @s3path /(?[^\\/]+)$/\n | sort @timestamp desc\n | display jobId, @logStream, @s3path, @log, @timestamp", + "query": "SOURCE '/ecs/JobExecutorSmall-${EnvLabel}' | SOURCE '/ecs/JobExecutorMedium-${EnvLabel}' | SOURCE '/ecs/JobExecutorLarge-${EnvLabel}' | parse @message \"{'appDataBasePath': '*'\" as @s3path\n | filter !isblank(@s3path)\n | parse @s3path /(?[^\\/]+)$/\n | sort @timestamp desc\n | display jobId, @logStream, @s3path, @log, @timestamp", "region": "${AWS::Region}", "stacked": false, + "title": "Job Executor Runs", "view": "table" } } diff --git a/cloudformation/templates/ecs.yml b/cloudformation/templates/ecs.yml index 48050210..87755b88 100644 --- a/cloudformation/templates/ecs.yml +++ b/cloudformation/templates/ecs.yml @@ -23,6 +23,9 @@ Parameters: S3AccessLoggingBucket: Description: Provide a name for the destination S3 bucket where your Data Integration S3 Bucket Access Logs will be saved. Type: String + SetCloudWatchRetentionLambdaFunctionArn: + Type: String + Description: ARN of the SetCloudWatchRetentionLambdaFunction Conditions: DoCreateEcsServiceLinkedRole: !Equals @@ -55,7 +58,25 @@ Resources: Action: - logs:CreateLogGroup Resource: !Sub 'arn:${AWS::Partition}:logs:${AWS::Region}:${AWS::AccountId}:log-group:*' - EcsCluster: + JobExecutorLogGroupSmall: + Type: Custom::CloudWatchRetention + Properties: + ServiceToken: !Ref SetCloudWatchRetentionLambdaFunctionArn + logGroupName: !Sub '/ecs/JobExecutorSmall-${EnvLabel}' + retentionInDays: 365 + JobExecutorLogGroupMedium: + Type: Custom::CloudWatchRetention + Properties: + ServiceToken: !Ref SetCloudWatchRetentionLambdaFunctionArn + logGroupName: !Sub '/ecs/JobExecutorMedium-${EnvLabel}' + retentionInDays: 365 + JobExecutorLogGroupLarge: + Type: Custom::CloudWatchRetention + Properties: + ServiceToken: !Ref SetCloudWatchRetentionLambdaFunctionArn + logGroupName: !Sub '/ecs/JobExecutorLarge-${EnvLabel}' + retentionInDays: 365 + EcsCluster: Type: AWS::ECS::Cluster Properties: CapacityProviders: diff --git a/cloudformation/templates/lambda-functions.yml b/cloudformation/templates/lambda-functions.yml index 35c4edd5..d58577c6 100644 --- a/cloudformation/templates/lambda-functions.yml +++ b/cloudformation/templates/lambda-functions.yml @@ -725,3 +725,6 @@ Outputs: Condition: EnableSlackAlerting Description: ARN of the ApplicationEventsToSlackFunction Value: !GetAtt 'ApplicationEventsToSlackFunction.Arn' + SetCloudWatchRetentionLambdaFunctionArn: + Description: ARN of the SetCloudWatchRetentionLambdaFunction + Value: !GetAtt 'SetCloudWatchRetentionLambdaFunction.Arn' diff --git a/cloudformation/templates/x-parameter-values.md b/cloudformation/templates/x-parameter-values.md index 39e8057d..7ef9f012 100644 --- a/cloudformation/templates/x-parameter-values.md +++ b/cloudformation/templates/x-parameter-values.md @@ -41,7 +41,7 @@ Please note that descriptions, values, and available parameters may evolve over - **DeploymentStrategy** - Beanstalk deployment strategy for platform updates and application versions. - **BeanstalkPlatformUpdateTime** - Beanstalk Platform update day and time (in UTC) using the format 'Day:HH:MM' (e.g., Wed:15:30). Platform updates will be disabled if no value is specified. - **BundleBranch** - Name of the bundle branch in the [earthmover Ed-Fi bundles repository](https://github.com/edanalytics/earthmover_edfi_bundles). -- **ExternalApiTokenIssuer** - URL of the identity provider that issues access tokens for external API requests. If not set, API access is disabled; the web application continues to function +- **OAuth2Issuer** - URL of the identity provider that issues access tokens for external API requests. If not set, API access is disabled; the web application continues to function **S3 Cloudfront Information** - **S3FrontEndBucket** - Name of the S3 bucket containing the front end web files. @@ -65,7 +65,8 @@ Please note that descriptions, values, and available parameters may evolve over - **ViteAlternateMatomoSiteId** - Internal to EA. Not needed for open-source deployments. Leave blank. - **ViteAlternateMatomoSubdomain** - Internal to EA. Not needed for open-source deployments. Leave blank. -**ECS Information** +**Job Executor and ECS Information** +- **CreateExecutorS3Bucket** - Choose whether to create S3 bucket named 'job-executor-codepipeline-artifact-store-{AWS::Region}-{AWS::AccountId}' in your AWS account. Only choose `Create the S3 bucket` for the very first deployment of Runway in your AWS account. Do not change this value if updating an existing stack. - **CreateEcsServiceLinkedRole** - Choose whether to create an ECS Service Linked Role in your AWS account. This role enables Amazon ECS to manage your cluster. If it already exists in your account, select False. **VPC Network Configuration** diff --git a/executor/Dockerfile b/executor/Dockerfile index 60f8afef..b5d61311 100644 --- a/executor/Dockerfile +++ b/executor/Dockerfile @@ -4,6 +4,9 @@ # Use a base image FROM public.ecr.aws/docker/library/python:3.12-slim-bookworm +RUN useradd -M -s /bin/bash executor && \ + mkdir -p /executor && chown executor:executor /executor + # Set the working directory WORKDIR /executor @@ -12,14 +15,14 @@ RUN apt update && \ # - git: clones bundle repositories apt install -y git -COPY ./ /executor -RUN pip install -r requirements.txt && \ - pip install -e . +COPY --chown=executor:executor requirements.txt /executor/ +RUN pip install -r requirements.txt -RUN git clone https://github.com/edanalytics/earthmover_edfi_bundles.git bundles +COPY --chown=executor:executor ./ /executor +RUN pip install -e . -RUN useradd -M -s /bin/bash executor && \ - chown -R executor:executor /executor USER executor +RUN git clone https://github.com/edanalytics/earthmover_edfi_bundles.git bundles + CMD ["python", "-u", "/executor/scripts/main.py"] diff --git a/executor/buildspec.yml b/executor/buildspec.yml index 27247dda..130b17e4 100644 --- a/executor/buildspec.yml +++ b/executor/buildspec.yml @@ -1,6 +1,9 @@ # config file to build Job Executor and push to ECR version: 0.2 phases: + install: + commands: + - docker buildx create --name containerd --driver=docker-container --driver-opt default-load=true --use pre_build: commands: - pwd @@ -9,17 +12,24 @@ phases: - export ACCOUNT_ID=$(aws sts get-caller-identity --query Account --output text) - export ECR_REPO=$ACCOUNT_ID.dkr.ecr.$AWS_REGION.amazonaws.com/$ECR_REPO_NAME - export DOCKER_TAG=$ECR_REPO:$CODEBUILD_RESOLVED_SOURCE_VERSION - - export DOCKER_CACHE=$ECR_REPO:latest + - export DOCKER_LATEST=$ECR_REPO:latest - aws ecr get-login-password --region $AWS_REGION | docker login --username AWS --password-stdin $ECR_REPO - env build: commands: - cd executor - - docker pull $DOCKER_CACHE || true - - docker build --cache-from $DOCKER_CACHE --build-arg GIT_HASH=${CODEBUILD_RESOLVED_SOURCE_VERSION} -t $DOCKER_TAG . - - docker tag $DOCKER_TAG $DOCKER_CACHE + - >- + docker buildx build + --cache-to type=inline,ref=${DOCKER_LATEST},mode=min + --cache-from type=registry,ref=${DOCKER_LATEST} + --tag ${DOCKER_TAG} + --builder=containerd + --build-arg GIT_HASH=${CODEBUILD_RESOLVED_SOURCE_VERSION} + --progress=plain + . + - docker tag $DOCKER_TAG $DOCKER_LATEST - docker push $DOCKER_TAG - - docker push $DOCKER_CACHE + - docker push $DOCKER_LATEST post_build: commands: - - echo "docker image built - $DOCKER_TAG" \ No newline at end of file + - echo "docker image built - $DOCKER_TAG" diff --git a/executor/executor/errors.py b/executor/executor/errors.py index ed819fff..f9a74ae2 100644 --- a/executor/executor/errors.py +++ b/executor/executor/errors.py @@ -13,6 +13,10 @@ def __init__(self, code, stacktrace): self.stacktrace = stacktrace def to_json(self): + max_bytes = 6144 # an error payload above 8kB will be rejected. Truncate to the last 6kB of the stacktrace + bytes = self.stacktrace.encode('utf-8') + truncated_bytes = bytes[-max_bytes:] + self.stacktrace = truncated_bytes.decode('utf-8', errors='ignore') all_keys = deepcopy(vars(self)) del all_keys["code"] return {"code": self.code, "payload": all_keys} diff --git a/executor/executor/executor.py b/executor/executor/executor.py index 313a05f8..36a14aa7 100644 --- a/executor/executor/executor.py +++ b/executor/executor/executor.py @@ -252,15 +252,29 @@ def refresh_bundle_code(self): except subprocess.CalledProcessError: self.error = error.GitPullError() raise + + def earthmover_cmd(self, **kwargs): + """Thinly wrap our em calls to handle invocation and logging. Returns either a CompletedProcess object or CalledProcessError object""" + + em=subprocess.run( + **kwargs + ) + + # Log stdout and stderror if they exist + if em.stdout: + self.logger.info(f"earthmover stdout: {em.stdout}") + if em.stderr: + self.logger.info(f"earthmover stderr: {em.stderr}") + + return em def earthmover_deps(self): """Create the Earthmover runtime environment by installing bundle dependencies""" self.set_action(action.EARTHMOVER_DEPS) try: - subprocess.run( - ["earthmover", "-c", self.wrapper_earthmover, "deps"], - ).check_returncode() + cmd=["earthmover", "-c", self.wrapper_earthmover, "deps"] + self.earthmover_cmd(args=cmd, check=True) except subprocess.CalledProcessError: self.error = error.EarthmoverDepsError() raise @@ -432,7 +446,6 @@ def orchestrate_earthmover(self): self.earthmover_run(artifact.EM_RESULTS.path) self.upload_artifact(artifact.EM_RESULTS) self.record_highest_match_rate() - self.enforce_match_threshold() self.output_sets = [OutputSet( local_dir=self.output_dir, @@ -445,12 +458,17 @@ def orchestrate_earthmover(self): # if we're here, the first pass of Earthmover was successful if (self.send_to_ods # i.e. we've only tried matching this year's students so far and self.cross_year_match_available # and we have access to EDU - and self.num_unmatched_students > 0 # and there are unmatched students from the first pass + and (self.num_unmatched_students is None # and either there were no matches at all + or self.num_unmatched_students > 0) # or there are any remaining unmatched students ): # then take a second pass with the cross-year roster from EDU # and thus produce a second output set to be sideloaded cross_year_output = self.cross_year_pass(self.output_sets[0]) self.output_sets.append(cross_year_output) + # If the conditions for a second pass are not met + # fall back to our typical process and enforce the match rate threshold + else: + self.enforce_match_threshold() self.upload_artifact(artifact.MATCH_RATES) @@ -467,27 +485,16 @@ def earthmover_run(self, results_path): fatal = False try: - em = subprocess.run( - ["earthmover", "-c", self.wrapper_earthmover, "compile"], - capture_output=True, - text=True - ) + cmd = ["earthmover", "-c", self.wrapper_earthmover, "compile"] + em = self.earthmover_cmd(args=cmd, capture_output=True, text=True) em.check_returncode() # attempt no. 1 cmd = ["earthmover", "-c", self.wrapper_earthmover, "run", "--results-file", results_path] cmd.extend(encoding_args) - em = subprocess.run( - cmd, - capture_output=True, - text=True - ) - - if em.stdout: - self.logger.info(f"earthmover stdout: {em.stdout}") - if em.stderr: - self.logger.info(f"earthmover stderr: {em.stderr}") + em = self.earthmover_cmd(args=cmd, capture_output=True, text=True) em.check_returncode() + except subprocess.CalledProcessError as err: self.logger.error("earthmover encountered an error") fatal = True @@ -498,11 +505,10 @@ def earthmover_run(self, results_path): self.logger.error(f"Failed to read file with {encoding} encoding. Retrying with Latin1...") try: # attempt no. 2 - need a new em object to overwrite the decoding error - em = subprocess.run( - ["earthmover", "-c", self.wrapper_earthmover, "run", "--results-file", results_path, "--set", "sources.input.encoding", "iso-8859-1"], - ) + cmd = ["earthmover", "-c", self.wrapper_earthmover, "run", "--results-file", results_path, "--set", "sources.input.encoding", "iso-8859-1"] + em = self.earthmover_cmd(args=cmd, capture_output=True, text=True) em.check_returncode() - + fatal = False # if we made it this far, we can abort the shutdown except subprocess.CalledProcessError: # failed again, move on to shutdown procedure @@ -523,38 +529,52 @@ def earthmover_run(self, results_path): def cross_year_pass(self, primary): """Run a second Earthmover pass on unmatched students using a cross-year roster in an attempt to match more students.""" - # constrain this pass to use the IDs that matched best in the first pass - first_run_id_name = self.highest_match_id_name - first_run_id_type = self.highest_match_id_type first_run_output_dir = os.path.abspath(config.OUTPUT_DIR_FIRST_RUN) os.rename(self.output_dir, first_run_output_dir) primary.local_dir = first_run_output_dir os.mkdir(self.output_dir) - # use only the students who failed to match the primary ID from the first run - unmatched_path = os.path.join(first_run_output_dir, os.path.basename(artifact.UNMATCHED_STUDENTS.path)) - os.environ["INPUT_FILE"] = unmatched_path - self.input_sources["INPUT_FILE"]["path"] = unmatched_path - self.get_roster_from_edu(config.CROSS_YEAR_ROSTER_PATH) artifact.CROSS_YEAR_ROSTER.needs_upload = True self.upload_artifact(artifact.CROSS_YEAR_ROSTER) os.environ["EDFI_ROSTER_FILE"] = os.path.abspath(config.CROSS_YEAR_ROSTER_PATH) - # Constrain to the ID column the first pass matched on. The bundle always appends - # studentUniqueId internally, so we pass an empty list when that's what won. - os.environ["POSSIBLE_STUDENT_ID_COLUMNS"] = first_run_id_name - os.environ["EDFI_STUDENT_ID_TYPES"] = ( - "" if first_run_id_type == "studentUniqueId" else first_run_id_type - ) - # we already know which ID to use so we should succeed no matter how many failed matches remain - os.environ["REQUIRED_ID_MATCH_RATE"] = "0.0" - self.logger.info(f"cross-year pass: matching on {first_run_id_name} ({first_run_id_type} ID)") + # Boolean to capture whether our first run met the match rate threshold + met_initial_threshold = self.highest_match_rate >= config.REQUIRED_ID_MATCH_RATE + + # If we hit the required match rate on the first pass, constrain to the ID column that won. + # Otherwise, we run again and check against all ID types. + # The bundle always appends studentUniqueId internally, so we pass an empty list when that's what won. + if met_initial_threshold: + + # use only the students who failed to match the primary ID from the first run + unmatched_path = os.path.join(first_run_output_dir, os.path.basename(artifact.UNMATCHED_STUDENTS.path)) + os.environ["INPUT_FILE"] = unmatched_path + self.input_sources["INPUT_FILE"]["path"] = unmatched_path + + # constrain this pass to use the IDs that matched best in the first pass + first_run_id_name = self.highest_match_id_name + first_run_id_type = self.highest_match_id_type + + os.environ["POSSIBLE_STUDENT_ID_COLUMNS"] = first_run_id_name + os.environ["EDFI_STUDENT_ID_TYPES"] = ( + "" if first_run_id_type == "studentUniqueId" else first_run_id_type + ) + # we already know which ID to use so we should succeed no matter how many failed matches remain + os.environ["REQUIRED_ID_MATCH_RATE"] = "0.0" + self.logger.info(f"cross-year pass: matching on {first_run_id_name} ({first_run_id_type} ID)") + else: + self.logger.info("cross-year pass: first pass below threshold, running again against all ID types") self.earthmover_run(artifact.EM_RESULTS_X_YEAR.path) artifact.EM_RESULTS_X_YEAR.needs_upload = True self.upload_artifact(artifact.EM_RESULTS_X_YEAR) + + if not met_initial_threshold: + # Only enforce our match threshold if the initial run did not hit + self.record_highest_match_rate() + self.enforce_match_threshold() self.logger.info(f"cross-year pass: match_rates: {load_match_rates()}") count = count_unmatched_students() diff --git a/executor/requirements.txt b/executor/requirements.txt index eface1db..4538b447 100644 --- a/executor/requirements.txt +++ b/executor/requirements.txt @@ -1,6 +1,58 @@ -awscli -boto3 -chardet +aiohappyeyeballs==2.6.2 +aiohttp==3.14.1 +aiohttp-retry==2.9.1 +aiosignal==1.4.0 +attrs==26.1.0 +awscli==1.45.33 +boto3==1.43.33 +botocore==1.43.33 +bottleneck==1.6.0 +certifi==2026.6.17 +chardet==7.4.3 +charset-normalizer==3.4.7 +click==8.4.1 +cloudpickle==3.1.2 +colorama==0.4.6 +dask==2024.8.2 +dask-expr==1.1.13 +docutils==0.19 earthmover==0.4.10 +frozenlist==1.8.0 +fsspec==2026.6.0 +idna==3.18 +Jinja2==3.1.6 +jmespath==1.1.0 +jsonschema==4.26.0 +jsonschema-specifications==2025.9.1 lightbeam==0.1.12 -requests +llvmlite==0.47.0 +locket==1.0.0 +MarkupSafe==3.0.3 +multidict==6.7.1 +networkx==3.6.1 +numba==0.65.1 +numexpr==2.14.1 +numpy==2.4.6 +packaging==26.2 +pandas==2.2.3 +partd==1.4.2 +propcache==0.5.2 +pyarrow==24.0.0 +pyasn1==0.6.4 +python-dateutil==2.9.0.post0 +pytz==2026.2 +PyYAML==6.0.3 +referencing==0.37.0 +requests==2.34.2 +rpds-py==2026.5.1 +rsa==4.7.2 +s3transfer==0.19.0 +setuptools==83.0.0 +six==1.17.0 +toolz==1.1.0 +typing_extensions==4.15.0 +tzdata==2026.2 +urllib3==2.7.0 +wheel==0.47.0 +yarl==1.24.2 +