diff --git a/.env.demo b/.env.demo index a686298f16..102d615088 100644 --- a/.env.demo +++ b/.env.demo @@ -382,4 +382,18 @@ ADMIN_KEYCLOAK_SECRET= # Add in case the keycloak id of the old platform client is changed and want to update all the users in database with the new admin client PLATFORM_ADMIN_OLD_CLIENT_ID= +# --- OpenBao secret storage (PR #1679) --- +# Enable remote secrets fetching via OpenBao. Set to false to fall back to +# plain environment variables (default local behaviour). +ENABLE_BAO=true +# Secret provider: only 'openbao' is supported today +SECRETS_PROVIDER=openbao +# OpenBao server URL (docker-compose.openbao.yml maps 8200) +BAO_URL=http://127.0.0.1:8200 +# Default KV v2 path fetched at boot and injected into process.env +BAO_SECRET_PATH=secret/data/credebl_resend_api_key +# AppRole credentials. Generate your own via: ./openbao-init.sh +# (values below are for the local 'openbao' container created by docker-compose.openbao.yml) +BAO_ROLE_ID=ca2a350b-1871-0851-8557-42b36aa9c623 +BAO_SECRET_ID=3f0d8d45-c232-f5c1-2984-fe58e5144be8 HIDE_EXPERIMENTAL_OIDC_CONTROLLERS=false diff --git a/README.md b/README.md index 20f15ecffd..acf7636992 100644 --- a/README.md +++ b/README.md @@ -63,6 +63,54 @@ The `docker-compose.yml` file is available in the root folder. docker-compose up ``` +## OpenBao Secret Storage (Optional) + +The platform can source the credentials used by email and file-storage integrations from an [OpenBao](https://openbao.org) server instead of plain environment variables. This is opt-in and disabled by default. + +### • Run OpenBao using Docker Compose + +```bash +docker compose -f docker-compose.openbao.yml up -d +``` + +This starts an OpenBao server on `http://127.0.0.1:8200` with file-backed storage persisted on a named volume. Server settings live in `config.hcl`. + +### • Provision the server + +OpenBao starts sealed and uninitialized. Run the one-shot provisioning script (idempotent — safe to re-run): + +```bash +./openbao-init.sh +``` + +The script initializes and unseals the server, enables the KV v2 secrets engine at `secret/`, enables the AppRole auth method, creates a `credebl` role scoped to the `credebl_*` secret paths, stores the secret values found in the environment (e.g. `RESEND_API_KEY`, `SMTP_HOST`, `AWS_ACCESS_KEY`), and prints the values to copy into your env file: + +```bash +BAO_URL=http://127.0.0.1:8200 +BAO_SECRET_PATH=secret/data/credebl_resend_api_key +BAO_ROLE_ID= +BAO_SECRET_ID= +``` + +Keep the printed `BAO_UNSEAL_KEY` and `BAO_ROOT_TOKEN` safe — they are shown only once. On later runs, pass `BAO_ROOT_TOKEN` (and `BAO_SECRET_ID` to re-print it). + +### • Configure the platform + +Add the following to your `.env` (values are already present as placeholders in `.env.demo`): + +```bash +ENABLE_BAO=true +SECRETS_PROVIDER=openbao +BAO_URL=http://127.0.0.1:8200 +BAO_SECRET_PATH=secret/data/credebl_resend_api_key +BAO_ROLE_ID= +BAO_SECRET_ID= +``` + +At startup every microservice authenticates to OpenBao via AppRole, fetches the secrets at `BAO_SECRET_PATH`, and injects them into `process.env` before the NATS listener starts. Email and S3 integrations re-fetch their own credential paths on demand (e.g. `secret/data/credebl_smtp_config`, `secret/data/credebl_aws_keys`) with a 10-minute TTL cache. + +Set `ENABLE_BAO=false` (or omit it) to fall back to local environment variables. + ## Run CREDEBL Microservices ### • Install Dependencies diff --git a/apps/agent-provisioning/src/main.ts b/apps/agent-provisioning/src/main.ts index 6b3e2963c6..f02b18dd2c 100644 --- a/apps/agent-provisioning/src/main.ts +++ b/apps/agent-provisioning/src/main.ts @@ -1,14 +1,22 @@ -import { NestFactory } from '@nestjs/core'; -import { HttpExceptionFilter } from 'libs/http-exception.filter'; -import { Logger } from '@nestjs/common'; +import * as dotenv from 'dotenv'; + import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + import { AgentProvisioningModule } from './agent-provisioning.module'; -import { getNatsOptions } from '@credebl/common/nats.config'; import { CommonConstants } from '@credebl/common/common.constant'; +import { HttpExceptionFilter } from 'libs/http-exception.filter'; +import { Logger } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); + const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(AgentProvisioningModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/agent-service/src/main.ts b/apps/agent-service/src/main.ts index c949023a57..f992d5b969 100644 --- a/apps/agent-service/src/main.ts +++ b/apps/agent-service/src/main.ts @@ -1,18 +1,25 @@ -import { HttpExceptionFilter } from 'libs/http-exception.filter'; -import { Logger } from '@nestjs/common'; -import { NestFactory } from '@nestjs/core'; -import { AgentServiceModule } from './agent-service.module'; -import { AgentServiceService } from './agent-service.service'; +import * as dotenv from 'dotenv'; + import { IAgentSpinupDto, IUserRequestInterface } from './interface/agent-service.interface'; import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; + +import { AgentServiceModule } from './agent-service.module'; +import { AgentServiceService } from './agent-service.service'; import { CommonConstants } from '@credebl/common/common.constant'; +import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Ledgers } from '@credebl/enum/enum'; +import { Logger } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(AgentServiceModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/api-gateway/src/main.ts b/apps/api-gateway/src/main.ts index d7934ae789..b8b8a9895d 100755 --- a/apps/api-gateway/src/main.ts +++ b/apps/api-gateway/src/main.ts @@ -17,10 +17,13 @@ import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapt import { UpdatableValidationPipe } from '@credebl/common/custom-overrideable-validation-pipe'; import * as useragent from 'express-useragent'; import { EcosystemSwaggerFilter } from './authz/guards/ecosystem-swagger.filter'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; dotenv.config(); async function bootstrap(): Promise { + await loadConfigSecrets(); + try { if (otelSDK) { await otelSDK.start(); diff --git a/apps/cloud-wallet/src/main.ts b/apps/cloud-wallet/src/main.ts index a411b07009..1da37b7ec0 100644 --- a/apps/cloud-wallet/src/main.ts +++ b/apps/cloud-wallet/src/main.ts @@ -1,15 +1,22 @@ -import { NestFactory } from '@nestjs/core'; +import * as dotenv from 'dotenv'; + +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + import { CloudWalletModule } from './cloud-wallet.module'; +import { CommonConstants } from '@credebl/common/common.constant'; import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Logger } from '@nestjs/common'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; -import { CommonConstants } from '@credebl/common/common.constant'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(CloudWalletModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/connection/src/main.ts b/apps/connection/src/main.ts index c98e3ea74c..10ee34ff81 100644 --- a/apps/connection/src/main.ts +++ b/apps/connection/src/main.ts @@ -1,15 +1,22 @@ -import { NestFactory } from '@nestjs/core'; +import * as dotenv from 'dotenv'; + +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + +import { CommonConstants } from '@credebl/common/common.constant'; import { ConnectionModule } from './connection.module'; import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Logger } from '@nestjs/common'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; -import { CommonConstants } from '@credebl/common/common.constant'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(ConnectionModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/ecosystem/src/main.ts b/apps/ecosystem/src/main.ts index ebb2a53df0..71fed8e971 100644 --- a/apps/ecosystem/src/main.ts +++ b/apps/ecosystem/src/main.ts @@ -1,3 +1,5 @@ +import * as dotenv from 'dotenv'; + import { MicroserviceOptions, Transport } from '@nestjs/microservices'; import { CommonConstants } from '@credebl/common/common.constant'; @@ -8,10 +10,14 @@ import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; // import { nkeyAuthenticator } from 'nats'; import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(EcosystemModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/geo-location/src/main.ts b/apps/geo-location/src/main.ts index 84800754f9..9e16df13e9 100644 --- a/apps/geo-location/src/main.ts +++ b/apps/geo-location/src/main.ts @@ -1,15 +1,22 @@ +import * as dotenv from 'dotenv'; + +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + +import { CommonConstants } from '@credebl/common/common.constant'; +import { GeoLocationModule } from './geo-location.module'; import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Logger } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; -import { GeoLocationModule } from './geo-location.module'; -import { CommonConstants } from '@credebl/common/common.constant'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(GeoLocationModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/issuance/src/main.ts b/apps/issuance/src/main.ts index b69e8e57c4..9d137b58d3 100644 --- a/apps/issuance/src/main.ts +++ b/apps/issuance/src/main.ts @@ -1,15 +1,22 @@ -import { NestFactory } from '@nestjs/core'; -import { HttpExceptionFilter } from 'libs/http-exception.filter'; -import { Logger } from '@nestjs/common'; +import * as dotenv from 'dotenv'; + import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { IssuanceModule } from '../src/issuance.module'; -import { getNatsOptions } from '@credebl/common/nats.config'; + import { CommonConstants } from '@credebl/common/common.constant'; +import { HttpExceptionFilter } from 'libs/http-exception.filter'; +import { IssuanceModule } from '../src/issuance.module'; +import { Logger } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(IssuanceModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/ledger/src/main.ts b/apps/ledger/src/main.ts index 022dffa761..b25a6a6b01 100644 --- a/apps/ledger/src/main.ts +++ b/apps/ledger/src/main.ts @@ -1,13 +1,20 @@ -import { NestFactory } from '@nestjs/core'; -import { LedgerModule } from './ledger.module'; -import { HttpExceptionFilter } from 'libs/http-exception.filter'; -import { Logger } from '@nestjs/common'; +import * as dotenv from 'dotenv'; + import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; + import { CommonConstants } from '@credebl/common/common.constant'; +import { HttpExceptionFilter } from 'libs/http-exception.filter'; +import { LedgerModule } from './ledger.module'; +import { Logger } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(LedgerModule, { transport: Transport.NATS, options: getNatsOptions(CommonConstants.LEDGER_SERVICE, process.env.LEDGER_NKEY_SEED, process.env.NATS_CREDS_FILE) diff --git a/apps/notification/src/main.ts b/apps/notification/src/main.ts index 64d505ed7a..892524c401 100644 --- a/apps/notification/src/main.ts +++ b/apps/notification/src/main.ts @@ -1,15 +1,22 @@ -import { NestFactory } from '@nestjs/core'; +import * as dotenv from 'dotenv'; + +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + +import { CommonConstants } from '@credebl/common/common.constant'; import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Logger } from '@nestjs/common'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; +import { NestFactory } from '@nestjs/core'; +import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; import { NotificationModule } from '../src/notification.module'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { CommonConstants } from '@credebl/common/common.constant'; -import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(NotificationModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/oid4vc-issuance/src/main.ts b/apps/oid4vc-issuance/src/main.ts index 68aa19bbfd..4e9702a079 100644 --- a/apps/oid4vc-issuance/src/main.ts +++ b/apps/oid4vc-issuance/src/main.ts @@ -1,15 +1,23 @@ -import { NestFactory } from '@nestjs/core'; -import { HttpExceptionFilter } from 'libs/http-exception.filter'; -import { Logger } from '@nestjs/common'; +import * as dotenv from 'dotenv'; + import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; + import { CommonConstants } from '@credebl/common/common.constant'; +import { HttpExceptionFilter } from 'libs/http-exception.filter'; +import { Logger } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; import { Oid4vcIssuanceModule } from './oid4vc-issuance.module'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); + if (!process.env.STATUS_LIST_HOST) { logger.warn('STATUS_LIST_HOST is not configured. Revocable SD-JWT flow will be disabled.'); } diff --git a/apps/oid4vc-verification/src/main.ts b/apps/oid4vc-verification/src/main.ts index 667a311027..2303643e63 100644 --- a/apps/oid4vc-verification/src/main.ts +++ b/apps/oid4vc-verification/src/main.ts @@ -1,14 +1,21 @@ -import { NestFactory } from '@nestjs/core'; -import { Logger } from '@nestjs/common'; +import * as dotenv from 'dotenv'; + import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; + import { CommonConstants } from '@credebl/common/common.constant'; +import { Logger } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; import { Oid4vpModule } from './oid4vc-verification.module'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(Oid4vpModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/organization/src/main.ts b/apps/organization/src/main.ts index 8ec4de9d2f..625c4f59cd 100644 --- a/apps/organization/src/main.ts +++ b/apps/organization/src/main.ts @@ -1,16 +1,23 @@ +import * as dotenv from 'dotenv'; + +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + +import { CommonConstants } from '@credebl/common/common.constant'; import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Logger } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; +import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; import { OrganizationModule } from './organization.module'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; // import { nkeyAuthenticator } from 'nats'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { CommonConstants } from '@credebl/common/common.constant'; -import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(OrganizationModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/user/src/main.ts b/apps/user/src/main.ts index 01411c4460..cd3d9ad776 100644 --- a/apps/user/src/main.ts +++ b/apps/user/src/main.ts @@ -1,3 +1,5 @@ +import * as dotenv from 'dotenv'; + import { MicroserviceOptions, Transport } from '@nestjs/microservices'; import { CommonConstants } from '@credebl/common/common.constant'; @@ -7,10 +9,14 @@ import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; import { UserModule } from './user.module'; import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(UserModule, { transport: Transport.NATS, options: getNatsOptions(CommonConstants.USER_SERVICE, process.env.USER_NKEY_SEED, process.env.NATS_CREDS_FILE) diff --git a/apps/utility/src/main.ts b/apps/utility/src/main.ts index fcf71b53a6..07d9bfa108 100644 --- a/apps/utility/src/main.ts +++ b/apps/utility/src/main.ts @@ -1,15 +1,22 @@ +import * as dotenv from 'dotenv'; + +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + +import { CommonConstants } from '@credebl/common/common.constant'; import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Logger } from '@nestjs/common'; import { NestFactory } from '@nestjs/core'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; -import { UtilitiesModule } from './utilities.module'; -import { CommonConstants } from '@credebl/common/common.constant'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { UtilitiesModule } from './utilities.module'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(UtilitiesModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/verification/src/main.ts b/apps/verification/src/main.ts index 0195a8ee1a..5fe2fd73d3 100644 --- a/apps/verification/src/main.ts +++ b/apps/verification/src/main.ts @@ -1,15 +1,22 @@ -import { NestFactory } from '@nestjs/core'; +import * as dotenv from 'dotenv'; + +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + +import { CommonConstants } from '@credebl/common/common.constant'; import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Logger } from '@nestjs/common'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; +import { NestFactory } from '@nestjs/core'; +import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; import { VerificationModule } from './verification.module'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { CommonConstants } from '@credebl/common/common.constant'; -import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(VerificationModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/webhook/src/main.ts b/apps/webhook/src/main.ts index 07ca3bcea1..a7cc3bb61b 100644 --- a/apps/webhook/src/main.ts +++ b/apps/webhook/src/main.ts @@ -1,15 +1,22 @@ -import { NestFactory } from '@nestjs/core'; +import * as dotenv from 'dotenv'; + +import { MicroserviceOptions, Transport } from '@nestjs/microservices'; + +import { CommonConstants } from '@credebl/common/common.constant'; import { HttpExceptionFilter } from 'libs/http-exception.filter'; import { Logger } from '@nestjs/common'; -import { MicroserviceOptions, Transport } from '@nestjs/microservices'; +import { NestFactory } from '@nestjs/core'; +import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; import { WebhookModule } from '../src/webhook.module'; import { getNatsOptions } from '@credebl/common/nats.config'; -import { CommonConstants } from '@credebl/common/common.constant'; -import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); const logger = new Logger(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(WebhookModule, { transport: Transport.NATS, options: getNatsOptions( diff --git a/apps/x509/src/main.ts b/apps/x509/src/main.ts index eb77bb5209..b181e45fde 100644 --- a/apps/x509/src/main.ts +++ b/apps/x509/src/main.ts @@ -1,13 +1,20 @@ -import { NestFactory } from '@nestjs/core'; -import { HttpExceptionFilter } from 'libs/http-exception.filter'; -import { Logger } from '@nestjs/common'; +import * as dotenv from 'dotenv'; + import { MicroserviceOptions, Transport } from '@nestjs/microservices'; -import { getNatsOptions } from '@credebl/common/nats.config'; + import { CommonConstants } from '@credebl/common/common.constant'; +import { HttpExceptionFilter } from 'libs/http-exception.filter'; +import { Logger } from '@nestjs/common'; +import { NestFactory } from '@nestjs/core'; import NestjsLoggerServiceAdapter from '@credebl/logger/nestjsLoggerServiceAdapter'; import { X509Module } from './x509.module'; +import { getNatsOptions } from '@credebl/common/nats.config'; +import { loadConfigSecrets } from '@credebl/config/secret-storage/secrets-loader'; + +dotenv.config(); async function bootstrap(): Promise { + await loadConfigSecrets(); const app = await NestFactory.createMicroservice(X509Module, { transport: Transport.NATS, options: getNatsOptions(CommonConstants.X509_SERVICE, process.env.X509_NKEY_SEED, process.env.NATS_CREDS_FILE) diff --git a/config.hcl b/config.hcl new file mode 100644 index 0000000000..4f0774dfd6 --- /dev/null +++ b/config.hcl @@ -0,0 +1,10 @@ +storage "file" { + path = "/openbao/file" +} + +listener "tcp" { + address = "0.0.0.0:8200" + tls_disable = 1 # In true production, configure actual TLS certificates here +} + +ui = true diff --git a/docker-compose.openbao.yml b/docker-compose.openbao.yml new file mode 100644 index 0000000000..f9e44b5f14 --- /dev/null +++ b/docker-compose.openbao.yml @@ -0,0 +1,20 @@ +services: + openbao: + image: openbao/openbao:latest + container_name: openbao + # Entrypoint runs as root, chowns the data volume, then drops to openbao user + ports: + - "8200:8200" + environment: + - BAO_ADDR=http://127.0.0.1:8200 + volumes: + - openbao_data:/openbao/file + - ./config.hcl:/bao/config/bao.hcl:ro + cap_add: + - IPC_LOCK + # Expressly point to the HCL config file + command: ["server", "-config=/bao/config/bao.hcl"] + restart: unless-stopped + +volumes: + openbao_data: \ No newline at end of file diff --git a/libs/common/src/common.constant.ts b/libs/common/src/common.constant.ts index 011ef9780f..9613ed4543 100644 --- a/libs/common/src/common.constant.ts +++ b/libs/common/src/common.constant.ts @@ -443,6 +443,12 @@ export enum CommonConstants { OIDC_VERIFIER_SESSION_RESPONSE_GET_BY_ID = 'get-oid4vp-verifier-session-response-id', OIDC_VERIFIER_SESSION_AUTH_RESPONSE_VERIFY = 'verify-oid4vp-verifier-session-auth-response', + // BAO_STORAGE_PATH + CREDEBL_RESEND_API_KEY_PATH = 'secret/data/credebl_resend_api_key', + CREDEBL_SMTP_CONFIG_PATH = 'secret/data/credebl_smtp_config', + CREDEBL_SENDGRID_API_KEY_PATH = 'secret/data/credebl_sendgrid_api_key', + CREDEBL_AWS_KEY_PATH = 'secret/data/credebl_aws_keys', + //X509 X509_CREATE_CERTIFICATE = 'create-x509-certificate', X509_IMPORT_CERTIFICATE = 'import-x509-certificate', @@ -451,7 +457,9 @@ export enum CommonConstants { // Storage Types STORAGE_TYPE_AWS = 'aws', STORAGE_TYPE_LOCAL = 'local', - STORAGE_TYPE_RUSTFS = 'rustfs' + STORAGE_TYPE_RUSTFS = 'rustfs', + + OPENBAO_REQUEST_TIMEOUT = 10000 } export const MICRO_SERVICE_NAME = Symbol('MICRO_SERVICE_NAME'); export const ATTRIBUTE_NAME_REGEX = /\['(.*?)'\]/; diff --git a/libs/common/src/resend-helper-file.spec.ts b/libs/common/src/resend-helper-file.spec.ts new file mode 100644 index 0000000000..0eb374ddef --- /dev/null +++ b/libs/common/src/resend-helper-file.spec.ts @@ -0,0 +1,81 @@ +import { sendWithResend } from './resend-helper-file'; +import { fetchSecrets } from './utils/secretLoader.util'; + +jest.mock('resend', () => ({ + Resend: jest.fn() +})); +jest.mock('./utils/secretLoader.util', () => ({ + fetchSecrets: jest.fn() +})); + +import { Resend } from 'resend'; +import { EmailDto } from './dtos/email.dto'; + +const MockedResend = Resend as unknown as jest.Mock; +const mockedFetchSecrets = fetchSecrets as jest.MockedFunction; + +const emailDto: EmailDto = { + emailFrom: 'from@credebl.id', + emailTo: ['to@example.com'], + emailSubject: 'subject', + emailText: 'text', + emailHtml: '

html

' +}; + +describe('sendWithResend', () => { + const originalEnv = { ...process.env }; + let sendMock: jest.Mock; + + beforeEach(() => { + delete process.env.RESEND_API_KEY; + sendMock = jest.fn().mockResolvedValue({ data: { id: 'email-1' } }); + MockedResend.mockImplementation(() => ({ emails: { send: sendMock } })); + jest.clearAllMocks(); + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + + it('sends an email using the provider-sourced API key', async () => { + mockedFetchSecrets.mockResolvedValue({ RESEND_API_KEY: 'resend-key' }); + + await expect(sendWithResend(emailDto)).resolves.toBe(true); + + expect(mockedFetchSecrets).toHaveBeenCalledWith('secret/data/credebl_resend_api_key'); + expect(MockedResend).toHaveBeenCalledWith('resend-key'); + expect(sendMock).toHaveBeenCalledWith( + expect.objectContaining({ to: emailDto.emailTo, subject: emailDto.emailSubject }) + ); + }); + + it('falls back to the environment variable when no secret is returned', async () => { + mockedFetchSecrets.mockResolvedValue({}); + process.env.RESEND_API_KEY = 'env-key'; + + await expect(sendWithResend(emailDto)).resolves.toBe(true); + + expect(MockedResend).toHaveBeenCalledWith('env-key'); + }); + + it('returns false when the API key is missing', async () => { + mockedFetchSecrets.mockResolvedValue({}); + + await expect(sendWithResend(emailDto)).resolves.toBe(false); + expect(MockedResend).not.toHaveBeenCalled(); + }); + + it('returns false when the send response has no id', async () => { + mockedFetchSecrets.mockResolvedValue({ RESEND_API_KEY: 'resend-key' }); + sendMock.mockResolvedValue({ data: {} }); + + await expect(sendWithResend(emailDto)).resolves.toBe(false); + }); + + it('returns false when the send rejects', async () => { + mockedFetchSecrets.mockResolvedValue({ RESEND_API_KEY: 'resend-key' }); + sendMock.mockRejectedValue(new Error('rate limited')); + + await expect(sendWithResend(emailDto)).resolves.toBe(false); + }); +}); diff --git a/libs/common/src/resend-helper-file.ts b/libs/common/src/resend-helper-file.ts index 4d0326bd8c..9950ee96a8 100644 --- a/libs/common/src/resend-helper-file.ts +++ b/libs/common/src/resend-helper-file.ts @@ -1,26 +1,21 @@ -import * as dotenv from 'dotenv'; - +import { CommonConstants } from './common.constant'; import { EmailDto } from './dtos/email.dto'; import { Logger } from '@nestjs/common'; import { Resend } from 'resend'; - -dotenv.config(); - -const emailProvider = process.env.EMAIL_PROVIDER; -const apiKey = process.env.RESEND_API_KEY; - -let resend: Resend | null = null; - -if ('resend' === emailProvider) { - if (!apiKey) { - throw new Error('Missing RESEND_API_KEY in environment variables.'); - } - resend = new Resend(apiKey); -} +import { fetchSecrets } from './utils/secretLoader.util'; export const sendWithResend = async (emailDto: EmailDto): Promise => { try { - const response = await resend.emails.send({ + const secretPath = CommonConstants.CREDEBL_RESEND_API_KEY_PATH; + const secrets = await fetchSecrets(secretPath); + const apiKey = secrets.RESEND_API_KEY ?? process.env.RESEND_API_KEY; + + if (!apiKey) { + throw new Error('Missing RESEND_API_KEY in secret payload.'); + } + + const client = new Resend(apiKey); + const response = await client.emails.send({ from: emailDto.emailFrom, to: emailDto.emailTo, subject: emailDto.emailSubject, @@ -28,7 +23,6 @@ export const sendWithResend = async (emailDto: EmailDto): Promise => { html: emailDto.emailHtml, attachments: emailDto.emailAttachments }); - return Boolean(response.data?.id); } catch (error) { Logger.error('Error while sending email with Resend', error); diff --git a/libs/common/src/send-grid-helper-file.spec.ts b/libs/common/src/send-grid-helper-file.spec.ts new file mode 100644 index 0000000000..cbfb527890 --- /dev/null +++ b/libs/common/src/send-grid-helper-file.spec.ts @@ -0,0 +1,73 @@ +import { sendWithSendGrid } from './send-grid-helper-file'; +import { fetchSecrets } from './utils/secretLoader.util'; + +jest.mock('@sendgrid/mail', () => ({ + setApiKey: jest.fn(), + send: jest.fn() +})); +jest.mock('./utils/secretLoader.util', () => ({ + fetchSecrets: jest.fn() +})); + +import * as sendgrid from '@sendgrid/mail'; +import { EmailDto } from './dtos/email.dto'; + +const mockedSetApiKey = sendgrid.setApiKey as jest.Mock; +const mockedSend = sendgrid.send as jest.Mock; +const mockedFetchSecrets = fetchSecrets as jest.MockedFunction; + +const emailDto: EmailDto = { + emailFrom: 'from@credebl.id', + emailTo: ['to@example.com'], + emailSubject: 'subject', + emailText: 'text', + emailHtml: '

html

' +}; + +describe('sendWithSendGrid', () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + delete process.env.SENDGRID_API_KEY; + jest.clearAllMocks(); + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + + it('sends an email using the provider-sourced API key', async () => { + mockedFetchSecrets.mockResolvedValue({ SENDGRID_API_KEY: 'sg-key' }); + mockedSend.mockResolvedValue(true); + + await expect(sendWithSendGrid(emailDto)).resolves.toBe(true); + + expect(mockedFetchSecrets).toHaveBeenCalledWith('secret/data/credebl_sendgrid_api_key'); + expect(mockedSetApiKey).toHaveBeenCalledWith('sg-key'); + expect(mockedSend).toHaveBeenCalledWith(expect.objectContaining({ to: emailDto.emailTo })); + }); + + it('falls back to the environment variable when no secret is returned', async () => { + mockedFetchSecrets.mockResolvedValue({}); + process.env.SENDGRID_API_KEY = 'env-key'; + mockedSend.mockResolvedValue(true); + + await expect(sendWithSendGrid(emailDto)).resolves.toBe(true); + + expect(mockedSetApiKey).toHaveBeenCalledWith('env-key'); + }); + + it('returns false when the API key is missing', async () => { + mockedFetchSecrets.mockResolvedValue({}); + + await expect(sendWithSendGrid(emailDto)).resolves.toBe(false); + expect(mockedSetApiKey).not.toHaveBeenCalled(); + }); + + it('returns false when send rejects', async () => { + mockedFetchSecrets.mockResolvedValue({ SENDGRID_API_KEY: 'sg-key' }); + mockedSend.mockRejectedValue(new Error('rate limited')); + + await expect(sendWithSendGrid(emailDto)).resolves.toBe(false); + }); +}); diff --git a/libs/common/src/send-grid-helper-file.ts b/libs/common/src/send-grid-helper-file.ts index 3ae07faf1f..01e3f666d2 100644 --- a/libs/common/src/send-grid-helper-file.ts +++ b/libs/common/src/send-grid-helper-file.ts @@ -1,14 +1,22 @@ -import * as dotenv from 'dotenv'; import * as sendgrid from '@sendgrid/mail'; +import { CommonConstants } from './common.constant'; import { EmailDto } from './dtos/email.dto'; - -dotenv.config(); - -sendgrid.setApiKey(process.env.SENDGRID_API_KEY); +import { Logger } from '@nestjs/common'; +import { fetchSecrets } from './utils/secretLoader.util'; export const sendWithSendGrid = async (EmailDto: EmailDto): Promise => { try { + const secretPath = CommonConstants.CREDEBL_SENDGRID_API_KEY_PATH; + const secrets = await fetchSecrets(secretPath); + const apiKey = secrets.SENDGRID_API_KEY ?? process.env.SENDGRID_API_KEY; + + if (!apiKey) { + throw new Error('Missing SENDGRID_API_KEY in secret payload.'); + } + + sendgrid.setApiKey(apiKey); + const msg = { to: EmailDto.emailTo, from: EmailDto.emailFrom, @@ -22,6 +30,7 @@ export const sendWithSendGrid = async (EmailDto: EmailDto): Promise => .then(() => true) .catch(() => false); } catch (error) { + Logger.error('Error while sending email with SendGrid', error); return false; } }; diff --git a/libs/common/src/smtp-helper-file.spec.ts b/libs/common/src/smtp-helper-file.spec.ts new file mode 100644 index 0000000000..a89b948d90 --- /dev/null +++ b/libs/common/src/smtp-helper-file.spec.ts @@ -0,0 +1,122 @@ +import { sendWithSMTP } from './smtp-helper-file'; +import { fetchSecrets } from './utils/secretLoader.util'; + +jest.mock('nodemailer', () => ({ + createTransport: jest.fn() +})); +jest.mock('./utils/secretLoader.util', () => ({ + fetchSecrets: jest.fn() +})); + +import * as nodemailer from 'nodemailer'; +import { EmailDto } from './dtos/email.dto'; + +const mockedCreateTransport = nodemailer.createTransport as jest.Mock; +const mockedFetchSecrets = fetchSecrets as jest.MockedFunction; + +const emailDto: EmailDto = { + emailFrom: 'from@credebl.id', + emailTo: ['to@example.com'], + emailSubject: 'subject', + emailText: 'text', + emailHtml: '

html

' +}; + +describe('sendWithSMTP', () => { + const originalEnv = { ...process.env }; + let sendMail: jest.Mock; + + beforeEach(() => { + delete process.env.SMTP_HOST; + delete process.env.SMTP_PORT; + delete process.env.SMTP_USER; + delete process.env.SMTP_PASS; + sendMail = jest.fn().mockResolvedValue({ messageId: 'm1' }); + mockedCreateTransport.mockReturnValue({ sendMail }); + jest.clearAllMocks(); + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + + it('sends an email using provider-sourced SMTP configuration', async () => { + mockedFetchSecrets.mockResolvedValue({ + SMTP_HOST: 'smtp.example.com', + SMTP_PORT: '587', + SMTP_USER: 'user', + SMTP_PASS: 'pass' + }); + + await expect(sendWithSMTP(emailDto)).resolves.toBe(true); + + expect(mockedFetchSecrets).toHaveBeenCalledWith('secret/data/credebl_smtp_config'); + expect(mockedCreateTransport).toHaveBeenCalledWith( + expect.objectContaining({ + host: 'smtp.example.com', + port: 587, + secure: false, + requireTLS: true, + auth: { user: 'user', pass: 'pass' } + }) + ); + expect(sendMail).toHaveBeenCalledWith(expect.objectContaining({ from: emailDto.emailFrom })); + }); + + it('sets secure mode for port 465', async () => { + mockedFetchSecrets.mockResolvedValue({ + SMTP_HOST: 'smtp.example.com', + SMTP_PORT: '465', + SMTP_USER: 'user', + SMTP_PASS: 'pass' + }); + + await sendWithSMTP(emailDto); + + expect(mockedCreateTransport).toHaveBeenCalledWith(expect.objectContaining({ secure: true, requireTLS: false })); + }); + + it('falls back to environment variables when no secrets are returned', async () => { + mockedFetchSecrets.mockResolvedValue({}); + process.env.SMTP_HOST = 'smtp.env.com'; + process.env.SMTP_PORT = '587'; + process.env.SMTP_USER = 'env-user'; + process.env.SMTP_PASS = 'env-pass'; + + await expect(sendWithSMTP(emailDto)).resolves.toBe(true); + + expect(mockedCreateTransport).toHaveBeenCalledWith( + expect.objectContaining({ host: 'smtp.env.com', auth: { user: 'env-user', pass: 'env-pass' } }) + ); + }); + + it('returns false when required SMTP configuration is missing', async () => { + mockedFetchSecrets.mockResolvedValue({}); + + await expect(sendWithSMTP(emailDto)).resolves.toBe(false); + expect(sendMail).not.toHaveBeenCalled(); + }); + + it('returns false when the SMTP port is invalid', async () => { + mockedFetchSecrets.mockResolvedValue({ + SMTP_HOST: 'smtp.example.com', + SMTP_PORT: 'not-a-number', + SMTP_USER: 'user', + SMTP_PASS: 'pass' + }); + + await expect(sendWithSMTP(emailDto)).resolves.toBe(false); + }); + + it('returns false when sendMail rejects', async () => { + mockedFetchSecrets.mockResolvedValue({ + SMTP_HOST: 'smtp.example.com', + SMTP_PORT: '587', + SMTP_USER: 'user', + SMTP_PASS: 'pass' + }); + sendMail.mockRejectedValue(new Error('connection refused')); + + await expect(sendWithSMTP(emailDto)).resolves.toBe(false); + }); +}); diff --git a/libs/common/src/smtp-helper-file.ts b/libs/common/src/smtp-helper-file.ts index be0559e0a1..1f629ad671 100644 --- a/libs/common/src/smtp-helper-file.ts +++ b/libs/common/src/smtp-helper-file.ts @@ -1,47 +1,40 @@ -import * as dotenv from 'dotenv'; import * as nodemailer from 'nodemailer'; +import { CommonConstants } from './common.constant'; import { EmailDto } from './dtos/email.dto'; import { Logger } from '@nestjs/common'; - -dotenv.config(); - -const emailProvider = process.env.EMAIL_PROVIDER?.toLowerCase(); - -let transporter: nodemailer.Transporter | null = null; - -if ('smtp' === emailProvider) { - const { SMTP_HOST, SMTP_PORT, SMTP_USER, SMTP_PASS } = process.env; - - if (!SMTP_HOST || !SMTP_PORT || !SMTP_USER || !SMTP_PASS) { - throw new Error('Missing SMTP configuration. Required: SMTP_HOST, SMTP_PORT, SMTP_USER, SMTP_PASS'); - } - - const port = Number(SMTP_PORT); - - if (!Number.isInteger(port) || 0 >= port) { - throw new Error(`Invalid SMTP_PORT value: "${SMTP_PORT}". Must be a valid number.`); - } - - transporter = nodemailer.createTransport({ - host: SMTP_HOST, - port: Number(SMTP_PORT), - secure: 465 === Number(SMTP_PORT), - auth: { - user: SMTP_USER, - pass: SMTP_PASS - }, - requireTLS: 587 === Number(SMTP_PORT) - }); -} +import { fetchSecrets } from './utils/secretLoader.util'; export const sendWithSMTP = async (emailDto: EmailDto): Promise => { - if (!transporter) { - Logger.error('SMTP email provider is not initialized'); - return false; - } - try { + const secretPath = CommonConstants.CREDEBL_SMTP_CONFIG_PATH; + const secrets = await fetchSecrets(secretPath); + const smtpHost = secrets.SMTP_HOST ?? process.env.SMTP_HOST; + const smtpPort = secrets.SMTP_PORT ?? process.env.SMTP_PORT; + const smtpUser = secrets.SMTP_USER ?? process.env.SMTP_USER; + const smtpPass = secrets.SMTP_PASS ?? process.env.SMTP_PASS; + + if (!smtpHost || !smtpPort || !smtpUser || !smtpPass) { + throw new Error('Missing SMTP configuration. Required: SMTP_HOST, SMTP_PORT, SMTP_USER, SMTP_PASS'); + } + + const port = Number(smtpPort); + + if (!Number.isInteger(port) || 0 >= port) { + throw new Error(`Invalid SMTP_PORT value: "${smtpPort}". Must be a valid number.`); + } + + const transporter = nodemailer.createTransport({ + host: smtpHost, + port, + secure: 465 === port, + auth: { + user: smtpUser, + pass: smtpPass + }, + requireTLS: 587 === port + }); + await transporter.sendMail({ from: emailDto.emailFrom, to: emailDto.emailTo, diff --git a/libs/common/src/utils/secretLoader.util.spec.ts b/libs/common/src/utils/secretLoader.util.spec.ts new file mode 100644 index 0000000000..7a9e0293c9 --- /dev/null +++ b/libs/common/src/utils/secretLoader.util.spec.ts @@ -0,0 +1,86 @@ +import { fetchSecrets } from './secretLoader.util'; +import { getSecretProvider } from 'libs/config/src/secret-storage/secrets-loader'; + +jest.mock('libs/config/src/secret-storage/secrets-loader', () => ({ + getSecretProvider: jest.fn() +})); + +const mockedGetSecretProvider = getSecretProvider as jest.MockedFunction; + +describe('fetchSecrets', () => { + const originalEnv = { ...process.env }; + + beforeEach(() => { + delete process.env.ENABLE_BAO; + delete process.env.SECRETS_PROVIDER; + jest.clearAllMocks(); + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + + it('returns an empty object when OpenBao is disabled', async () => { + process.env.ENABLE_BAO = 'false'; + + await expect(fetchSecrets('secret/data/credebl_smtp_config')).resolves.toEqual({}); + expect(mockedGetSecretProvider).not.toHaveBeenCalled(); + }); + + it('returns an empty object when SECRETS_PROVIDER is not set', async () => { + process.env.ENABLE_BAO = 'true'; + delete process.env.SECRETS_PROVIDER; + + await expect(fetchSecrets('secret/data/credebl_smtp_config_2')).resolves.toEqual({}); + expect(mockedGetSecretProvider).not.toHaveBeenCalled(); + }); + + it('returns an empty object when the provider is unsupported', async () => { + process.env.ENABLE_BAO = 'true'; + process.env.SECRETS_PROVIDER = 'consul'; + mockedGetSecretProvider.mockReturnValue(null); + + await expect(fetchSecrets('secret/data/credebl_smtp_config_3')).resolves.toEqual({}); + expect(mockedGetSecretProvider).toHaveBeenCalledWith('consul'); + }); + + it('fetches the secrets for the requested path via the active provider', async () => { + process.env.ENABLE_BAO = 'true'; + process.env.SECRETS_PROVIDER = 'openbao'; + const loadSecrets = jest.fn().mockResolvedValue({ SMTP_HOST: 'smtp.example.com', SMTP_PORT: '587' }); + mockedGetSecretProvider.mockReturnValue({ name: 'OpenBao', loadSecrets }); + + await expect(fetchSecrets('secret/data/credebl_smtp_config_4')).resolves.toEqual({ + SMTP_HOST: 'smtp.example.com', + SMTP_PORT: '587' + }); + + expect(loadSecrets).toHaveBeenCalledWith({ customPath: 'secret/data/credebl_smtp_config_4' }); + }); + + it('caches the result per path and does not re-invoke the provider', async () => { + process.env.ENABLE_BAO = 'true'; + process.env.SECRETS_PROVIDER = 'openbao'; + const loadSecrets = jest.fn().mockResolvedValue({ RESEND_API_KEY: 'k' }); + mockedGetSecretProvider.mockReturnValue({ name: 'OpenBao', loadSecrets }); + + await fetchSecrets('secret/data/credebl_resend_api_key'); + await fetchSecrets('secret/data/credebl_resend_api_key'); + + expect(loadSecrets).toHaveBeenCalledTimes(1); + }); + + it('uses an independent cache per secret path', async () => { + process.env.ENABLE_BAO = 'true'; + process.env.SECRETS_PROVIDER = 'openbao'; + const loadSecrets = jest.fn().mockResolvedValue({ SOME_KEY: 'v' }); + mockedGetSecretProvider.mockReturnValue({ name: 'OpenBao', loadSecrets }); + + await fetchSecrets('secret/data/path_a'); + await fetchSecrets('secret/data/path_b'); + + expect(loadSecrets).toHaveBeenCalledTimes(2); + expect(loadSecrets).toHaveBeenCalledWith({ customPath: 'secret/data/path_a' }); + expect(loadSecrets).toHaveBeenCalledWith({ customPath: 'secret/data/path_b' }); + }); +}); diff --git a/libs/common/src/utils/secretLoader.util.ts b/libs/common/src/utils/secretLoader.util.ts new file mode 100644 index 0000000000..63cd533525 --- /dev/null +++ b/libs/common/src/utils/secretLoader.util.ts @@ -0,0 +1,28 @@ +import { TtlCache } from './ttl-cache.util'; +import { getSecretProvider } from 'libs/config/src/secret-storage/secrets-loader'; + +const CACHE_TTL_MS = 10 * 60 * 1000; + +const secretCaches = new Map>>(); + +export async function fetchSecrets(secretPath: string): Promise> { + if ('true' !== process.env.ENABLE_BAO?.trim()?.toLowerCase()) { + return {}; + } + let cache = secretCaches.get(secretPath); + if (!cache) { + cache = new TtlCache>(CACHE_TTL_MS); + secretCaches.set(secretPath, cache); + } + return cache.get(async () => { + const providerType = process.env.SECRETS_PROVIDER?.trim(); + if (!providerType) { + return {}; + } + const provider = getSecretProvider(providerType); + if (!provider) { + return {}; + } + return provider.loadSecrets({ customPath: secretPath }); + }); +} diff --git a/libs/common/src/utils/ttl-cache.util.spec.ts b/libs/common/src/utils/ttl-cache.util.spec.ts new file mode 100644 index 0000000000..0551557408 --- /dev/null +++ b/libs/common/src/utils/ttl-cache.util.spec.ts @@ -0,0 +1,68 @@ +import { TtlCache } from './ttl-cache.util'; + +describe('TtlCache', () => { + beforeEach(() => { + jest.useFakeTimers(); + }); + + afterEach(() => { + jest.useRealTimers(); + }); + + it('calls the fetch function only once and returns the cached value on subsequent gets', async () => { + const cache = new TtlCache(1000); + const fetchFn = jest.fn().mockResolvedValue(42); + + await expect(cache.get(fetchFn)).resolves.toBe(42); + await expect(cache.get(fetchFn)).resolves.toBe(42); + + expect(fetchFn).toHaveBeenCalledTimes(1); + }); + + it('re-fetches after the ttl has elapsed', async () => { + const cache = new TtlCache(1000); + const fetchFn = jest.fn().mockResolvedValue(1); + + await cache.get(fetchFn); + expect(fetchFn).toHaveBeenCalledTimes(1); + + jest.advanceTimersByTime(1001); + fetchFn.mockResolvedValue(2); + + await expect(cache.get(fetchFn)).resolves.toBe(2); + expect(fetchFn).toHaveBeenCalledTimes(2); + }); + + it('treats the ttl as a sliding window while the value is read frequently', async () => { + const cache = new TtlCache(1000); + const fetchFn = jest.fn().mockResolvedValue(1); + + await cache.get(fetchFn); + + for (let i = 0; 5 > i; i += 1) { + jest.advanceTimersByTime(900); + await expect(cache.get(fetchFn)).resolves.toBe(1); + } + + expect(fetchFn).toHaveBeenCalledTimes(1); + }); + + it('clears the cached value and cancels the pending timer on clear()', async () => { + const cache = new TtlCache(1000); + const fetchFn = jest.fn().mockResolvedValue(1); + + await cache.get(fetchFn); + cache.clear(); + fetchFn.mockResolvedValue(2); + + await expect(cache.get(fetchFn)).resolves.toBe(2); + expect(fetchFn).toHaveBeenCalledTimes(2); + }); + + it('propagates a rejection from the fetch function', async () => { + const cache = new TtlCache(1000); + const fetchFn = jest.fn().mockRejectedValue(new Error('boom')); + + await expect(cache.get(fetchFn)).rejects.toThrow('boom'); + }); +}); diff --git a/libs/common/src/utils/ttl-cache.util.ts b/libs/common/src/utils/ttl-cache.util.ts new file mode 100644 index 0000000000..5574ea441b --- /dev/null +++ b/libs/common/src/utils/ttl-cache.util.ts @@ -0,0 +1,38 @@ +export class TtlCache { + private value: T | null = null; + private timer: ReturnType | null = null; + private readonly ttlMs: number; + + constructor(ttlMs: number) { + this.ttlMs = ttlMs; + } + + async get(fetchFn: () => Promise): Promise { + if (null !== this.value) { + this.refreshTimer(); + return this.value; + } + + this.value = await fetchFn(); + this.refreshTimer(); + return this.value; + } + + clear(): void { + this.value = null; + if (this.timer) { + clearTimeout(this.timer); + this.timer = null; + } + } + + private refreshTimer(): void { + if (this.timer) { + clearTimeout(this.timer); + } + this.timer = setTimeout(() => { + this.clear(); + }, this.ttlMs); + this.timer.unref(); + } +} diff --git a/libs/config/src/index.ts b/libs/config/src/index.ts index 8d518dbf6a..e5e70bc39e 100644 --- a/libs/config/src/index.ts +++ b/libs/config/src/index.ts @@ -1,3 +1,4 @@ export * from './config.service'; export * from './config.module'; -export * from './global-config.module'; \ No newline at end of file +export * from './global-config.module'; +export { loadConfigSecrets } from './secret-storage/secrets-loader'; diff --git a/libs/config/src/secret-storage/openbao.provider.spec.ts b/libs/config/src/secret-storage/openbao.provider.spec.ts new file mode 100644 index 0000000000..1bd8f0daaf --- /dev/null +++ b/libs/config/src/secret-storage/openbao.provider.spec.ts @@ -0,0 +1,130 @@ +import { OpenBaoProvider } from './openbao.provider'; + +describe('OpenBaoProvider', () => { + const originalEnv = { ...process.env }; + + let provider: OpenBaoProvider; + let fetchMock: jest.Mock; + + function jsonResponse(body: unknown, ok = true, status = 200): Response { + return { + ok, + status, + json: jest.fn().mockResolvedValue(body) + } as unknown as Response; + } + + beforeEach(() => { + provider = new OpenBaoProvider(); + process.env.BAO_URL = 'http://bao:8200'; + process.env.BAO_SECRET_PATH = 'secret/data/credebl_resend_api_key'; + process.env.BAO_ROLE_ID = 'role-id'; + process.env.BAO_SECRET_ID = 'secret-id'; + + fetchMock = jest.fn(); + global.fetch = fetchMock as unknown as typeof fetch; + }); + + afterEach(() => { + process.env = { ...originalEnv }; + jest.restoreAllMocks(); + }); + + it('rejects when any required environment variable is missing', async () => { + delete process.env.BAO_ROLE_ID; + + await expect(provider.loadSecrets()).rejects.toThrow( + 'BAO_URL, BAO_SECRET_PATH, BAO_ROLE_ID, and BAO_SECRET_ID must be set.' + ); + expect(fetchMock).not.toHaveBeenCalled(); + }); + + it('authenticates with AppRole and fetches secrets from the configured path', async () => { + fetchMock + .mockResolvedValueOnce(jsonResponse({ auth: { client_token: 'client-token' } })) + .mockResolvedValueOnce(jsonResponse({ data: { data: { RESEND_API_KEY: 'resend-key' } } })); + + await expect(provider.loadSecrets()).resolves.toEqual({ RESEND_API_KEY: 'resend-key' }); + + expect(fetchMock).toHaveBeenNthCalledWith( + 1, + 'http://bao:8200/v1/auth/approle/login', + expect.objectContaining({ + method: 'POST', + body: JSON.stringify({ role_id: 'role-id', secret_id: 'secret-id' }) + }) + ); + expect(fetchMock).toHaveBeenNthCalledWith( + 2, + 'http://bao:8200/v1/secret/data/credebl_resend_api_key', + expect.objectContaining({ + method: 'GET', + headers: expect.objectContaining({ 'X-Vault-Token': 'client-token' }) + }) + ); + }); + + it('fetches from a custom path when one is supplied', async () => { + fetchMock + .mockResolvedValueOnce(jsonResponse({ auth: { client_token: 'client-token' } })) + .mockResolvedValueOnce(jsonResponse({ data: { data: { KEY: 'value' } } })); + + await provider.loadSecrets({ customPath: 'secret/data/credebl_smtp_config' }); + + expect(fetchMock).toHaveBeenNthCalledWith( + 2, + 'http://bao:8200/v1/secret/data/credebl_smtp_config', + expect.anything() + ); + }); + + it('rejects when the AppRole login response is not ok', async () => { + fetchMock.mockResolvedValueOnce(jsonResponse({}, false, 500)); + + await expect(provider.loadSecrets()).rejects.toThrow('Authentication failed: Status 500'); + }); + + it('rejects when the login response carries no client token', async () => { + fetchMock.mockResolvedValueOnce(jsonResponse({ auth: {} })); + + await expect(provider.loadSecrets()).rejects.toThrow('Failed to retrieve client token from OpenBao.'); + }); + + it('rejects when the secrets fetch response is not ok', async () => { + fetchMock + .mockResolvedValueOnce(jsonResponse({ auth: { client_token: 'client-token' } })) + .mockResolvedValueOnce(jsonResponse({}, false, 404)); + + await expect(provider.loadSecrets()).rejects.toThrow('Fetch failed: Status 404'); + }); + + it('rejects when the payload has an unexpected structure', async () => { + fetchMock + .mockResolvedValueOnce(jsonResponse({ auth: { client_token: 'client-token' } })) + .mockResolvedValueOnce(jsonResponse({ data: {} })); + + await expect(provider.loadSecrets()).rejects.toThrow('Unexpected secrets payload structure.'); + }); + + it('rejects when the payload data is an array', async () => { + fetchMock + .mockResolvedValueOnce(jsonResponse({ auth: { client_token: 'client-token' } })) + .mockResolvedValueOnce(jsonResponse({ data: { data: [] } })); + + await expect(provider.loadSecrets()).rejects.toThrow('Unexpected secrets payload structure.'); + }); + + it('rejects with a timeout error when the login request is aborted', async () => { + fetchMock.mockRejectedValueOnce(new DOMException('The operation was aborted.', 'AbortError')); + + await expect(provider.loadSecrets()).rejects.toThrow('OpenBao authentication request timed out.'); + }); + + it('rejects with a timeout error when the secrets fetch is aborted', async () => { + fetchMock + .mockResolvedValueOnce(jsonResponse({ auth: { client_token: 'client-token' } })) + .mockRejectedValueOnce(new DOMException('The operation was aborted.', 'AbortError')); + + await expect(provider.loadSecrets()).rejects.toThrow('OpenBao secrets fetch request timed out.'); + }); +}); diff --git a/libs/config/src/secret-storage/openbao.provider.ts b/libs/config/src/secret-storage/openbao.provider.ts new file mode 100644 index 0000000000..2bffbe7fd0 --- /dev/null +++ b/libs/config/src/secret-storage/openbao.provider.ts @@ -0,0 +1,97 @@ +import { CommonConstants } from '@credebl/common/common.constant'; +import { Logger } from '@nestjs/common'; +import { SecretProvider } from './secret-provider.interface'; + +async function fetchWithTimeout(url: string, options: RequestInit, timeoutMs: number): Promise { + const controller = new AbortController(); + const timeoutId = setTimeout(() => controller.abort(), timeoutMs); + try { + const response = await fetch(url, { ...options, signal: controller.signal }); + return response; + } finally { + clearTimeout(timeoutId); + } +} + +export class OpenBaoProvider implements SecretProvider { + readonly name = 'OpenBao'; + private readonly logger = new Logger(OpenBaoProvider.name); + + isEnabled(): boolean { + return 'true' === process.env.ENABLE_BAO?.trim()?.toLowerCase(); + } + + async loadSecrets(options?: { customPath?: string }): Promise> { + const baoUrl = process.env.BAO_URL; + const secretPath = options?.customPath || process.env.BAO_SECRET_PATH; + const roleId = process.env.BAO_ROLE_ID; + const secretId = process.env.BAO_SECRET_ID; + if (!baoUrl || !secretPath || !roleId || !secretId) { + throw new Error('BAO_URL, BAO_SECRET_PATH, BAO_ROLE_ID, and BAO_SECRET_ID must be set.'); + } + this.logger.log(`Fetching secrets from ${baoUrl}/v1/${secretPath}`); + this.logger.log(`Authenticating with AppRole at ${baoUrl}/v1/auth/approle/login`); + // --- Authentication --- + let authResponse: Response; + try { + authResponse = await fetchWithTimeout( + `${baoUrl}/v1/auth/approle/login`, + { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + // eslint-disable-next-line camelcase + body: JSON.stringify({ role_id: roleId, secret_id: secretId }) + }, + CommonConstants.OPENBAO_REQUEST_TIMEOUT + ); + } catch (error) { + if (error instanceof DOMException && 'AbortError' === error.name) { + throw new Error('OpenBao authentication request timed out.'); + } + throw error; + } + if (!authResponse.ok) { + throw new Error(`Authentication failed: Status ${authResponse.status}`); + } + + const authData = await authResponse.json(); + const baoToken = authData.auth?.client_token; + + if (!baoToken) { + throw new Error('Failed to retrieve client token from OpenBao.'); + } + this.logger.log(`Fetching secrets from ${baoUrl}/v1/${secretPath}`); + // --- Fetch Secrets --- + let response: Response; + try { + response = await fetchWithTimeout( + `${baoUrl}/v1/${secretPath}`, + { + method: 'GET', + headers: { + 'X-Vault-Token': baoToken, + 'Content-Type': 'application/json' + } + }, + CommonConstants.OPENBAO_REQUEST_TIMEOUT + ); + } catch (error) { + if (error instanceof DOMException && 'AbortError' === error.name) { + throw new Error('OpenBao secrets fetch request timed out.'); + } + throw error; + } + if (!response.ok) { + throw new Error(`Fetch failed: Status ${response.status}`); + } + + const result = await response.json(); + const secrets = result.data?.data; + this.logger.log('Successfully fetched secrets from OpenBao'); + if (!secrets || 'object' !== typeof secrets || Array.isArray(secrets)) { + throw new Error('Unexpected secrets payload structure.'); + } + + return secrets as Record; + } +} diff --git a/libs/config/src/secret-storage/secret-provider.interface.ts b/libs/config/src/secret-storage/secret-provider.interface.ts new file mode 100644 index 0000000000..f982c868a1 --- /dev/null +++ b/libs/config/src/secret-storage/secret-provider.interface.ts @@ -0,0 +1,13 @@ +export interface SecretProvider { + /** + * A human-readable name used for logging purposes (e.g., 'AWS Secrets Manager') + */ + name: string; + + /** + * Fetches the secrets from the external service and returns them + * as a flat key-value object of strings. + * * @returns A promise resolving to Record + */ + loadSecrets(options?: Record): Promise>; +} diff --git a/libs/config/src/secret-storage/secrets-loader.spec.ts b/libs/config/src/secret-storage/secrets-loader.spec.ts new file mode 100644 index 0000000000..47c091210e --- /dev/null +++ b/libs/config/src/secret-storage/secrets-loader.spec.ts @@ -0,0 +1,104 @@ +import { OpenBaoProvider } from './openbao.provider'; +import { getSecretProvider, loadConfigSecrets } from './secrets-loader'; + +jest.mock('./openbao.provider', () => ({ + OpenBaoProvider: jest.fn() +})); + +const MockedOpenBaoProvider = OpenBaoProvider as unknown as jest.Mock; + +describe('getSecretProvider', () => { + it('returns an OpenBaoProvider for the "openbao" value', () => { + expect(getSecretProvider('openbao')).toBeInstanceOf(OpenBaoProvider); + }); + + it('matches the provider type case-insensitively', () => { + expect(getSecretProvider('OpenBao')).toBeInstanceOf(OpenBaoProvider); + expect(getSecretProvider('OPENBAO')).toBeInstanceOf(OpenBaoProvider); + }); + + it('returns null for unsupported provider types', () => { + expect(getSecretProvider('consul')).toBeNull(); + expect(getSecretProvider('aws')).toBeNull(); + expect(getSecretProvider('')).toBeNull(); + }); +}); + +describe('loadConfigSecrets', () => { + const originalEnv = { ...process.env }; + let loadSecrets: jest.Mock; + + beforeEach(() => { + delete process.env.ENABLE_BAO; + delete process.env.SECRETS_PROVIDER; + loadSecrets = jest.fn(); + MockedOpenBaoProvider.mockImplementation(() => ({ + name: 'OpenBao', + loadSecrets + })); + jest.clearAllMocks(); + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + + it('skips remote fetching when ENABLE_BAO is not "true"', async () => { + process.env.ENABLE_BAO = 'false'; + process.env.SECRETS_PROVIDER = 'openbao'; + + await loadConfigSecrets(); + + expect(loadSecrets).not.toHaveBeenCalled(); + }); + + it('skips remote fetching when SECRETS_PROVIDER is not set', async () => { + process.env.ENABLE_BAO = 'true'; + delete process.env.SECRETS_PROVIDER; + + await loadConfigSecrets(); + + expect(loadSecrets).not.toHaveBeenCalled(); + }); + + it('logs a warning and continues for an unsupported provider', async () => { + process.env.ENABLE_BAO = 'true'; + process.env.SECRETS_PROVIDER = 'consul'; + + await expect(loadConfigSecrets()).resolves.toBeUndefined(); + + expect(loadSecrets).not.toHaveBeenCalled(); + }); + + it('injects fetched secrets into process.env', async () => { + process.env.ENABLE_BAO = 'true'; + process.env.SECRETS_PROVIDER = 'openbao'; + loadSecrets.mockResolvedValue({ SMTP_HOST: 'smtp.example.com', SMTP_PASS: 'secret' }); + + await loadConfigSecrets(); + + expect(process.env.SMTP_HOST).toBe('smtp.example.com'); + expect(process.env.SMTP_PASS).toBe('secret'); + }); + + it('skips forbidden prototype-pollution keys during injection', async () => { + process.env.ENABLE_BAO = 'true'; + process.env.SECRETS_PROVIDER = 'openbao'; + loadSecrets.mockResolvedValue({ __proto__: { polluted: true }, constructor: 'x', SAFE: 'y' }); + + await loadConfigSecrets(); + + expect((process.env as Record & { polluted?: boolean }).polluted).toBeUndefined(); + expect(process.env.SAFE).toBe('y'); + }); + + it('logs a critical boot failure and resolves when the provider rejects', async () => { + process.env.ENABLE_BAO = 'true'; + process.env.SECRETS_PROVIDER = 'openbao'; + loadSecrets.mockRejectedValue(new Error('BAO is down')); + + await expect(loadConfigSecrets()).resolves.toBeUndefined(); + + expect(process.env.SMTP_HOST).toBeUndefined(); + }); +}); diff --git a/libs/config/src/secret-storage/secrets-loader.ts b/libs/config/src/secret-storage/secrets-loader.ts new file mode 100644 index 0000000000..1d9c546ad4 --- /dev/null +++ b/libs/config/src/secret-storage/secrets-loader.ts @@ -0,0 +1,64 @@ +import { Logger } from '@nestjs/common'; +import { OpenBaoProvider } from './openbao.provider'; +import { SecretProvider } from './secret-provider.interface'; + +const FORBIDDEN_KEYS = new Set(['__proto__', 'constructor', 'prototype']); +const logger = new Logger('SecretsLoader'); + +/** + * Factory function to get the requested secret provider + */ +export function getSecretProvider(providerType: string): SecretProvider | null { + if ('openbao' === providerType.toLowerCase()) { + return new OpenBaoProvider(); + } + + return null; +} + +export async function loadConfigSecrets(): Promise { + if ('true' !== process.env.ENABLE_BAO?.trim()?.toLowerCase()) { + logger.log('OpenBao secrets management is disabled. Skipping remote secrets fetching.'); + return; + } + const providerType = process.env.SECRETS_PROVIDER?.trim(); + + if (!providerType) { + logger.log('SECRETS_PROVIDER not set. Skipping remote secrets fetching.'); + return; + } + + const provider = getSecretProvider(providerType); + + if (!provider) { + logger.warn( + `⚠️ Unsupported SECRETS_PROVIDER value "${providerType}". Falling back to local environment variables.` + ); + return; + } + + logger.log(`Secrets management enabled via [${provider.name}]. Fetching...`); + + try { + const secrets = await provider.loadSecrets(); + + for (const [key, value] of Object.entries(secrets)) { + if (FORBIDDEN_KEYS.has(key)) { + continue; + } + + Object.defineProperty(process.env, key, { + value: String(value), + enumerable: true, + configurable: true, + writable: true + }); + } + + logger.log(`Successfully injected secrets from ${provider.name} into process.env`); + } catch (error) { + const errorMessage = error instanceof Error ? error.message : String(error); + logger.error(`Critical Lifecycle Boot Failure (${provider.name}): ${errorMessage}`); + logger.error(`Failed to load secrets from ${provider.name}`); + } +} diff --git a/libs/enum/src/enum.ts b/libs/enum/src/enum.ts index e0ec32df28..5dab5c1623 100755 --- a/libs/enum/src/enum.ts +++ b/libs/enum/src/enum.ts @@ -373,7 +373,8 @@ export enum X509ExtendedKeyUsage { export enum CredentialFormat { SdJwtVc = 'dc+sd-jwt', Mdoc = 'mso_mdoc', - JwtVcJsonLd = 'jwt_vc_json-ld' + JwtVcJsonLd = 'jwt_vc_json-ld', + LdpVc = 'ldp_vc' } export enum AttributeType { diff --git a/libs/storage/src/providers/base-s3-storage.provider.ts b/libs/storage/src/providers/base-s3-storage.provider.ts index 4403130d38..fbb55b9b3e 100644 --- a/libs/storage/src/providers/base-s3-storage.provider.ts +++ b/libs/storage/src/providers/base-s3-storage.provider.ts @@ -6,20 +6,32 @@ import { RpcException } from '@nestjs/microservices'; import { S3 } from 'aws-sdk'; import { promisify } from 'node:util'; -export abstract class BaseS3StorageService implements IStorageService { - protected s3: S3; - protected s4: S3; - protected s3StoreObject: S3; +function extToMimeType(ext: string): string { + const mime: Record = { + png: 'image/png', + jpg: 'image/jpeg', + jpeg: 'image/jpeg', + gif: 'image/gif', + webp: 'image/webp', + svg: 'image/svg+xml', + bmp: 'image/bmp', + pdf: 'application/pdf', + csv: 'text/csv', + json: 'application/json', + txt: 'text/plain', + html: 'text/html', + xml: 'application/xml', + zip: 'application/zip', + doc: 'application/msword', + docx: 'application/vnd.openxmlformats-officedocument.wordprocessingml.document' + }; + return mime[ext.toLowerCase()] ?? 'application/octet-stream'; +} - constructor( - s3Config: Record, - s4Config: Record, - storeObjectConfig: Record - ) { - this.s3 = new S3(s3Config); - this.s4 = new S3(s4Config); - this.s3StoreObject = new S3(storeObjectConfig); - } +export abstract class BaseS3StorageService implements IStorageService { + protected abstract getS3Client(): Promise; + protected abstract getPublicS3Client(): Promise; + protected abstract getStoreObjectS3Client(): Promise; abstract getPublicUrl(bucketName: string, fileKey: string): string; @@ -31,8 +43,9 @@ export abstract class BaseS3StorageService implements IStorageService { encoding: string, pathAWS: string = '' ): Promise { + const s4 = await this.getPublicS3Client(); const timestamp = Date.now(); - const putObjectAsync = promisify(this.s4.putObject).bind(this.s4); + const putObjectAsync = promisify(s4.putObject).bind(s4); const fileKey = `${pathAWS}/${encodeURIComponent(filename)}-${timestamp}.${ext}`; try { await putObjectAsync({ @@ -40,7 +53,7 @@ export abstract class BaseS3StorageService implements IStorageService { Key: `${pathAWS}/${encodeURIComponent(filename)}-${timestamp}.${ext}`, Body: fileBuffer, ContentEncoding: encoding, - ContentType: `image/png` + ContentType: extToMimeType(ext) }); return this.getPublicUrl(bucketName, fileKey); } catch (error) { @@ -49,6 +62,7 @@ export abstract class BaseS3StorageService implements IStorageService { } async uploadCsvFile(key: string, body: unknown): Promise { + const s3 = await this.getS3Client(); let data: string; if ('string' === typeof body) { data = body; @@ -64,37 +78,40 @@ export abstract class BaseS3StorageService implements IStorageService { Body: data }; try { - await this.s3.upload(params).promise(); + await s3.upload(params).promise(); } catch (error) { throw new RpcException(error.response ? error.response : error); } } async getFile(key: string): Promise { + const s3 = await this.getS3Client(); const params: AWS.S3.GetObjectRequest = { Bucket: process.env.FILE_SHARING_BUCKET, Key: key }; try { - return this.s3.getObject(params).promise(); + return s3.getObject(params).promise(); } catch (error) { throw new RpcException(error.response ? error.response : error); } } async deleteFile(key: string): Promise { + const s3 = await this.getS3Client(); const params: AWS.S3.DeleteObjectRequest = { Bucket: process.env.FILE_SHARING_BUCKET, Key: key }; try { - await this.s3.deleteObject(params).promise(); + await s3.deleteObject(params).promise(); } catch (error) { throw new RpcException(error.response ? error.response : error); } } async storeObject(persistent: boolean, key: string, body: unknown): Promise { + const s3StoreObject = await this.getStoreObjectS3Client(); const objKey: string = persistent.valueOf() ? `persist/${key}` : `default/${key}`; const buf = Buffer.from(JSON.stringify(body)); const params: AWS.S3.PutObjectRequest = { @@ -106,7 +123,7 @@ export abstract class BaseS3StorageService implements IStorageService { }; try { - return await this.s3StoreObject.upload(params).promise(); + return await s3StoreObject.upload(params).promise(); } catch (error) { throw new RpcException(error.response ? error.response : error); } diff --git a/libs/storage/src/providers/rustfs-storage.provider.ts b/libs/storage/src/providers/rustfs-storage.provider.ts index c90287b952..9dee2d1448 100644 --- a/libs/storage/src/providers/rustfs-storage.provider.ts +++ b/libs/storage/src/providers/rustfs-storage.provider.ts @@ -1,27 +1,30 @@ import { BaseS3StorageService } from './base-s3-storage.provider'; import { Injectable } from '@nestjs/common'; +import { S3 } from 'aws-sdk'; @Injectable() export class RustFsStorageService extends BaseS3StorageService { - constructor() { - const endpoint = process.env.RUSTFS_ENDPOINT || 'http://localhost:9000'; + private readonly endpoint = process.env.RUSTFS_ENDPOINT || 'http://localhost:9000'; + private readonly baseConfig = { + accessKeyId: process.env.RUSTFS_ACCESS_KEY_ID, + secretAccessKey: process.env.RUSTFS_SECRET_ACCESS_KEY, + endpoint: this.endpoint, + s3ForcePathStyle: true + }; - const baseConfig = { - accessKeyId: process.env.RUSTFS_ACCESS_KEY_ID, - secretAccessKey: process.env.RUSTFS_SECRET_ACCESS_KEY, - endpoint, - s3ForcePathStyle: true - }; + protected async getS3Client(): Promise { + return new S3({ ...this.baseConfig, region: process.env.AWS_REGION }); + } - const s3Config = { ...baseConfig, region: process.env.AWS_REGION }; - const s4Config = { ...baseConfig, region: process.env.AWS_PUBLIC_REGION }; - const storeObjectConfig = { ...baseConfig, region: process.env.AWS_S3_STOREOBJECT_REGION }; + protected async getPublicS3Client(): Promise { + return new S3({ ...this.baseConfig, region: process.env.AWS_PUBLIC_REGION }); + } - super(s3Config, s4Config, storeObjectConfig); + protected async getStoreObjectS3Client(): Promise { + return new S3({ ...this.baseConfig, region: process.env.AWS_S3_STOREOBJECT_REGION }); } getPublicUrl(bucketName: string, fileKey: string): string { - const endpoint = process.env.RUSTFS_ENDPOINT || 'http://localhost:9000'; - return `${endpoint}/${bucketName}/${fileKey}`; + return `${this.endpoint}/${bucketName}/${fileKey}`; } } diff --git a/libs/storage/src/providers/s3-storage.provider.spec.ts b/libs/storage/src/providers/s3-storage.provider.spec.ts new file mode 100644 index 0000000000..5ff6c4c6ca --- /dev/null +++ b/libs/storage/src/providers/s3-storage.provider.spec.ts @@ -0,0 +1,108 @@ +import { S3StorageService } from './s3-storage.provider'; +import { fetchSecrets } from '@credebl/common/utils/secretLoader.util'; + +jest.mock('aws-sdk', () => ({ + S3: jest.fn() +})); +jest.mock('@credebl/common/utils/secretLoader.util', () => ({ + fetchSecrets: jest.fn() +})); + +import { S3 } from 'aws-sdk'; + +const MockedS3 = S3 as unknown as jest.Mock; +const mockedFetchSecrets = fetchSecrets as jest.MockedFunction; + +type ClientGetter = () => Promise; + +describe('S3StorageService', () => { + const originalEnv = { ...process.env }; + let service: S3StorageService; + + beforeEach(() => { + delete process.env.AWS_REGION; + delete process.env.AWS_ACCESS_KEY; + delete process.env.AWS_SECRET_KEY; + delete process.env.AWS_PUBLIC_REGION; + delete process.env.AWS_PUBLIC_ACCESS_KEY; + delete process.env.AWS_PUBLIC_SECRET_KEY; + delete process.env.AWS_S3_STOREOBJECT_REGION; + delete process.env.AWS_S3_STOREOBJECT_ACCESS_KEY; + delete process.env.AWS_S3_STOREOBJECT_SECRET_KEY; + service = new S3StorageService(); + jest.clearAllMocks(); + }); + + afterEach(() => { + process.env = { ...originalEnv }; + }); + + it('builds the internal S3 client from provider-sourced credentials', async () => { + mockedFetchSecrets.mockResolvedValue({ AWS_ACCESS_KEY: 'secret-key', AWS_SECRET_KEY: 'secret-secret' }); + process.env.AWS_REGION = 'us-east-1'; + + await (service as unknown as { getS3Client: ClientGetter }).getS3Client(); + + expect(mockedFetchSecrets).toHaveBeenCalledWith('secret/data/credebl_aws_keys'); + expect(MockedS3).toHaveBeenCalledWith({ + accessKeyId: 'secret-key', + secretAccessKey: 'secret-secret', + region: 'us-east-1' + }); + }); + + it('falls back to environment variables for the internal client', async () => { + mockedFetchSecrets.mockResolvedValue({}); + process.env.AWS_REGION = 'us-east-1'; + process.env.AWS_ACCESS_KEY = 'env-key'; + process.env.AWS_SECRET_KEY = 'env-secret'; + + await (service as unknown as { getS3Client: ClientGetter }).getS3Client(); + + expect(MockedS3).toHaveBeenCalledWith({ + accessKeyId: 'env-key', + secretAccessKey: 'env-secret', + region: 'us-east-1' + }); + }); + + it('builds the public S3 client from the public credential pair', async () => { + mockedFetchSecrets.mockResolvedValue({ + AWS_PUBLIC_ACCESS_KEY: 'public-key', + AWS_PUBLIC_SECRET_KEY: 'public-secret' + }); + process.env.AWS_PUBLIC_REGION = 'ap-south-1'; + + await (service as unknown as { getPublicS3Client: ClientGetter }).getPublicS3Client(); + + expect(MockedS3).toHaveBeenCalledWith({ + accessKeyId: 'public-key', + secretAccessKey: 'public-secret', + region: 'ap-south-1' + }); + }); + + it('builds the store-object S3 client from the store-object credential pair', async () => { + mockedFetchSecrets.mockResolvedValue({ + AWS_S3_STOREOBJECT_ACCESS_KEY: 'store-key', + AWS_S3_STOREOBJECT_SECRET_KEY: 'store-secret' + }); + process.env.AWS_S3_STOREOBJECT_REGION = 'eu-west-1'; + + await (service as unknown as { getStoreObjectS3Client: ClientGetter }).getStoreObjectS3Client(); + + expect(MockedS3).toHaveBeenCalledWith({ + accessKeyId: 'store-key', + secretAccessKey: 'store-secret', + region: 'eu-west-1' + }); + }); + + it('builds the public URL from the bucket and public region', () => { + process.env.AWS_PUBLIC_REGION = 'ap-south-1'; + + expect(service.getPublicUrl('my-bucket', 'logos/logo.png')).toBe( + 'https://my-bucket.s3.ap-south-1.amazonaws.com/logos/logo.png' + ); + }); +}); diff --git a/libs/storage/src/providers/s3-storage.provider.ts b/libs/storage/src/providers/s3-storage.provider.ts index 546c82bdc0..1fba7d5b09 100644 --- a/libs/storage/src/providers/s3-storage.provider.ts +++ b/libs/storage/src/providers/s3-storage.provider.ts @@ -1,28 +1,36 @@ import { BaseS3StorageService } from './base-s3-storage.provider'; +import { CommonConstants } from 'libs/common/src/common.constant'; import { Injectable } from '@nestjs/common'; +import { S3 } from 'aws-sdk'; +import { fetchSecrets } from '@credebl/common/utils/secretLoader.util'; @Injectable() export class S3StorageService extends BaseS3StorageService { - constructor() { - const s3Config = { - accessKeyId: process.env.AWS_ACCESS_KEY, - secretAccessKey: process.env.AWS_SECRET_KEY, + protected async getS3Client(): Promise { + const secrets = await fetchSecrets(CommonConstants.CREDEBL_AWS_KEY_PATH); + return new S3({ + accessKeyId: secrets.AWS_ACCESS_KEY ?? process.env.AWS_ACCESS_KEY, + secretAccessKey: secrets.AWS_SECRET_KEY ?? process.env.AWS_SECRET_KEY, region: process.env.AWS_REGION - }; + }); + } - const s4Config = { - accessKeyId: process.env.AWS_PUBLIC_ACCESS_KEY, - secretAccessKey: process.env.AWS_PUBLIC_SECRET_KEY, + protected async getPublicS3Client(): Promise { + const secrets = await fetchSecrets(CommonConstants.CREDEBL_AWS_KEY_PATH); + return new S3({ + accessKeyId: secrets.AWS_PUBLIC_ACCESS_KEY ?? process.env.AWS_PUBLIC_ACCESS_KEY, + secretAccessKey: secrets.AWS_PUBLIC_SECRET_KEY ?? process.env.AWS_PUBLIC_SECRET_KEY, region: process.env.AWS_PUBLIC_REGION - }; + }); + } - const storeObjectConfig = { - accessKeyId: process.env.AWS_S3_STOREOBJECT_ACCESS_KEY, - secretAccessKey: process.env.AWS_S3_STOREOBJECT_SECRET_KEY, + protected async getStoreObjectS3Client(): Promise { + const secrets = await fetchSecrets(CommonConstants.CREDEBL_AWS_KEY_PATH); + return new S3({ + accessKeyId: secrets.AWS_S3_STOREOBJECT_ACCESS_KEY ?? process.env.AWS_S3_STOREOBJECT_ACCESS_KEY, + secretAccessKey: secrets.AWS_S3_STOREOBJECT_SECRET_KEY ?? process.env.AWS_S3_STOREOBJECT_SECRET_KEY, region: process.env.AWS_S3_STOREOBJECT_REGION - }; - - super(s3Config, s4Config, storeObjectConfig); + }); } getPublicUrl(bucketName: string, fileKey: string): string { diff --git a/openbao-init.sh b/openbao-init.sh new file mode 100755 index 0000000000..7ab376dad1 --- /dev/null +++ b/openbao-init.sh @@ -0,0 +1,165 @@ +#!/usr/bin/env bash +# +# Bootstrap OpenBao (docker-compose.openbao.yml) for the CREDEBL platform demo. +# +# Provisions, idempotently: +# - OpenBao initialization + unseal (if not already initialized) +# - KV v2 secrets engine at secret/ +# - AppRole auth method +# - a "credebl" AppRole role + policy scoped to the credebl_* secret paths +# - the platform's secret data, sourced from the matching environment variables +# +# Usage: +# ./openbao-init.sh +# +# On the first run the script prints the unseal key, root token, role_id and +# secret_id. Copy BAO_ROLE_ID and BAO_SECRET_ID into .env.demo. +# On later runs it is idempotent: it re-prints role_id and re-uses an existing +# secret_id instead of generating a new one. +# +# Optional inputs (read from the environment): +# BAO_UNSEAL_KEY - unseal key from a previous run (needed if already +# initialized but sealed) +# BAO_ROOT_TOKEN - root token from a previous run (avoids re-printing) +# CONTAINER_NAME - docker container name (default: openbao) +# RESEND_API_KEY, SENDGRID_API_KEY, SMTP_HOST/PORT/USER/PASS, +# AWS_* - secret values to store; only the ones that are set +# (non-empty) are written +# +# NOTE: for demo only. The unseal key and root token are secrets; do not commit +# them to the repository. + +set -euo pipefail + +CONTAINER_NAME="${CONTAINER_NAME:-openbao}" +BAO_ADDR_IN_CONTAINER="${BAO_ADDR_IN_CONTAINER:-http://127.0.0.1:8200}" + +run_bao() { + docker exec -e BAO_TOKEN="${BAO_TOKEN:-}" "$CONTAINER_NAME" bao "$@" +} + +echo_step() { + local message="$1" + printf '\n==> %s\n' "$message" +} + +echo "Connecting to OpenBao in container '${CONTAINER_NAME}' (${BAO_ADDR_IN_CONTAINER})..." + +status="$(docker exec "$CONTAINER_NAME" bao status 2>&1 || true)" +initialized="$(grep -E '^Initialized' <<<"$status" | awk '{print $2}')" +sealed="$(grep -E '^Sealed' <<<"$status" | awk '{print $2}')" + +if [[ "$initialized" != "true" ]]; then + echo_step "OpenBao is not initialized. Running 'bao operator init' (1 unseal key)..." + init_out="$(docker exec "$CONTAINER_NAME" bao operator init -key-shares=1 -key-threshold=1)" + BAO_UNSEAL_KEY="${BAO_UNSEAL_KEY:-$(grep 'Unseal Key 1:' <<<"$init_out" | awk '{print $NF}')}" + BAO_ROOT_TOKEN="${BAO_ROOT_TOKEN:-$(grep 'Initial Root Token:' <<<"$init_out" | awk '{print $NF}')}" +else + BAO_UNSEAL_KEY="${BAO_UNSEAL_KEY:-}" + BAO_ROOT_TOKEN="${BAO_ROOT_TOKEN:-}" +fi + +if [[ "$sealed" == "true" ]]; then + if [[ -z "$BAO_UNSEAL_KEY" ]]; then + echo "ERROR: OpenBao is sealed. Re-run with BAO_UNSEAL_KEY= (from the first run)." >&2 + exit 1 + fi + echo_step "Unsealing OpenBao..." + docker exec "$CONTAINER_NAME" bao operator unseal "$BAO_UNSEAL_KEY" >/dev/null +fi + +if [[ -z "$BAO_ROOT_TOKEN" ]]; then + echo "ERROR: no root token available. Re-run with BAO_ROOT_TOKEN= (from the first run)." >&2 + exit 1 +fi +BAO_TOKEN="$BAO_ROOT_TOKEN" + +if ! run_bao secrets list 2>/dev/null | grep -q '^secret/'; then + echo_step "Enabling KV v2 secrets engine at secret/..." + run_bao secrets enable -path=secret kv-v2 +fi + +if ! run_bao auth list 2>/dev/null | grep -q '^approle/'; then + echo_step "Enabling AppRole auth method..." + run_bao auth enable approle +fi + +if ! run_bao policy list 2>/dev/null | grep -qx 'credebl-secrets'; then + echo_step "Writing 'credebl-secrets' policy..." + docker exec -i -e BAO_TOKEN="$BAO_TOKEN" "$CONTAINER_NAME" bao policy write credebl-secrets - <<'POLICY' +path "secret/data/credebl_*" { + capabilities = ["read"] +} +path "secret/metadata/credebl_*" { + capabilities = ["read", "list"] +} +POLICY +fi + +role_exists="false" +if run_bao read auth/approle/role/credebl >/dev/null 2>&1; then + role_exists="true" +fi + +if [[ "$role_exists" == "false" ]]; then + echo_step "Creating AppRole role 'credebl' (non-expiring secret_id, read-only policy)..." + run_bao write auth/approle/role/credebl \ + token_policies=credebl-secrets \ + secret_id_ttl=0 \ + token_ttl=30m \ + token_max_ttl=1h +fi + +role_id="$(run_bao read -field=role_id auth/approle/role/credebl/role-id)" + +if [[ "$role_exists" == "false" ]]; then + echo_step "Generating secret_id for the 'credebl' role..." + secret_id="$(run_bao write -f -field=secret_id auth/approle/role/credebl/secret-id)" +else + secret_id="${BAO_SECRET_ID:-}" +fi + +echo_step "Storing platform secrets..." + +kv_put() { + local path="$1" + shift + if [[ "$#" -gt 0 ]]; then + docker exec -e BAO_TOKEN="$BAO_TOKEN" "$CONTAINER_NAME" bao kv put "$path" "$@" >/dev/null + echo " wrote ${path}" + else + echo " skipped ${path} (no matching environment variables set)" + fi +} + +kv_put secret/credebl_resend_api_key \ + $( [[ -n "${RESEND_API_KEY:-}" ]] && printf 'RESEND_API_KEY=%s' "$RESEND_API_KEY" ) + +kv_put secret/credebl_sendgrid_api_key \ + $( [[ -n "${SENDGRID_API_KEY:-}" ]] && printf 'SENDGRID_API_KEY=%s' "$SENDGRID_API_KEY" ) + +kv_put secret/credebl_smtp_config \ + $( [[ -n "${SMTP_HOST:-}" ]] && printf 'SMTP_HOST=%s' "$SMTP_HOST" ) \ + $( [[ -n "${SMTP_PORT:-}" ]] && printf 'SMTP_PORT=%s' "$SMTP_PORT" ) \ + $( [[ -n "${SMTP_USER:-}" ]] && printf 'SMTP_USER=%s' "$SMTP_USER" ) \ + $( [[ -n "${SMTP_PASS:-}" ]] && printf 'SMTP_PASS=%s' "$SMTP_PASS" ) + +kv_put secret/credebl_aws_keys \ + $( [[ -n "${AWS_ACCESS_KEY:-}" ]] && printf 'AWS_ACCESS_KEY=%s' "$AWS_ACCESS_KEY" ) \ + $( [[ -n "${AWS_SECRET_KEY:-}" ]] && printf 'AWS_SECRET_KEY=%s' "$AWS_SECRET_KEY" ) \ + $( [[ -n "${AWS_PUBLIC_ACCESS_KEY:-}" ]] && printf 'AWS_PUBLIC_ACCESS_KEY=%s' "$AWS_PUBLIC_ACCESS_KEY" ) \ + $( [[ -n "${AWS_PUBLIC_SECRET_KEY:-}" ]] && printf 'AWS_PUBLIC_SECRET_KEY=%s' "$AWS_PUBLIC_SECRET_KEY" ) \ + $( [[ -n "${AWS_S3_STOREOBJECT_ACCESS_KEY:-}" ]] && printf 'AWS_S3_STOREOBJECT_ACCESS_KEY=%s' "$AWS_S3_STOREOBJECT_ACCESS_KEY" ) \ + $( [[ -n "${AWS_S3_STOREOBJECT_SECRET_KEY:-}" ]] && printf 'AWS_S3_STOREOBJECT_SECRET_KEY=%s' "$AWS_S3_STOREOBJECT_SECRET_KEY" ) + +echo_step "Done. Add the following to .env.demo:" +cat <)` method returning a flat `Record` of key/value secrets. +- **Must** expose a `getSecretProvider(providerType)` factory that returns a provider for the value `openbao` (case-insensitive) and `null` for any other value. +- **Will** support future providers (e.g. AWS Secrets Manager) via the same factory without call-site changes. + +#### Scenario: unknown provider value + +- **When** `getSecretProvider` is called with a value other than `openbao` +- **Then** it returns `null`. + +#### Scenario: case-insensitive provider lookup + +- **When** `getSecretProvider` is called with `OpenBao` or `OPENBAO` +- **Then** it returns an `OpenBaoProvider` instance. + +### openbao-provider-configuration + +The OpenBao provider reads its connection settings from the process environment. + +- **Must** read `BAO_URL`, `BAO_SECRET_PATH`, `BAO_ROLE_ID`, and `BAO_SECRET_ID` from `process.env`. +- **Must** throw an error listing all required variables when any of them is missing or empty. +- **Must** allow the secret path to be overridden per call via a `customPath` option passed to `loadSecrets`. +- **Must** enforce a request timeout of `CommonConstants.OPENBAO_REQUEST_TIMEOUT` (10 seconds) on every OpenBao HTTP request. +- **Will** treat a request aborted by the timeout as a failure and throw a `...timed out` error. + +#### Scenario: missing environment configuration + +- **When** any of `BAO_URL`, `BAO_SECRET_PATH`, `BAO_ROLE_ID`, `BAO_SECRET_ID` is unset +- **Then** `loadSecrets` rejects with an error naming the missing variables. + +#### Scenario: custom secret path + +- **When** `loadSecrets` is called with `{ customPath: 'secret/data/custom' }` +- **Then** the provider fetches `BAO_URL/v1/secret/data/custom` instead of the `BAO_SECRET_PATH` value. + +### openbao-approle-authentication + +The provider authenticates to OpenBao using the AppRole auth method and exchanges role credentials for a client token. + +- **Must** POST `{ role_id, secret_id }` as a JSON body to `{BAO_URL}/v1/auth/approle/login`. +- **Must** throw an `Authentication failed: Status ` error when the login response is not ok. +- **Must** throw a `Failed to retrieve client token from OpenBao.` error when the login succeeds but the response contains no `auth.client_token`. +- **Must** send the obtained client token on subsequent secret requests via the `X-Vault-Token` header. +- **Will** reject with `OpenBao authentication request timed out.` when the login request is aborted by the timeout. + +#### Scenario: rejected AppRole credentials + +- **When** OpenBao answers the login request with a non-2xx status +- **Then** the provider throws `Authentication failed: Status `. + +#### Scenario: login response without a token + +- **When** the login response is ok but carries no `auth.client_token` +- **Then** the provider throws `Failed to retrieve client token from OpenBao.`. + +### openbao-kv-secret-retrieval + +The provider reads a KV secrets engine path using the authenticated token. + +- **Must** GET `{BAO_URL}/v1/{secretPath}` with the `X-Vault-Token` header. +- **Must** throw `Fetch failed: Status ` when the read response is not ok. +- **Must** read the secret data from the KV v2 payload shape `result.data.data`. +- **Must** throw `Unexpected secrets payload structure.` when `data.data` is missing, not an object, or an array. +- **Must** return the secret map as a flat `Record`. +- **Will** reject with `OpenBao secrets fetch request timed out.` when the read request is aborted by the timeout. + +#### Scenario: unexpected payload shape + +- **When** OpenBao returns a payload without a `data.data` object +- **Then** the provider throws `Unexpected secrets payload structure.`. + +#### Scenario: successful secret retrieval + +- **When** authentication succeeds and the KV path returns `{ data: { data: { RESEND_API_KEY: 'x' } } }` +- **Then** `loadSecrets` resolves to `{ RESEND_API_KEY: 'x' }`. + +### startup-secret-injection + +Every microservice loads remote secrets into `process.env` before its NATS microservice is created and starts listening. + +- **Must** gate startup injection on `ENABLE_BAO` being exactly `true` after trimming and lowercasing. +- **Must** skip injection without error when `ENABLE_BAO` is not `true`. +- **Must** skip injection without error when `SECRETS_PROVIDER` is not set. +- **Must** log a warning and continue with local environment variables when `SECRETS_PROVIDER` names an unsupported provider. +- **Must** inject every fetched secret into `process.env` as an enumerable, configurable, writable string property. +- **Must** skip keys named `__proto__`, `constructor`, or `prototype` during injection. +- **Must** log a `Critical Lifecycle Boot Failure` error and continue startup when the provider throws, rather than crashing the bootstrap. +- **Must** run `await loadConfigSecrets()` before `NestFactory.createMicroservice(...)` in every `apps/*/src/main.ts` bootstrap. +- **Will** be invoked through the `@credebl/config` package root export. + +#### Scenario: secrets management disabled + +- **When** `ENABLE_BAO` is unset, `false`, or `FALSE` +- **Then** `loadConfigSecrets` returns without reading from any provider. + +#### Scenario: unsupported provider value + +- **When** `SECRETS_PROVIDER=consul` +- **Then** `loadConfigSecrets` logs a warning and returns without throwing. + +#### Scenario: malicious secret keys are filtered + +- **When** the provider returns a secret whose key is `__proto__` +- **Then** that entry is skipped and `process.env` is not polluted. + +#### Scenario: provider failure does not block boot + +- **When** the provider rejects while loading secrets +- **Then** the error is logged and `loadConfigSecrets` resolves instead of rejecting. + +### runtime-secret-loading-with-cache + +Email and storage integrations fetch their credential sets on demand from the configured provider, cached per secret path. + +- **Must** expose a `fetchSecrets(secretPath)` helper in `libs/common/src/utils/secretLoader.util.ts`. +- **Must** return an empty object without calling the provider when `ENABLE_BAO` is not `true`. +- **Must** return an empty object when `SECRETS_PROVIDER` is unset or names an unsupported provider. +- **Must** call `provider.loadSecrets({ customPath: secretPath })` when a provider is active. +- **Must** cache the result per secret path for a TTL of 10 minutes. +- **Must** share the cache module-wide across all consumers (email + storage) for a given path. + +#### Scenario: runtime fetch for a specific path + +- **When** `fetchSecrets('secret/data/credebl_smtp_config')` is called with OpenBao enabled +- **Then** the provider is invoked with `customPath` set to that path and its result is returned. + +#### Scenario: subsequent reads are served from cache + +- **When** `fetchSecrets` is called twice with the same path while the entry is unexpired +- **Then** the provider is only invoked once. + +### ttl-cache-semantics + +The TTL cache preserves a fetched value for a duration and re-fetches once it lapses. + +- **Must** invoke the fetch function on first read and store the result. +- **Must** return the stored value on subsequent reads without re-invoking the fetch function. +- **Must** treat the TTL as sliding: any read refreshes the expiry timer. +- **Must** clear the stored value once `ttlMs` of inactivity elapses. +- **Must** provide a `clear()` method that drops the stored value and cancels the pending expiry timer. +- **Will** schedule expiry so that an idle process is not kept alive by the pending timer. + +#### Scenario: expiry triggers a re-fetch + +- **When** a value is read again after `ttlMs` of inactivity has elapsed +- **Then** the fetch function runs again and the fresh value is returned. + +#### Scenario: sliding window keeps a hot entry alive + +- **When** a value is read repeatedly at intervals shorter than `ttlMs` +- **Then** the fetch function is never re-invoked and the same value is returned. + +### email-credential-sourcing + +Email providers resolve their credentials at runtime, preferring provider-sourced secrets over environment variables. + +- **Must** resolve the Resend API key from `secret/data/credebl_resend_api_key` (`RESEND_API_KEY`) or fall back to `process.env.RESEND_API_KEY`. +- **Must** resolve the SendGrid API key from `secret/data/credebl_sendgrid_api_key` (`SENDGRID_API_KEY`) or fall back to `process.env.SENDGRID_API_KEY`. +- **Must** resolve SMTP settings (`SMTP_HOST`, `SMTP_PORT`, `SMTP_USER`, `SMTP_PASS`) from `secret/data/credebl_smtp_config` or fall back to the matching environment variables. +- **Must** treat a missing/invalid SMTP port as a failure. +- **Must** return `false` and log the error when a required credential is missing or a send fails, rather than throwing to the caller. +- **Must** configure the SMTP transport with `secure: true` for port 465 and `requireTLS: true` for port 587. + +#### Scenario: missing email credential + +- **When** the provider returns no Resend key and `RESEND_API_KEY` is unset +- **Then** `sendWithResend` logs an error and returns `false`. + +#### Scenario: invalid SMTP port + +- **When** `SMTP_PORT` is not a positive integer +- **Then** `sendWithSMTP` logs an error and returns `false`. + +#### Scenario: provider secrets take precedence + +- **When** the provider returns `RESEND_API_KEY` and the same env var is set to a different value +- **Then** the client is constructed with the provider-sourced key. + +### storage-credential-sourcing + +S3 storage clients resolve their AWS credentials at runtime from the shared `secret/data/credebl_aws_keys` path. + +- **Must** resolve internal S3 credentials from `AWS_ACCESS_KEY`/`AWS_SECRET_KEY` or fall back to the matching env vars. +- **Must** resolve public (logo/URL) S3 credentials from `AWS_PUBLIC_ACCESS_KEY`/`AWS_PUBLIC_SECRET_KEY` or fall back to the matching env vars. +- **Must** resolve store-object S3 credentials from `AWS_S3_STOREOBJECT_ACCESS_KEY`/`AWS_S3_STOREOBJECT_SECRET_KEY` or fall back to the matching env vars. +- **Must** read the region for each client from `AWS_REGION`, `AWS_PUBLIC_REGION`, and `AWS_S3_STOREOBJECT_REGION` respectively. +- **Will** leave the RustFS/local storage providers reading credentials exclusively from the environment. + +#### Scenario: provider-sourced AWS keys + +- **When** OpenBao returns `AWS_ACCESS_KEY`/`AWS_SECRET_KEY` at the AWS key path +- **Then** the internal S3 client is constructed with those values. + +### deployment-artifacts + +The repository ships demo configuration and container artifacts for running OpenBao. + +- **Must** document the `ENABLE_BAO` and `SECRETS_PROVIDER` gating variables and the `BAO_URL`, `BAO_SECRET_PATH`, `BAO_ROLE_ID`, and `BAO_SECRET_ID` placeholders in `.env.demo`. +- **Must** provide a `docker-compose.openbao.yml` that runs an `openbao/openbao` server exposing port 8200 with file-backed storage persisted on a named volume. +- **Must** provide an HCL server configuration (`config.hcl`) declaring file storage at `/openbao/file`, a TCP listener on `0.0.0.0:8200` with TLS disabled, and the UI enabled. +- **Must** provide a `openbao-init.sh` script that idempotently initializes and unseals the server, enables KV v2 at `secret/`, enables AppRole auth, creates the `credebl` role scoped to the `credebl_*` paths, generates the `role_id`/`secret_id`, and stores the secret data from the matching environment variables. +- **Will** require the operator to run `openbao-init.sh` once and copy the printed `BAO_ROLE_ID`/`BAO_SECRET_ID` into `.env.demo` before the platform can use the provider. + +#### Scenario: demo environment placeholders + +- **When** an operator follows the demo setup +- **Then** `.env.demo` exposes the two gating variables and the four `BAO_*` variables to configure the provider. + +#### Scenario: local OpenBao server + +- **When** `docker compose -f docker-compose.openbao.yml up` is run +- **Then** an OpenBao server is available on `127.0.0.1:8200` and its data survives restarts via the named volume. + +#### Scenario: one-shot provisioning via the init script + +- **When** `./openbao-init.sh` is run against a freshly started server +- **Then** the server is initialized and unsealed, KV v2 and AppRole are enabled, the `credebl` role exists, and the script prints `BAO_ROLE_ID`/`BAO_SECRET_ID` ready for `.env.demo`. +- **And** re-running the script does not re-initialize the server or regenerate the `role_id`/`secret_id`. diff --git a/package.json b/package.json index 79f2dbe33c..348906944b 100644 --- a/package.json +++ b/package.json @@ -80,7 +80,7 @@ "express-useragent": "^1.0.15", "firebase-admin": "^13.6.1", "fs": "0.0.1-security", - "generate-password": "^1.7.1", + "generate-password": "^1.7.1", "helmet": "^8.1.0", "html-pdf": "^3.0.1", "html-to-image": "^1.11.13", @@ -201,11 +201,11 @@ "^@credebl/user-org-roles(|/.*)$": "/libs/user-org-roles/src/$1", "^y/user-activity(|/.*)$": "/libs/user-activity/src/$1", "^@app/supabase(|/.*)$": "/libs/supabase/src/$1", - "^credebl/utility(|/.*)$": "/libs/utility/src/$1", "^@credebl/config(|/.*)$": "/libs/config/src/$1", "^@credebl/context(|/.*)$": "/libs/context/src/$1", - "^@credebl/logger(|/.*)$": "/libs/logger/src/$1" + "^@credebl/logger(|/.*)$": "/libs/logger/src/$1", + "^libs/(.*)$": "/libs/$1" } }, "packageManager": "pnpm@9.15.3+sha512.1f79bc245a66eb0b07c5d4d83131240774642caaa86ef7d0434ab47c0d16f66b04e21e0c086eb61e62c77efc4d7f7ec071afad3796af64892fae66509173893a",