diff --git a/.example.env b/.example.env index f4db772..868c5e7 100644 --- a/.example.env +++ b/.example.env @@ -16,16 +16,18 @@ POSTGRES_DB=db PG_LOG_CHECKPOINTS='off' S3_MOCK=0 ## Options: [0, 1] corresponds to [no S3 Mock, s3 Mock]. This integer value configures the initialization mode in docker-compose. +# COMPOSE_PROFILES=s3mock ## uncomment with S3_MOCK=1 to start the S3 mock container (not pulled otherwise) #### Optional: INIT_AUTH= ##set to zero if you do not want to initialize auth (used when s3api is a package in a non auth app) -## MINIO -MINIO_S3_ENDPOINT='http://minio:9000' -MINIO_S3_REGION='s3-region-string' -MINIO_S3_DISABLE_SSL='bool-string' -MINIO_S3_FORCE_PATH_STYLE='bool-string' -MINIO_SECRET_ACCESS_KEY='access-key-string' +## S3 mock (S3_MOCK=1). Variable names are historical; the mock is Moto, which accepts any keys. +MINIO_S3_ENDPOINT='http://s3mock:5000' +MINIO_S3_REGION='us-east-1' +MINIO_S3_DISABLE_SSL='true' +MINIO_S3_FORCE_PATH_STYLE='true' +MINIO_ACCESS_KEY_ID='test' +MINIO_SECRET_ACCESS_KEY='test' ## Download size limits (optional will default to 5 and 50 respectively) ZIP_DOWNLOAD_SIZE_LIMIT = 5 #gb diff --git a/.example.env.json b/.example.env.json index 8b058b7..6e61dfc 100644 --- a/.example.env.json +++ b/.example.env.json @@ -1,14 +1,15 @@ { + "use_iam_role": false, "accounts": [ - { "AWS_ACCESS_KEY_ID": "", "AWS_SECRET_ACCESS_KEY": "" }, { - "AWS_ACCESS_KEY_ID": "", - "AWS_SECRET_ACCESS_KEY": "" + "AWS_ACCESS_KEY_ID": "", + "AWS_SECRET_ACCESS_KEY": "" } ], - "bucket_allow_list":["*"] # "*" for allowing access to all buckets -} \ No newline at end of file + "bucket_allow_list": ["*"], + "bucket_block_list": [] +} diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml index c928046..18de1b2 100644 --- a/.github/workflows/e2e-test.yml +++ b/.github/workflows/e2e-test.yml @@ -19,7 +19,7 @@ jobs: touch .env echo KEYCLOAK_PUBLIC_KEYS_URL='https://auth.dewberryanalytics.com/auth/realms/testing/protocol/openid-connect/certs' >> .env - echo MINIO_S3_ENDPOINT='http://minio:9000' >> .env + echo MINIO_S3_ENDPOINT='http://s3mock:5000' >> .env echo AWS_REGION='us-east-1' >> .env echo MINIO_S3_DISABLE_SSL='true' >> .env echo MINIO_S3_FORCE_PATH_STYLE='true' >> .env @@ -27,6 +27,7 @@ jobs: echo MINIO_SECRET_ACCESS_KEY=password >> .env echo MINIO_S3_REGION='us-east-1' >> .env echo S3_MOCK=1 >> .env + echo COMPOSE_PROFILES=s3mock >> .env echo AWS_S3_BUCKET='test-bucket' >> .env echo S3API_SERVICE_PORT='5005' >> .env echo AUTH_LEVEL=0 >> .env @@ -40,13 +41,13 @@ jobs: echo "Is KEYCLOAK_SECRET set: $(if [ -z "$KEYCLOAK_SECRET" ]; then echo "No"; else echo "Yes"; fi)" envsubst < e2e-test/e2eEnv.template.json > e2e-test/e2eEnv.json - name: Build the docker-compose stack - run: docker-compose build -d + run: docker compose build - name: Create network run: docker network create s3_net - name: Build and Run the docker-compose stack - run: docker-compose up -d + run: docker compose up -d - name: Run newman tests run: > diff --git a/.gitignore b/.gitignore index c547b13..faa97b6 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ main .data .env.json .vscode/settings.json +.env* diff --git a/README.md b/README.md index d8e883a..119e461 100644 --- a/README.md +++ b/README.md @@ -11,3 +11,33 @@ For testing fine grain access control, add an entry to the permissions table lik ``` INSERT INTO public.permissions (user_email, operation, allowed_s3_prefixes) VALUES ('sputnam@dewberry.com', 'write', ARRAY['/ffrd-trinity/sputnam/']); ``` + +## AWS credentials (`.env.json`) + +Static keys (unchanged): +```json +{ + "accounts": [{ "AWS_ACCESS_KEY_ID": "...", "AWS_SECRET_ACCESS_KEY": "..." }], + "bucket_allow_list": ["*"], + "bucket_block_list": [] +} +``` + +EC2 instance role (no keys): +```json +{ + "use_iam_role": true, + "bucket_allow_list": ["bucket-a", "bucket-b"], + "bucket_block_list": [] +} +``` + +- IAM mode is used when `use_iam_role` is true, `accounts` is empty, or the file is empty. Credentials then come only from EC2 instance metadata; static keys and `AWS_*` env vars are ignored. +- `bucket_block_list`: buckets that are never exposed, even when `bucket_allow_list` is `["*"]`. +- With named buckets, s3api checks each with HeadBucket at startup (no `s3:ListAllMyBuckets` needed). With `["*"]` the role needs `s3:ListAllMyBuckets`. +- Role permissions: `s3:ListBucket`, `s3:GetBucketLocation` on each bucket; `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:AbortMultipartUpload` on `bucket/*`. Cross-account buckets also need a bucket-policy statement allowing the role. +- Docker on EC2 with IMDSv2: set the instance metadata hop limit to 2. + +## Local S3 mock + +Set `S3_MOCK=1` and `COMPOSE_PROFILES=s3mock` in `.env` to run against an in-memory Moto S3 (no AWS needed). diff --git a/blobstore/blobhandler.go b/blobstore/blobhandler.go index c39c029..1440d26 100644 --- a/blobstore/blobhandler.go +++ b/blobstore/blobhandler.go @@ -12,6 +12,7 @@ import ( envcheck "github.com/Dewberry/s3api/env-checker" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds" "github.com/aws/aws-sdk-go/aws/session" "github.com/aws/aws-sdk-go/service/s3" "github.com/labstack/echo/v4" @@ -44,6 +45,7 @@ type BlobHandler struct { S3Controllers []S3Controller Mu sync.Mutex AllowAllBuckets bool + BlockedBuckets map[string]struct{} DB auth.Database Config *Config } @@ -101,28 +103,40 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) { } // Using AWS S3 - // Load AWS credentials from the provided .env.json file + // Load AWS settings from the provided .env.json file log.Debug("looking for .env.json") - awsConfig, err := newAWSConfig(envJson) + awsConfig, useIAMRole, err := newAWSConfig(envJson) // Check if loading AWS credentials from .env.json failed if err != nil { return nil, fmt.Errorf("env.json credentials extraction failed, please check `.env.json.example` for reference on formatting, %s", err.Error()) } + if useIAMRole { + log.Info("using the EC2 instance role (instance metadata) for S3 access; static keys and AWS_* env vars are ignored") + } //does it contain "*" config.AllowAllBuckets = arrayContains("*", awsConfig.BucketAllowList) + // Blocked buckets win over the allow list + config.BlockedBuckets = make(map[string]struct{}) + for _, bucket := range awsConfig.BucketBlockList { + config.BlockedBuckets[bucket] = struct{}{} + } + // Convert allowed buckets to a map for efficient lookup allowedBucketsMap := make(map[string]struct{}) for _, bucket := range awsConfig.BucketAllowList { + if bucket == "*" || config.IsBlocked(bucket) { + continue + } allowedBucketsMap[bucket] = struct{}{} } // Load AWS credentials for multiple accounts from .env.json for _, creds := range awsConfig.Accounts { // Create an AWS session and S3 client for each account - s3SVC, sess, err := aWSSessionManager(creds) + s3SVC, sess, err := aWSSessionManager(creds, useIAMRole) if err != nil { errMsg := fmt.Errorf("failed to create AWS session: %s", err.Error()) log.Error(errMsg.Error()) @@ -130,10 +144,31 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) { } S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC} + + // With the IAM role and an explicit allow list, check each bucket with + // HeadBucket so the role does not need s3:ListAllMyBuckets. + if useIAMRole && !config.AllowAllBuckets { + var bucketNames []string + for bucket := range allowedBucketsMap { + if _, err := s3SVC.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucket)}); err != nil { + log.Errorf("bucket '%s' is not accessible with the IAM role: %s", bucket, err.Error()) + continue + } + bucketNames = append(bucketNames, bucket) + } + for _, bucket := range bucketNames { + delete(allowedBucketsMap, bucket) + } + if len(bucketNames) > 0 { + config.S3Controllers = append(config.S3Controllers, S3Controller{Sess: sess, S3Svc: s3SVC, Buckets: bucketNames, S3Mock: false}) + } + continue + } + // Retrieve the list of buckets for each account result, err := S3Ctrl.ListBuckets() if err != nil { - errMsg := fmt.Errorf("failed to retrieve list of buckets with access key: %s, error: %s", creds.AWS_ACCESS_KEY_ID, err.Error()) + errMsg := fmt.Errorf("failed to retrieve list of buckets with %s, error: %s", credsLabel(creds), err.Error()) return nil, errMsg } @@ -141,7 +176,9 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) { if config.AllowAllBuckets { // Directly add all bucket names if allowAllBucket is true for _, bucket := range result.Buckets { - bucketNames = append(bucketNames, aws.StringValue(bucket.Name)) + if name := aws.StringValue(bucket.Name); !config.IsBlocked(name) { + bucketNames = append(bucketNames, name) + } } } else { // Filter and add only the allowed buckets @@ -171,12 +208,35 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) { return &config, nil } -func aWSSessionManager(creds AWSCreds) (*s3.S3, *session.Session, error) { +// IsBlocked reports whether a bucket is in bucket_block_list. +func (bh *BlobHandler) IsBlocked(bucket string) bool { + _, blocked := bh.BlockedBuckets[bucket] + return blocked +} + +func credsLabel(creds AWSCreds) string { + if creds.AWS_ACCESS_KEY_ID == "" { + return "the EC2 instance role" + } + return "access key: " + creds.AWS_ACCESS_KEY_ID +} + +func aWSSessionManager(creds AWSCreds, useIAMRole bool) (*s3.S3, *session.Session, error) { log.Info("Using AWS S3") - sess, err := session.NewSession(&aws.Config{ - Region: aws.String("us-east-1"), - Credentials: credentials.NewStaticCredentials(creds.AWS_ACCESS_KEY_ID, creds.AWS_SECRET_ACCESS_KEY, ""), - }) + cfg := &aws.Config{Region: aws.String("us-east-1")} + if useIAMRole { + // Use only the EC2 instance profile. The SDK default chain would pick + // up AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY from the environment + // first (e.g. a shared .env), silently bypassing the role. + metaSess, err := session.NewSession(&aws.Config{Region: aws.String("us-east-1")}) + if err != nil { + return nil, nil, fmt.Errorf("error creating metadata session: %s", err.Error()) + } + cfg.Credentials = ec2rolecreds.NewCredentials(metaSess) + } else { + cfg.Credentials = credentials.NewStaticCredentials(creds.AWS_ACCESS_KEY_ID, creds.AWS_SECRET_ACCESS_KEY, "") + } + sess, err := session.NewSession(cfg) if err != nil { return nil, nil, fmt.Errorf("error creating s3 session: %s", err.Error()) } diff --git a/blobstore/buckets.go b/blobstore/buckets.go index 9ddff6f..0245877 100644 --- a/blobstore/buckets.go +++ b/blobstore/buckets.go @@ -104,7 +104,9 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error { } var mostRecentBucketList []string for _, b := range result.Buckets { - mostRecentBucketList = append(mostRecentBucketList, *b.Name) + if !bh.IsBlocked(*b.Name) { + mostRecentBucketList = append(mostRecentBucketList, *b.Name) + } } if !isIdenticalArray(controller.Buckets, mostRecentBucketList) { controller.Buckets = mostRecentBucketList diff --git a/blobstore/creds.go b/blobstore/creds.go index de1d49e..013273a 100644 --- a/blobstore/creds.go +++ b/blobstore/creds.go @@ -14,8 +14,13 @@ type AWSCreds struct { } type AWSConfig struct { + // UseIAMRole ignores any static keys (in this file or AWS_* env vars) + // and uses only the EC2 instance profile from instance metadata. + UseIAMRole bool `json:"use_iam_role"` Accounts []AWSCreds `json:"accounts"` BucketAllowList []string `json:"bucket_allow_list"` + // BucketBlockList buckets are never exposed, even when allowed (e.g. by "*"). + BucketBlockList []string `json:"bucket_block_list"` } type MinioConfig struct { @@ -52,60 +57,58 @@ func (creds MinioConfig) validateMinioConfig() error { return nil } -func validateEnvJSON(filePath string) error { - // Read the contents of the .env.json file - jsonData, err := os.ReadFile(filePath) - if err != nil { - return fmt.Errorf("error reading .env.json: %s", err.Error()) - } - - // Parse the JSON data into the AWSConfig struct - var awsConfig AWSConfig - if err := json.Unmarshal(jsonData, &awsConfig); err != nil { - return fmt.Errorf("error parsing .env.json: %s", err.Error()) - } - - // Check if there is at least one account defined - if len(awsConfig.Accounts) == 0 { - return fmt.Errorf("no AWS accounts defined in .env.json") - } - - // Check if each account has the required fields - for i, account := range awsConfig.Accounts { - missingFields := []string{} - if account.AWS_ACCESS_KEY_ID == "" { - missingFields = append(missingFields, "AWS_ACCESS_KEY_ID") - } - if account.AWS_SECRET_ACCESS_KEY == "" { - missingFields = append(missingFields, "AWS_SECRET_ACCESS_KEY") - } - - if len(missingFields) > 0 { - return fmt.Errorf("missing fields (%s) for AWS account %d in envJson file", strings.Join(missingFields, ", "), i+1) +// validateAWSConfig checks a parsed .env.json. Static keys are only required +// when the IAM role is not in use. +func validateAWSConfig(awsConfig AWSConfig, useIAMRole bool) error { + if !useIAMRole { + for i, account := range awsConfig.Accounts { + missingFields := []string{} + if account.AWS_ACCESS_KEY_ID == "" { + missingFields = append(missingFields, "AWS_ACCESS_KEY_ID") + } + if account.AWS_SECRET_ACCESS_KEY == "" { + missingFields = append(missingFields, "AWS_SECRET_ACCESS_KEY") + } + if len(missingFields) > 0 { + return fmt.Errorf("missing fields (%s) for AWS account %d in envJson file", strings.Join(missingFields, ", "), i+1) + } } } if len(awsConfig.BucketAllowList) == 0 { return fmt.Errorf("no buckets in the `bucket_allow_list`, please provide required buckets, or `*` for access to all buckets") } - // If all checks pass, return nil (no error) return nil } -func newAWSConfig(envJson string) (AWSConfig, error) { - var awsConfig AWSConfig - err := validateEnvJSON(envJson) - if err != nil { - return awsConfig, fmt.Errorf(err.Error()) - } +// newAWSConfig reads .env.json. The IAM role is used when the file is empty, +// defines no "accounts", or sets "use_iam_role": true. Otherwise the static +// keys in "accounts" are used, as before. +func newAWSConfig(envJson string) (awsConfig AWSConfig, useIAMRole bool, err error) { jsonData, err := os.ReadFile(envJson) if err != nil { - return awsConfig, err + return awsConfig, false, fmt.Errorf("error reading .env.json: %s", err.Error()) + } + + if len(strings.TrimSpace(string(jsonData))) == 0 { + // Empty file: IAM role with access to all buckets. + awsConfig.BucketAllowList = []string{"*"} + useIAMRole = true + } else { + if err := json.Unmarshal(jsonData, &awsConfig); err != nil { + return awsConfig, false, fmt.Errorf("error parsing .env.json: %s", err.Error()) + } + useIAMRole = awsConfig.UseIAMRole || len(awsConfig.Accounts) == 0 + } + + if err := validateAWSConfig(awsConfig, useIAMRole); err != nil { + return awsConfig, false, err } - if err := json.Unmarshal(jsonData, &awsConfig); err != nil { - return awsConfig, err + if useIAMRole { + // One account with no keys: the session falls back to the IAM role. + awsConfig.Accounts = []AWSCreds{{}} } - return awsConfig, nil + return awsConfig, useIAMRole, nil } func newMinioConfig() MinioConfig { diff --git a/docker-compose.yml b/docker-compose.yml index 4d2bf58..df1bb00 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,25 +15,24 @@ services: networks: - s3_net depends_on: - minio: + s3mock: condition: service_started + required: false # only started with the "s3mock" profile postgres: condition: service_healthy - minio: - container_name: s3api_minio - image: minio/minio:RELEASE.2023-08-23T10-07-06Z.fips - environment: - - MINIO_ROOT_USER=user - - MINIO_ROOT_PASSWORD=password - command: server /data --console-address ":9001" - deploy: - replicas: $S3_MOCK + # Free, in-memory S3 mock (Moto, Apache-2.0) used when S3_MOCK=1. + # Opt-in only, so the image is not pulled unless mocking S3. + # Enable with S3_MOCK=1 plus COMPOSE_PROFILES=s3mock in .env + # (or `docker compose --profile s3mock up`). + # Host port 9000 is kept because presigned upload URLs in mock mode + # are generated for http://localhost:9000 (see blobstore/upload.go). + s3mock: + container_name: s3api_s3mock + image: motoserver/moto:5.1.0 + profiles: ['s3mock'] ports: - - '9000:9000' - - '9001:9001' - volumes: - - ./.data/minio:/data + - '9000:5000' networks: - s3_net