Use EC2 instance role instead of AWS keys

.example.env

@@ -16,16 +16,18 @@ POSTGRES_DB=db
 PG_LOG_CHECKPOINTS='off'
 
 S3_MOCK=0 ##  Options: [0, 1] corresponds to [no S3 Mock, s3 Mock]. This integer value configures the initialization mode in docker-compose.
+# COMPOSE_PROFILES=s3mock  ## uncomment with S3_MOCK=1 to start the S3 mock container (not pulled otherwise)
 #### Optional:
 
 INIT_AUTH= ##set to zero if you do not want to initialize auth (used when s3api is a package in a non auth app)
 
-## MINIO
-MINIO_S3_ENDPOINT='http://minio:9000'
-MINIO_S3_REGION='s3-region-string'
-MINIO_S3_DISABLE_SSL='bool-string'
-MINIO_S3_FORCE_PATH_STYLE='bool-string'
-MINIO_SECRET_ACCESS_KEY='access-key-string'
+## S3 mock (S3_MOCK=1). Variable names are historical; the mock is Moto, which accepts any keys.
+MINIO_S3_ENDPOINT='http://s3mock:5000'
+MINIO_S3_REGION='us-east-1'
+MINIO_S3_DISABLE_SSL='true'
+MINIO_S3_FORCE_PATH_STYLE='true'
+MINIO_ACCESS_KEY_ID='test'
+MINIO_SECRET_ACCESS_KEY='test'
 
 ## Download size limits (optional will default to 5 and 50 respectively)
 ZIP_DOWNLOAD_SIZE_LIMIT = 5 #gb

.example.env.json

@@ -1,14 +1,15 @@
 {
+  "use_iam_role": false,
   "accounts": [
-
     {
       "AWS_ACCESS_KEY_ID": "",
       "AWS_SECRET_ACCESS_KEY": ""
     },
     {
-       "AWS_ACCESS_KEY_ID": "",
-       "AWS_SECRET_ACCESS_KEY": ""
+      "AWS_ACCESS_KEY_ID": "",
+      "AWS_SECRET_ACCESS_KEY": ""
     }
   ],
-  "bucket_allow_list":["*"] # "*" for allowing access to all buckets
-}
+  "bucket_allow_list": ["*"],
+  "bucket_block_list": []
+}

.github/workflows/e2e-test.yml

@@ -19,14 +19,15 @@ jobs:
           touch .env
 
           echo KEYCLOAK_PUBLIC_KEYS_URL='https://auth.dewberryanalytics.com/auth/realms/testing/protocol/openid-connect/certs' >> .env
-          echo MINIO_S3_ENDPOINT='http://minio:9000' >> .env
+          echo MINIO_S3_ENDPOINT='http://s3mock:5000' >> .env
           echo AWS_REGION='us-east-1' >> .env
           echo MINIO_S3_DISABLE_SSL='true' >> .env
           echo MINIO_S3_FORCE_PATH_STYLE='true' >> .env
           echo MINIO_ACCESS_KEY_ID=user >> .env
           echo MINIO_SECRET_ACCESS_KEY=password >> .env
           echo MINIO_S3_REGION='us-east-1' >> .env
           echo S3_MOCK=1 >> .env
+          echo COMPOSE_PROFILES=s3mock >> .env
           echo AWS_S3_BUCKET='test-bucket' >> .env
           echo S3API_SERVICE_PORT='5005' >> .env
           echo AUTH_LEVEL=0 >> .env
@@ -40,13 +41,13 @@ jobs:
           echo "Is KEYCLOAK_SECRET set: $(if [ -z "$KEYCLOAK_SECRET" ]; then echo "No"; else echo "Yes"; fi)"
           envsubst < e2e-test/e2eEnv.template.json > e2e-test/e2eEnv.json
       - name: Build the docker-compose stack
-        run: docker-compose build -d
+        run: docker compose build
 
       - name: Create network
         run: docker network create s3_net
 
       - name: Build and Run the docker-compose stack
-        run: docker-compose up -d
+        run: docker compose up -d
 
       - name: Run newman tests
         run: >

.gitignore

@@ -25,3 +25,4 @@ main
 .data
 .env.json
 .vscode/settings.json
+.env*

README.md

@@ -11,3 +11,33 @@ For testing fine grain access control, add an entry to the permissions table lik
 ```
 INSERT INTO public.permissions (user_email, operation, allowed_s3_prefixes) VALUES ('sputnam@dewberry.com', 'write', ARRAY['/ffrd-trinity/sputnam/']);
 ```
+
+## AWS credentials (`.env.json`)
+
+Static keys (unchanged):
+```json
+{
+  "accounts": [{ "AWS_ACCESS_KEY_ID": "...", "AWS_SECRET_ACCESS_KEY": "..." }],
+  "bucket_allow_list": ["*"],
+  "bucket_block_list": []
+}
+```
+
+EC2 instance role (no keys):
+```json
+{
+  "use_iam_role": true,
+  "bucket_allow_list": ["bucket-a", "bucket-b"],
+  "bucket_block_list": []
+}
+```
+
+- IAM mode is used when `use_iam_role` is true, `accounts` is empty, or the file is empty. Credentials then come only from EC2 instance metadata; static keys and `AWS_*` env vars are ignored.
+- `bucket_block_list`: buckets that are never exposed, even when `bucket_allow_list` is `["*"]`.
+- With named buckets, s3api checks each with HeadBucket at startup (no `s3:ListAllMyBuckets` needed). With `["*"]` the role needs `s3:ListAllMyBuckets`.
+- Role permissions: `s3:ListBucket`, `s3:GetBucketLocation` on each bucket; `s3:GetObject`, `s3:PutObject`, `s3:DeleteObject`, `s3:AbortMultipartUpload` on `bucket/*`. Cross-account buckets also need a bucket-policy statement allowing the role.
+- Docker on EC2 with IMDSv2: set the instance metadata hop limit to 2.
+
+## Local S3 mock
+
+Set `S3_MOCK=1` and `COMPOSE_PROFILES=s3mock` in `.env` to run against an in-memory Moto S3 (no AWS needed).

blobstore/blobhandler.go

@@ -12,6 +12,7 @@ import (
 	envcheck "github.com/Dewberry/s3api/env-checker"
 	"github.com/aws/aws-sdk-go/aws"
 	"github.com/aws/aws-sdk-go/aws/credentials"
+	"github.com/aws/aws-sdk-go/aws/credentials/ec2rolecreds"
 	"github.com/aws/aws-sdk-go/aws/session"
 	"github.com/aws/aws-sdk-go/service/s3"
 	"github.com/labstack/echo/v4"
@@ -44,6 +45,7 @@ type BlobHandler struct {
 	S3Controllers   []S3Controller
 	Mu              sync.Mutex
 	AllowAllBuckets bool
+	BlockedBuckets  map[string]struct{}
 	DB              auth.Database
 	Config          *Config
 }
@@ -101,47 +103,82 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 	}
 
 	// Using AWS S3
-	// Load AWS credentials from the provided .env.json file
+	// Load AWS settings from the provided .env.json file
 	log.Debug("looking for .env.json")
-	awsConfig, err := newAWSConfig(envJson)
+	awsConfig, useIAMRole, err := newAWSConfig(envJson)
 
 	// Check if loading AWS credentials from .env.json failed
 	if err != nil {
 		return nil, fmt.Errorf("env.json credentials extraction failed, please check `.env.json.example` for reference on formatting, %s", err.Error())
 	}
+	if useIAMRole {
+		log.Info("using the EC2 instance role (instance metadata) for S3 access; static keys and AWS_* env vars are ignored")
+	}
 
 	//does it contain "*"
 	config.AllowAllBuckets = arrayContains("*", awsConfig.BucketAllowList)
 
+	// Blocked buckets win over the allow list
+	config.BlockedBuckets = make(map[string]struct{})
+	for _, bucket := range awsConfig.BucketBlockList {
+		config.BlockedBuckets[bucket] = struct{}{}
+	}
+
 	// Convert allowed buckets to a map for efficient lookup
 	allowedBucketsMap := make(map[string]struct{})
 	for _, bucket := range awsConfig.BucketAllowList {
+		if bucket == "*" || config.IsBlocked(bucket) {
+			continue
+		}
 		allowedBucketsMap[bucket] = struct{}{}
 	}
 
 	// Load AWS credentials for multiple accounts from .env.json
 	for _, creds := range awsConfig.Accounts {
 		// Create an AWS session and S3 client for each account
-		s3SVC, sess, err := aWSSessionManager(creds)
+		s3SVC, sess, err := aWSSessionManager(creds, useIAMRole)
 		if err != nil {
 			errMsg := fmt.Errorf("failed to create AWS session: %s", err.Error())
 			log.Error(errMsg.Error())
 			return nil, errMsg
 		}
 
 		S3Ctrl := S3Controller{Sess: sess, S3Svc: s3SVC}
+
+		// With the IAM role and an explicit allow list, check each bucket with
+		// HeadBucket so the role does not need s3:ListAllMyBuckets.
+		if useIAMRole && !config.AllowAllBuckets {
+			var bucketNames []string
+			for bucket := range allowedBucketsMap {
+				if _, err := s3SVC.HeadBucket(&s3.HeadBucketInput{Bucket: aws.String(bucket)}); err != nil {
+					log.Errorf("bucket '%s' is not accessible with the IAM role: %s", bucket, err.Error())
+					continue
+				}
+				bucketNames = append(bucketNames, bucket)
+			}
+			for _, bucket := range bucketNames {
+				delete(allowedBucketsMap, bucket)
+			}
+			if len(bucketNames) > 0 {
+				config.S3Controllers = append(config.S3Controllers, S3Controller{Sess: sess, S3Svc: s3SVC, Buckets: bucketNames, S3Mock: false})
+			}
+			continue
+		}
+
 		// Retrieve the list of buckets for each account
 		result, err := S3Ctrl.ListBuckets()
 		if err != nil {
-			errMsg := fmt.Errorf("failed to retrieve list of buckets with access key: %s, error: %s", creds.AWS_ACCESS_KEY_ID, err.Error())
+			errMsg := fmt.Errorf("failed to retrieve list of buckets with %s, error: %s", credsLabel(creds), err.Error())
 			return nil, errMsg
 		}
 
 		var bucketNames []string
 		if config.AllowAllBuckets {
 			// Directly add all bucket names if allowAllBucket is true
 			for _, bucket := range result.Buckets {
-				bucketNames = append(bucketNames, aws.StringValue(bucket.Name))
+				if name := aws.StringValue(bucket.Name); !config.IsBlocked(name) {
+					bucketNames = append(bucketNames, name)
+				}
 			}
 		} else {
 			// Filter and add only the allowed buckets
@@ -171,12 +208,35 @@ func NewBlobHandler(envJson string, authLvl int) (*BlobHandler, error) {
 	return &config, nil
 }
 
-func aWSSessionManager(creds AWSCreds) (*s3.S3, *session.Session, error) {
+// IsBlocked reports whether a bucket is in bucket_block_list.
+func (bh *BlobHandler) IsBlocked(bucket string) bool {
+	_, blocked := bh.BlockedBuckets[bucket]
+	return blocked
+}
+
+func credsLabel(creds AWSCreds) string {
+	if creds.AWS_ACCESS_KEY_ID == "" {
+		return "the EC2 instance role"
+	}
+	return "access key: " + creds.AWS_ACCESS_KEY_ID
+}
+
+func aWSSessionManager(creds AWSCreds, useIAMRole bool) (*s3.S3, *session.Session, error) {
 	log.Info("Using AWS S3")
-	sess, err := session.NewSession(&aws.Config{
-		Region:      aws.String("us-east-1"),
-		Credentials: credentials.NewStaticCredentials(creds.AWS_ACCESS_KEY_ID, creds.AWS_SECRET_ACCESS_KEY, ""),
-	})
+	cfg := &aws.Config{Region: aws.String("us-east-1")}
+	if useIAMRole {
+		// Use only the EC2 instance profile. The SDK default chain would pick
+		// up AWS_ACCESS_KEY_ID/AWS_SECRET_ACCESS_KEY from the environment
+		// first (e.g. a shared .env), silently bypassing the role.
+		metaSess, err := session.NewSession(&aws.Config{Region: aws.String("us-east-1")})
+		if err != nil {
+			return nil, nil, fmt.Errorf("error creating metadata session: %s", err.Error())
+		}
+		cfg.Credentials = ec2rolecreds.NewCredentials(metaSess)
+	} else {
+		cfg.Credentials = credentials.NewStaticCredentials(creds.AWS_ACCESS_KEY_ID, creds.AWS_SECRET_ACCESS_KEY, "")
+	}
+	sess, err := session.NewSession(cfg)
 	if err != nil {
 		return nil, nil, fmt.Errorf("error creating s3 session: %s", err.Error())
 	}

blobstore/buckets.go

@@ -104,7 +104,9 @@ func (bh *BlobHandler) HandleListBuckets(c echo.Context) error {
 			}
 			var mostRecentBucketList []string
 			for _, b := range result.Buckets {
-				mostRecentBucketList = append(mostRecentBucketList, *b.Name)
+				if !bh.IsBlocked(*b.Name) {
+					mostRecentBucketList = append(mostRecentBucketList, *b.Name)
+				}
 			}
 			if !isIdenticalArray(controller.Buckets, mostRecentBucketList) {
 				controller.Buckets = mostRecentBucketList

blobstore/creds.go

@@ -14,8 +14,13 @@ type AWSCreds struct {
 }
 
 type AWSConfig struct {
+	// UseIAMRole ignores any static keys (in this file or AWS_* env vars)
+	// and uses only the EC2 instance profile from instance metadata.
+	UseIAMRole      bool       `json:"use_iam_role"`
 	Accounts        []AWSCreds `json:"accounts"`
 	BucketAllowList []string   `json:"bucket_allow_list"`
+	// BucketBlockList buckets are never exposed, even when allowed (e.g. by "*").
+	BucketBlockList []string `json:"bucket_block_list"`
 }
 
 type MinioConfig struct {
@@ -52,60 +57,58 @@ func (creds MinioConfig) validateMinioConfig() error {
 	return nil
 }
 
-func validateEnvJSON(filePath string) error {
-	// Read the contents of the .env.json file
-	jsonData, err := os.ReadFile(filePath)
-	if err != nil {
-		return fmt.Errorf("error reading .env.json: %s", err.Error())
-	}
-
-	// Parse the JSON data into the AWSConfig struct
-	var awsConfig AWSConfig
-	if err := json.Unmarshal(jsonData, &awsConfig); err != nil {
-		return fmt.Errorf("error parsing .env.json: %s", err.Error())
-	}
-
-	// Check if there is at least one account defined
-	if len(awsConfig.Accounts) == 0 {
-		return fmt.Errorf("no AWS accounts defined in .env.json")
-	}
-
-	// Check if each account has the required fields
-	for i, account := range awsConfig.Accounts {
-		missingFields := []string{}
-		if account.AWS_ACCESS_KEY_ID == "" {
-			missingFields = append(missingFields, "AWS_ACCESS_KEY_ID")
-		}
-		if account.AWS_SECRET_ACCESS_KEY == "" {
-			missingFields = append(missingFields, "AWS_SECRET_ACCESS_KEY")
-		}
-
-		if len(missingFields) > 0 {
-			return fmt.Errorf("missing fields (%s) for AWS account %d in envJson file", strings.Join(missingFields, ", "), i+1)
+// validateAWSConfig checks a parsed .env.json. Static keys are only required
+// when the IAM role is not in use.
+func validateAWSConfig(awsConfig AWSConfig, useIAMRole bool) error {
+	if !useIAMRole {
+		for i, account := range awsConfig.Accounts {
+			missingFields := []string{}
+			if account.AWS_ACCESS_KEY_ID == "" {
+				missingFields = append(missingFields, "AWS_ACCESS_KEY_ID")
+			}
+			if account.AWS_SECRET_ACCESS_KEY == "" {
+				missingFields = append(missingFields, "AWS_SECRET_ACCESS_KEY")
+			}
+			if len(missingFields) > 0 {
+				return fmt.Errorf("missing fields (%s) for AWS account %d in envJson file", strings.Join(missingFields, ", "), i+1)
+			}
 		}
 	}
 	if len(awsConfig.BucketAllowList) == 0 {
 		return fmt.Errorf("no buckets in the `bucket_allow_list`, please provide required buckets, or `*` for access to all buckets")
 	}
-	// If all checks pass, return nil (no error)
 	return nil
 }
 
-func newAWSConfig(envJson string) (AWSConfig, error) {
-	var awsConfig AWSConfig
-	err := validateEnvJSON(envJson)
-	if err != nil {
-		return awsConfig, fmt.Errorf(err.Error())
-	}
+// newAWSConfig reads .env.json. The IAM role is used when the file is empty,
+// defines no "accounts", or sets "use_iam_role": true. Otherwise the static
+// keys in "accounts" are used, as before.
+func newAWSConfig(envJson string) (awsConfig AWSConfig, useIAMRole bool, err error) {
 	jsonData, err := os.ReadFile(envJson)
 	if err != nil {
-		return awsConfig, err
+		return awsConfig, false, fmt.Errorf("error reading .env.json: %s", err.Error())
+	}
+
+	if len(strings.TrimSpace(string(jsonData))) == 0 {
+		// Empty file: IAM role with access to all buckets.
+		awsConfig.BucketAllowList = []string{"*"}
+		useIAMRole = true
+	} else {
+		if err := json.Unmarshal(jsonData, &awsConfig); err != nil {
+			return awsConfig, false, fmt.Errorf("error parsing .env.json: %s", err.Error())
+		}
+		useIAMRole = awsConfig.UseIAMRole || len(awsConfig.Accounts) == 0
+	}
+
+	if err := validateAWSConfig(awsConfig, useIAMRole); err != nil {
+		return awsConfig, false, err
 	}
 
-	if err := json.Unmarshal(jsonData, &awsConfig); err != nil {
-		return awsConfig, err
+	if useIAMRole {
+		// One account with no keys: the session falls back to the IAM role.
+		awsConfig.Accounts = []AWSCreds{{}}
 	}
-	return awsConfig, nil
+	return awsConfig, useIAMRole, nil
 }
 
 func newMinioConfig() MinioConfig {