Add generic AWS S3 domain support (#1470)

Signed-off-by: Bala.FA <bala@minio.io>

Author: balamurugana

api/src/main/java/io/minio/BucketArgs.java

@@ -16,7 +16,10 @@
 
 package io.minio;
 
+import io.minio.http.HttpUtils;
+import io.minio.org.apache.commons.validator.routines.InetAddressValidator;
 import java.util.Objects;
+import java.util.regex.Pattern;
 
 /** Base argument class holds bucket name and region. */
 public abstract class BucketArgs extends BaseArgs {
@@ -34,32 +37,35 @@ public String region() {
   /** Base argument builder class for {@link BucketArgs}. */
   public abstract static class Builder<B extends Builder<B, A>, A extends BucketArgs>
       extends BaseArgs.Builder<B, A> {
+    private static final Pattern BUCKET_NAME_REGEX =
+        Pattern.compile("^[a-z0-9][a-z0-9\\.\\-]{1,61}[a-z0-9]$");
+
     protected void validateBucketName(String name) {
       validateNotNull(name, "bucket name");
 
-      // Bucket names cannot be no less than 3 and no more than 63 characters long.
-      if (name.length() < 3 || name.length() > 63) {
+      if (!BUCKET_NAME_REGEX.matcher(name).find()) {
         throw new IllegalArgumentException(
-            name + " : " + "bucket name must be at least 3 and no more than 63 characters long");
+            "bucket name '"
+                + name
+                + "' does not follow Amazon S3 standards. For more information refer "
+                + "https://docs.aws.amazon.com/AmazonS3/latest/userguide/bucketnamingrules.html");
       }
-      // Successive periods in bucket names are not allowed.
-      if (name.contains("..")) {
-        String msg =
-            "bucket name cannot contain successive periods. For more information refer "
-                + "http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html";
-        throw new IllegalArgumentException(name + " : " + msg);
+
+      if (InetAddressValidator.getInstance().isValidInet4Address(name)) {
+        throw new IllegalArgumentException(
+            "bucket name '" + name + "' must not be formatted as an IP address");
       }
-      // Bucket names should be dns compatible.
-      if (!name.matches("^[a-z0-9][a-z0-9\\.\\-]+[a-z0-9]$")) {
-        String msg =
-            "bucket name does not follow Amazon S3 standards. For more information refer "
-                + "http://docs.aws.amazon.com/AmazonS3/latest/dev/BucketRestrictions.html";
-        throw new IllegalArgumentException(name + " : " + msg);
+
+      if (name.contains("..") || name.contains(".-") || name.contains("-.")) {
+        throw new IllegalArgumentException(
+            "bucket name '" + name + "' cannot contain successive characters '..', '.-' and '-.'");
       }
     }
 
     private void validateRegion(String region) {
-      validateNullOrNotEmptyString(region, "region");
+      if (region != null && !HttpUtils.REGION_REGEX.matcher(region).find()) {
+        throw new IllegalArgumentException("invalid region " + region);
+      }
     }
 
     @Override

api/src/main/java/io/minio/MinioAsyncClient.java

@@ -66,6 +66,7 @@
 import java.nio.file.StandardOpenOption;
 import java.security.InvalidKeyException;
 import java.security.NoSuchAlgorithmException;
+import java.util.Arrays;
 import java.util.Date;
 import java.util.Iterator;
 import java.util.LinkedList;
@@ -75,6 +76,7 @@
 import java.util.concurrent.CompletableFuture;
 import java.util.concurrent.CompletionException;
 import java.util.concurrent.ExecutionException;
+import java.util.regex.Matcher;
 import okhttp3.HttpUrl;
 import okhttp3.OkHttpClient;
 import okhttp3.Request;
@@ -131,22 +133,20 @@
 public class MinioAsyncClient extends S3Base {
   private MinioAsyncClient(
       HttpUrl baseUrl,
-      String region,
-      boolean isAwsHost,
-      boolean isFipsHost,
-      boolean isAccelerateHost,
-      boolean isDualStackHost,
+      String awsS3Prefix,
+      String awsDomainSuffix,
+      boolean awsDualstack,
       boolean useVirtualStyle,
+      String region,
       Provider provider,
       OkHttpClient httpClient) {
     super(
         baseUrl,
-        region,
-        isAwsHost,
-        isFipsHost,
-        isAccelerateHost,
-        isDualStackHost,
+        awsS3Prefix,
+        awsDomainSuffix,
+        awsDualstack,
         useVirtualStyle,
+        region,
         provider,
         httpClient);
   }
@@ -3213,87 +3213,66 @@ public static Builder builder() {
   /** Argument builder of {@link MinioClient}. */
   public static final class Builder {
     private HttpUrl baseUrl;
-    private String region;
-    private boolean isAwsHost;
-    private boolean isFipsHost;
-    private boolean isAccelerateHost;
-    private boolean isDualStackHost;
+    private String awsS3Prefix;
+    private String awsDomainSuffix;
+    private boolean awsDualstack;
     private boolean useVirtualStyle;
+
+    private String region;
     private Provider provider;
     private OkHttpClient httpClient;
 
-    private boolean isAwsChinaHost;
-    private String regionInUrl;
+    private void setAwsInfo(String host, boolean https) {
+      this.awsS3Prefix = null;
+      this.awsDomainSuffix = null;
+      this.awsDualstack = false;
 
-    private boolean isAwsFipsEndpoint(String endpoint) {
-      return endpoint.startsWith("s3-fips.");
-    }
+      if (!HttpUtils.HOSTNAME_REGEX.matcher(host).find()) return;
 
-    private boolean isAwsAccelerateEndpoint(String endpoint) {
-      return endpoint.startsWith("s3-accelerate.");
-    }
+      if (HttpUtils.AWS_ELB_ENDPOINT_REGEX.matcher(host).find()) {
+        String[] tokens = host.split("\\.elb\\.amazonaws\\.com", 1)[0].split("\\.");
+        this.region = tokens[tokens.length - 1];
+        return;
+      }
 
-    private boolean isAwsEndpoint(String endpoint) {
-      return (endpoint.startsWith("s3.")
-              || isAwsFipsEndpoint(endpoint)
-              || isAwsAccelerateEndpoint(endpoint))
-          && (endpoint.endsWith(".amazonaws.com") || endpoint.endsWith(".amazonaws.com.cn"));
-    }
+      if (!HttpUtils.AWS_ENDPOINT_REGEX.matcher(host).find()) return;
 
-    private boolean isAwsDualStackEndpoint(String endpoint) {
-      return endpoint.contains(".dualstack.");
-    }
+      if (!HttpUtils.AWS_S3_ENDPOINT_REGEX.matcher(host).find()) {
+        throw new IllegalArgumentException("invalid Amazon AWS host " + host);
+      }
+
+      Matcher matcher = HttpUtils.AWS_S3_PREFIX_REGEX.matcher(host);
+      matcher.lookingAt();
+      int end = matcher.end();
+
+      this.awsS3Prefix = host.substring(0, end);
+      if (this.awsS3Prefix.contains("s3-accesspoint") && !https) {
+        throw new IllegalArgumentException("use HTTPS scheme for host " + host);
+      }
 
-    /**
-     * Extracts region from AWS endpoint if available. Region is placed at second token normal
-     * endpoints and third token for dualstack endpoints.
-     *
-     * <p>Region is marked in square brackets in below examples.
-     * <pre>
-     * https://s3.[us-east-2].amazonaws.com
-     * https://s3.dualstack.[ca-central-1].amazonaws.com
-     * https://s3.[cn-north-1].amazonaws.com.cn
-     * https://s3.dualstack.[cn-northwest-1].amazonaws.com.cn
-     */
-    private String extractRegion(String endpoint) {
-      String[] tokens = endpoint.split("\\.");
-      String token = tokens[1];
-
-      // If token is "dualstack", then region might be in next token.
-      if (token.equals("dualstack")) {
-        token = tokens[2];
+      String[] tokens = host.substring(end).split("\\.");
+      awsDualstack = "dualstack".equals(tokens[0]);
+      if (awsDualstack) tokens = Arrays.copyOfRange(tokens, 1, tokens.length);
+      String regionInHost = null;
+      if (!tokens[0].equals("vpce") && !tokens[0].equals("amazonaws")) {
+        regionInHost = tokens[0];
+        tokens = Arrays.copyOfRange(tokens, 1, tokens.length);
       }
+      this.awsDomainSuffix = String.join(".", tokens);
 
-      // If token is equal to "amazonaws", region is not passed in the endpoint.
-      if (token.equals("amazonaws")) {
-        return null;
+      if (host.equals("s3-external-1.amazonaws.com")) regionInHost = "us-east-1";
+      if (host.equals("s3-us-gov-west-1.amazonaws.com")
+          || host.equals("s3-fips-us-gov-west-1.amazonaws.com")) {
+        regionInHost = "us-gov-west-1";
       }
 
-      // Return token as region.
-      return token;
+      if (regionInHost != null) this.region = regionInHost;
     }
 
     private void setBaseUrl(HttpUrl url) {
-      String host = url.host();
-
-      this.isAwsHost = isAwsEndpoint(host);
-      this.isAwsChinaHost = false;
-      if (this.isAwsHost) {
-        this.isAwsChinaHost = host.endsWith(".cn");
-        url =
-            url.newBuilder()
-                .host(this.isAwsChinaHost ? "amazonaws.com.cn" : "amazonaws.com")
-                .build();
-        this.isFipsHost = isAwsFipsEndpoint(host);
-        this.isAccelerateHost = isAwsAccelerateEndpoint(host);
-        this.isDualStackHost = isAwsDualStackEndpoint(host);
-        this.regionInUrl = extractRegion(host);
-        this.useVirtualStyle = true;
-      } else {
-        this.useVirtualStyle = host.endsWith("aliyuncs.com");
-      }
-
       this.baseUrl = url;
+      this.setAwsInfo(url.host(), url.isHttps());
+      this.useVirtualStyle = this.awsDomainSuffix != null || url.host().endsWith("aliyuncs.com");
     }
 
     public Builder endpoint(String endpoint) {
@@ -3325,8 +3304,9 @@ public Builder endpoint(HttpUrl url) {
     }
 
     public Builder region(String region) {
-      HttpUtils.validateNullOrNotEmptyString(region, "region");
-      this.regionInUrl = region;
+      if (region != null && !HttpUtils.REGION_REGEX.matcher(region).find()) {
+        throw new IllegalArgumentException("invalid region " + region);
+      }
       this.region = region;
       return this;
     }
@@ -3349,7 +3329,11 @@ public Builder httpClient(OkHttpClient httpClient) {
 
     public MinioAsyncClient build() {
       HttpUtils.validateNotNull(this.baseUrl, "endpoint");
-      if (this.isAwsChinaHost && this.regionInUrl == null && this.region == null) {
+
+      if (this.awsDomainSuffix != null
+          && this.awsDomainSuffix.endsWith(".cn")
+          && !this.awsS3Prefix.endsWith("s3-accelerate.")
+          && this.region == null) {
         throw new IllegalArgumentException(
             "Region missing in Amazon S3 China endpoint " + this.baseUrl);
       }
@@ -3358,17 +3342,15 @@ public MinioAsyncClient build() {
         this.httpClient =
             HttpUtils.newDefaultHttpClient(
                 DEFAULT_CONNECTION_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT, DEFAULT_CONNECTION_TIMEOUT);
-        if (this.region == null) this.region = regionInUrl;
       }
 
       return new MinioAsyncClient(
           baseUrl,
-          region,
-          isAwsHost,
-          isFipsHost,
-          isAccelerateHost,
-          isDualStackHost,
+          awsS3Prefix,
+          awsDomainSuffix,
+          awsDualstack,
           useVirtualStyle,
+          region,
           provider,
           httpClient);
     }

api/src/main/java/io/minio/MinioClient.java

@@ -2405,12 +2405,22 @@ public void traceOff() throws IOException {
     asyncClient.traceOff();
   }
 
-  /** Enables accelerate endpoint for Amazon S3 endpoint. */
+  /**
+   * Enables accelerate endpoint for Amazon S3 endpoint.
+   *
+   * @deprecated This method is no longer supported.
+   */
+  @Deprecated
   public void enableAccelerateEndpoint() {
     asyncClient.enableAccelerateEndpoint();
   }
 
-  /** Disables accelerate endpoint for Amazon S3 endpoint. */
+  /**
+   * Disables accelerate endpoint for Amazon S3 endpoint.
+   *
+   * @deprecated This method is no longer supported.
+   */
+  @Deprecated
   public void disableAccelerateEndpoint() {
     asyncClient.disableAccelerateEndpoint();
   }
@@ -2435,6 +2445,11 @@ public void disableVirtualStyleEndpoint() {
     asyncClient.disableVirtualStyleEndpoint();
   }
 
+  /** Sets AWS S3 domain prefix. */
+  public void setAwsS3Prefix(String awsS3Prefix) {
+    asyncClient.setAwsS3Prefix(awsS3Prefix);
+  }
+
   public static Builder builder() {
     return new Builder();
   }