Merge pull request #2246 from HubSpot/request-level-maxscale

Add request-level maximum scale option

Author: ashdza

SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequest.java

@@ -14,13 +14,14 @@
 import java.util.Set;
 
 @Schema(
-  description = "Settings that apply to all tasks and deploys assocaited with this request"
+  description = "Settings that apply to all tasks and deploys associated with this request"
 )
 public class SingularityRequest {
   private final String id;
   private final RequestType requestType;
   private final Optional<List<String>> owners;
   private final Optional<Integer> numRetriesOnFailure;
+  private final Optional<Integer> maxScale;
   private final Optional<String> schedule;
   private final Optional<String> quartzSchedule;
   private final Optional<ScheduleType> scheduleType;
@@ -61,6 +62,7 @@ public SingularityRequest(
     @JsonProperty("requestType") RequestType requestType,
     @JsonProperty("owners") Optional<List<String>> owners,
     @JsonProperty("numRetriesOnFailure") Optional<Integer> numRetriesOnFailure,
+    @JsonProperty("maxScale") Optional<Integer> maxScale,
     @JsonProperty("schedule") Optional<String> schedule,
     @JsonProperty("instances") Optional<Integer> instances,
     @JsonProperty("rackSensitive") Optional<Boolean> rackSensitive,
@@ -129,6 +131,7 @@ public SingularityRequest(
     this.id = checkNotNull(id, "id cannot be null");
     this.owners = owners;
     this.numRetriesOnFailure = numRetriesOnFailure;
+    this.maxScale = maxScale;
     this.schedule = schedule;
     this.rackSensitive = rackSensitive;
     this.instances = instances;
@@ -183,6 +186,7 @@ public SingularityRequestBuilder toBuilder() {
       .setLoadBalanced(loadBalanced)
       .setInstances(instances)
       .setNumRetriesOnFailure(numRetriesOnFailure)
+      .setMaxScale(maxScale)
       .setOwners(copyOfList(owners))
       .setRackSensitive(rackSensitive)
       .setSchedule(schedule)
@@ -243,6 +247,10 @@ public Optional<Integer> getNumRetriesOnFailure() {
     return numRetriesOnFailure;
   }
 
+  public Optional<Integer> getMaxScale() {
+    return maxScale;
+  }
+
   @Schema(nullable = true, description = "A schedule in cron, RFC5545, or quartz format")
   public Optional<String> getSchedule() {
     return schedule;
@@ -586,6 +594,7 @@ public boolean equals(Object o) {
       requestType == that.requestType &&
       Objects.equals(owners, that.owners) &&
       Objects.equals(numRetriesOnFailure, that.numRetriesOnFailure) &&
+      Objects.equals(maxScale, that.maxScale) &&
       Objects.equals(schedule, that.schedule) &&
       Objects.equals(quartzSchedule, that.quartzSchedule) &&
       Objects.equals(scheduleType, that.scheduleType) &&
@@ -639,6 +648,7 @@ public int hashCode() {
       requestType,
       owners,
       numRetriesOnFailure,
+      maxScale,
       schedule,
       quartzSchedule,
       scheduleType,
@@ -686,6 +696,8 @@ public String toString() {
       owners +
       ", numRetriesOnFailure=" +
       numRetriesOnFailure +
+      ", maxScale=" +
+      maxScale +
       ", schedule=" +
       schedule +
       ", quartzSchedule=" +

SingularityBase/src/main/java/com/hubspot/singularity/SingularityRequestBuilder.java

@@ -14,6 +14,7 @@ public class SingularityRequestBuilder {
 
   private Optional<List<String>> owners;
   private Optional<Integer> numRetriesOnFailure;
+  private Optional<Integer> maxScale;
 
   private Optional<String> schedule;
   private Optional<String> quartzSchedule;
@@ -60,6 +61,7 @@ public SingularityRequestBuilder(String id, RequestType requestType) {
     this.requestType = checkNotNull(requestType, "requestType cannot be null");
     this.owners = Optional.empty();
     this.numRetriesOnFailure = Optional.empty();
+    this.maxScale = Optional.empty();
     this.schedule = Optional.empty();
     this.scheduleType = Optional.empty();
     this.killOldNonLongRunningTasksAfterMillis = Optional.empty();
@@ -99,6 +101,7 @@ public SingularityRequest build() {
       requestType,
       owners,
       numRetriesOnFailure,
+      maxScale,
       schedule,
       instances,
       rackSensitive,
@@ -187,6 +190,15 @@ public SingularityRequestBuilder setNumRetriesOnFailure(
     return this;
   }
 
+  public Optional<Integer> getMaxScale() {
+    return maxScale;
+  }
+
+  public SingularityRequestBuilder setMaxScale(Optional<Integer> maxScale) {
+    this.maxScale = maxScale;
+    return this;
+  }
+
   public Optional<String> getSchedule() {
     return schedule;
   }
@@ -535,6 +547,7 @@ public boolean equals(Object o) {
       requestType == that.requestType &&
       Objects.equals(owners, that.owners) &&
       Objects.equals(numRetriesOnFailure, that.numRetriesOnFailure) &&
+      Objects.equals(maxScale, that.maxScale) &&
       Objects.equals(schedule, that.schedule) &&
       Objects.equals(quartzSchedule, that.quartzSchedule) &&
       Objects.equals(scheduleTimeZone, that.scheduleTimeZone) &&
@@ -588,6 +601,7 @@ public int hashCode() {
       requestType,
       owners,
       numRetriesOnFailure,
+      maxScale,
       schedule,
       quartzSchedule,
       scheduleTimeZone,
@@ -635,6 +649,8 @@ public String toString() {
       owners +
       ", numRetriesOnFailure=" +
       numRetriesOnFailure +
+      ", maxScale=" +
+      maxScale +
       ", schedule=" +
       schedule +
       ", quartzSchedule=" +

SingularityService/src/main/java/com/hubspot/singularity/data/SingularityValidator.java

@@ -119,6 +119,8 @@ public class SingularityValidator {
   private final AgentManager agentManager;
   private final LoadBalancerClient loadBalancerClient;
 
+  private final SingularityConfiguration singularityConfiguration;
+
   @Inject
   public SingularityValidator(
     SingularityConfiguration configuration,
@@ -187,6 +189,8 @@ public SingularityValidator(
     this.disasterManager = disasterManager;
     this.agentManager = agentManager;
     this.loadBalancerClient = loadBalancerClient;
+
+    this.singularityConfiguration = configuration;
   }
 
   public SingularityRequest checkSingularityRequest(
@@ -230,13 +234,23 @@ public SingularityRequest checkSingularityRequest(
       !request.getInstances().isPresent() || request.getInstances().get() > 0,
       "Instances must be greater than 0"
     );
+
     checkBadRequest(
       request.getInstancesSafe() <= maxInstancesPerRequest,
       "Instances (%s) be greater than %s (maxInstancesPerRequest in mesos configuration)",
       request.getInstancesSafe(),
       maxInstancesPerRequest
     );
 
+    if (request.getRequestType().isLongRunning() && request.getMaxScale().isPresent()) {
+      checkBadRequest(
+        request.getInstancesSafe() <= request.getMaxScale().get(),
+        "Instances (%s) cannot be greater than %s (maxScale in request)",
+        request.getInstancesSafe(),
+        request.getMaxScale().get()
+      );
+    }
+
     if (request.getTaskPriorityLevel().isPresent()) {
       checkBadRequest(
         request.getTaskPriorityLevel().get() >= 0 &&
@@ -1200,6 +1214,15 @@ public void checkScale(SingularityRequest request, Optional<Integer> previousSca
         requiredAgentCount,
         currentActiveAgentCount
       );
+
+      if (request.getRequestType().isLongRunning() && request.getMaxScale().isPresent()) {
+        checkBadRequest(
+          request.getInstancesSafe() <= request.getMaxScale().get(),
+          "Instances (%s) cannot be greater than %s (maxScale in request)",
+          request.getInstancesSafe(),
+          request.getMaxScale().get()
+        );
+      }
     }
   }
 

SingularityService/src/test/java/com/hubspot/singularity/data/ValidatorTest.java

@@ -161,6 +161,58 @@ public void itForbidsTooLongDeployId() {
     );
   }
 
+  @Test
+  public void itForbidsInstancesGreaterThanRequestMaxScale() {
+    int globalMaxScale = configuration
+      .getMesosConfiguration()
+      .getMaxNumInstancesPerRequest();
+    int requestMaxScale = globalMaxScale - 5;
+    SingularityRequest request = new SingularityRequestBuilder(
+      "requestId",
+      RequestType.SERVICE
+    )
+      .setMaxScale(Optional.of(requestMaxScale)) // request level max scale < global max scale
+      .setInstances(Optional.of(requestMaxScale + 1)) // instances > request level max scale
+      .build();
+
+    Assertions.assertThrows(
+      WebApplicationException.class,
+      () ->
+        validator.checkSingularityRequest(
+          request,
+          Optional.empty(),
+          Optional.empty(),
+          Optional.empty()
+        )
+    );
+  }
+
+  @Test
+  public void itForbidsInstancesGreaterThanGlobalMaxScale() {
+    int globalMaxScale = configuration
+      .getMesosConfiguration()
+      .getMaxNumInstancesPerRequest();
+    int requestMaxScale = globalMaxScale + 5;
+    SingularityRequest request = new SingularityRequestBuilder(
+      "requestId",
+      RequestType.SERVICE
+    )
+      .setMaxScale(Optional.of(requestMaxScale)) // global max scale < request level max scale
+      .setInstances(Optional.of(globalMaxScale + 1)) // instances > global max scale (mesos config)
+      .build();
+
+    Assertions.assertThrows(
+      WebApplicationException.class,
+      () ->
+        validator.checkSingularityRequest(
+          request,
+          Optional.empty(),
+          Optional.empty(),
+          Optional.empty()
+        )
+    );
+  }
+
   @Test
   public void itForbidsRunNowOfScheduledWhenAlreadyRunning() {
     String deployID = "deploy";