Add decom reasons to task cleaning messages

Author: pschoenfelder

SingularityService/src/main/java/com/hubspot/singularity/scheduler/SingularityScheduler.java

@@ -24,6 +24,7 @@
 import com.hubspot.singularity.SingularityDeployStatisticsBuilder;
 import com.hubspot.singularity.SingularityKilledTaskIdRecord;
 import com.hubspot.singularity.SingularityMachineAbstraction;
+import com.hubspot.singularity.SingularityMachineStateHistoryUpdate;
 import com.hubspot.singularity.SingularityMainModule;
 import com.hubspot.singularity.SingularityManagedThreadPoolFactory;
 import com.hubspot.singularity.SingularityPendingDeploy;
@@ -152,7 +153,8 @@ private void cleanupTaskDueToDecomission(
     final Map<String, Optional<String>> requestIdsToUserToReschedule,
     final Set<SingularityTaskId> matchingTaskIds,
     SingularityTask task,
-    SingularityMachineAbstraction<?> decommissioningObject
+    SingularityMachineAbstraction<?> decommissioningObject,
+    Optional<String> decomissionReason
   ) {
     requestIdsToUserToReschedule.put(
       task.getTaskRequest().getRequest().getId(),
@@ -175,9 +177,10 @@ private void cleanupTaskDueToDecomission(
         task.getTaskId(),
         Optional.of(
           String.format(
-            "%s %s is decomissioning",
+            "%s %s is decomissioning%s",
             decommissioningObject.getTypeName(),
-            decommissioningObject.getName()
+            decommissioningObject.getName(),
+            decomissionReason.map(r -> String.format(" (message: %s)", r)).orElse("")
           )
         ),
         Optional.<String>empty(),
@@ -211,6 +214,17 @@ public void checkForDecomissions() {
 
     for (SingularityAgent agent : agents.keySet()) {
       boolean foundTask = false;
+      List<SingularityMachineStateHistoryUpdate> history = agentManager.getHistory(
+        agent.getId()
+      );
+
+      // TODO: Is there a guaranteed order these come out of ZK in to prevent sort?
+      //       Probably nbd because this list will be very short, but still
+      SingularityMachineStateHistoryUpdate stateHistoryUpdate = history
+        .stream()
+        .filter(update -> update.getState() == MachineState.STARTING_DECOMMISSION)
+        .max(Comparator.comparing(SingularityMachineStateHistoryUpdate::getTimestamp))
+        .get();
 
       for (SingularityTask activeTask : taskManager.getTasksOnAgent(
         activeTaskIds,
@@ -220,7 +234,8 @@ public void checkForDecomissions() {
           requestIdsToUserToReschedule,
           matchingTaskIds,
           activeTask,
-          agent
+          agent,
+          stateHistoryUpdate.getMessage()
         );
         foundTask = true;
       }
@@ -256,7 +271,8 @@ public void checkForDecomissions() {
             requestIdsToUserToReschedule,
             matchingTaskIds,
             maybeTask.get(),
-            rack
+            rack,
+            Optional.empty()
           );
         }
       }