Skip to content

Commit 2859aae

Browse files
ssalinasssalinas
committed
Fix merge
2 parents 8eef30d + e246c3c commit 2859aae

19 files changed

Lines changed: 239 additions & 63 deletions

‎SingularityBase/src/main/java/com/hubspot/singularity/SingularityScheduledTasksInfo.java‎

Lines changed: 33 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,20 +1,25 @@
11
package com.hubspot.singularity;
22

3+
import com.fasterxml.jackson.annotation.JsonCreator;
4+
import com.fasterxml.jackson.annotation.JsonProperty;
5+
import io.swagger.v3.oas.annotations.media.Schema;
36
import java.util.List;
47

8+
@Schema(description = "Describes the current scheduled tasks in Singularity")
59
public class SingularityScheduledTasksInfo {
610
private final int numFutureTasks;
711
private final long maxTaskLag;
812
private final long timestamp;
913
private final List<SingularityPendingTaskId> lateTasks;
1014
private final List<SingularityPendingTaskId> onDemandLateTasks;
1115

16+
@JsonCreator
1217
public SingularityScheduledTasksInfo(
13-
List<SingularityPendingTaskId> lateTasks,
14-
List<SingularityPendingTaskId> onDemandLateTasks,
15-
int numFutureTasks,
16-
long maxTaskLag,
17-
long timestamp
18+
@JsonProperty("lateTasks") List<SingularityPendingTaskId> lateTasks,
19+
@JsonProperty("onDemandLateTasks") List<SingularityPendingTaskId> onDemandLateTasks,
20+
@JsonProperty("numFutureTasks") int numFutureTasks,
21+
@JsonProperty("maxTaskLag") long maxTaskLag,
22+
@JsonProperty("timestamp") long timestamp
1823
) {
1924
this.lateTasks = lateTasks;
2025
this.onDemandLateTasks = onDemandLateTasks;
@@ -23,23 +28,46 @@ public SingularityScheduledTasksInfo(
2328
this.timestamp = timestamp;
2429
}
2530

31+
@Schema(description = "List of late task ids")
2632
public List<SingularityPendingTaskId> getLateTasks() {
2733
return lateTasks;
2834
}
2935

36+
@Schema(description = "List of on demand late task ids")
3037
public List<SingularityPendingTaskId> getOnDemandLateTasks() {
3138
return onDemandLateTasks;
3239
}
3340

41+
@Schema(description = "Number of future tasks")
3442
public int getNumFutureTasks() {
3543
return numFutureTasks;
3644
}
3745

46+
@Schema(description = "Maximum task lag in ms")
3847
public long getMaxTaskLag() {
3948
return maxTaskLag;
4049
}
4150

51+
@Schema(description = "Timestamp")
4252
public long getTimestamp() {
4353
return timestamp;
4454
}
55+
56+
@Override
57+
public String toString() {
58+
return (
59+
"SingularityScheduledTasksInfo{" +
60+
"lateTasks=" +
61+
lateTasks +
62+
", onDemandLateTasks=" +
63+
onDemandLateTasks +
64+
", numFutureTasks" +
65+
numFutureTasks +
66+
", maxTaskLag" +
67+
maxTaskLag +
68+
", timestamp" +
69+
timestamp +
70+
'}'
71+
);
72+
}
4573
}

‎SingularityClient/src/main/java/com/hubspot/singularity/client/SingularityClient.java‎

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import com.hubspot.singularity.SingularityRequestWithState;
5454
import com.hubspot.singularity.SingularityS3Log;
5555
import com.hubspot.singularity.SingularitySandbox;
56+
import com.hubspot.singularity.SingularityScheduledTasksInfo;
5657
import com.hubspot.singularity.SingularityShellCommand;
5758
import com.hubspot.singularity.SingularitySlave;
5859
import com.hubspot.singularity.SingularityState;
@@ -118,6 +119,8 @@ public class SingularityClient {
118119
AUTH_FORMAT + "/groups/auth-check";
119120

120121
private static final String STATE_FORMAT = "%s/state";
122+
private static final String SCHEDULED_TASKS_INFO_FORMAT =
123+
STATE_FORMAT + "/scheduled-tasks-info";
121124
private static final String TASK_RECONCILIATION_FORMAT =
122125
STATE_FORMAT + "/task-reconciliation";
123126

@@ -829,6 +832,28 @@ private HttpResponse executeRequest(
829832
// GLOBAL
830833
//
831834

835+
public SingularityScheduledTasksInfo getScheduledTasksInfo() {
836+
final Function<String, String> uri = host ->
837+
String.format(SCHEDULED_TASKS_INFO_FORMAT, getApiBase(host));
838+
839+
LOG.info("Fetching scheduled tasks info from {}", uri);
840+
841+
final long start = System.currentTimeMillis();
842+
843+
HttpResponse response = executeRequest(
844+
uri,
845+
Method.GET,
846+
Optional.empty(),
847+
Collections.emptyMap()
848+
);
849+
850+
checkResponse("singularity scheduled tasks info", response);
851+
852+
LOG.info("Got scheduled tasks info in {}ms", System.currentTimeMillis() - start);
853+
854+
return response.getAs(SingularityScheduledTasksInfo.class);
855+
}
856+
832857
public SingularityState getState(
833858
Optional<Boolean> skipCache,
834859
Optional<Boolean> includeRequestIds

‎SingularityService/src/main/java/com/hubspot/singularity/SingularityService.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ public void initialize(final Bootstrap<T> bootstrap) {
100100

101101
GuiceBundle.Builder<SingularityConfiguration> guiceBundleBuilder = GuiceBundle
102102
.defaultBuilder(SingularityConfiguration.class)
103-
.modules(getServiceModule(), getObjectMapperModule(), new SingularityAuthModule())
103+
.modules(getServiceModule(), getObjectMapperModule(), getAuthModule())
104104
.modules(additionalModules)
105105
.enableGuiceEnforcer(false)
106106
.stage(getGuiceStage());
@@ -171,6 +171,10 @@ public Module getObjectMapperModule() {
171171
.in(Scopes.SINGLETON);
172172
}
173173

174+
public Module getAuthModule() {
175+
return new SingularityAuthModule();
176+
}
177+
174178
/**
175179
* Dropwizard bundles used in addition to the bundles required by Singularity. This is an extension point when embedding
176180
* Singularity into a custom service.

‎SingularityService/src/main/java/com/hubspot/singularity/auth/SingularityAuthModule.java‎

Lines changed: 22 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -19,15 +19,21 @@
1919
import com.ning.http.client.AsyncHttpClient;
2020
import com.ning.http.client.AsyncHttpClientConfig;
2121
import java.util.HashSet;
22+
import java.util.Optional;
2223
import java.util.Set;
2324

2425
public class SingularityAuthModule
2526
extends DropwizardAwareModule<SingularityConfiguration> {
2627
public static final String WEBHOOK_AUTH_HTTP_CLIENT =
2728
"singularity.webhook.auth.http.client";
29+
private Optional<Class<? extends SingularityAuthorizer>> authorizerClass = Optional.empty();
2830

2931
public SingularityAuthModule() {}
3032

33+
public void setAuthorizerClass(Class<? extends SingularityAuthorizer> authorizerClass) {
34+
this.authorizerClass = Optional.of(authorizerClass);
35+
}
36+
3137
@Override
3238
public void configure(Binder binder) {
3339
binder
@@ -64,27 +70,10 @@ public void configure(Binder binder) {
6470
}
6571
}
6672

67-
switch (getConfiguration().getAuthConfiguration().getAuthMode()) {
68-
case GROUPS_SCOPES:
69-
binder
70-
.bind(SingularityAuthorizer.class)
71-
.to(SingularityGroupsScopesAuthorizer.class)
72-
.in(Scopes.SINGLETON);
73-
break;
74-
case GROUPS_LOG_SCOPES:
75-
binder
76-
.bind(SingularityAuthorizer.class)
77-
.to(SingularityDualAuthorizer.class)
78-
.in(Scopes.SINGLETON);
79-
break;
80-
case GROUPS:
81-
default:
82-
binder
83-
.bind(SingularityAuthorizer.class)
84-
.to(SingularityGroupsAuthorizer.class)
85-
.in(Scopes.SINGLETON);
86-
break;
87-
}
73+
binder
74+
.bind(SingularityAuthorizer.class)
75+
.to(authorizerClass.orElseGet(this::getAuthClassFromConfig))
76+
.in(Scopes.SINGLETON);
8877

8978
switch (getConfiguration().getAuthConfiguration().getAuthResponseParser()) {
9079
case RAW:
@@ -110,4 +99,16 @@ public void configure(Binder binder) {
11099
getConfiguration().getAuthConfiguration().getDatastore().getAuthDatastoreClass()
111100
);
112101
}
102+
103+
private Class<? extends SingularityAuthorizer> getAuthClassFromConfig() {
104+
switch (getConfiguration().getAuthConfiguration().getAuthMode()) {
105+
case GROUPS_SCOPES:
106+
return SingularityGroupsScopesAuthorizer.class;
107+
case GROUPS_LOG_SCOPES:
108+
return SingularityDualAuthorizer.class;
109+
case GROUPS:
110+
default:
111+
return SingularityGroupsAuthorizer.class;
112+
}
113+
}
113114
}

‎SingularityService/src/main/java/com/hubspot/singularity/auth/SingularityAuthorizer.java‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
import com.google.common.collect.Sets;
1111
import com.hubspot.singularity.InvalidSingularityTaskIdException;
1212
import com.hubspot.singularity.SingularityAuthorizationScope;
13+
import com.hubspot.singularity.SingularityDeploy;
1314
import com.hubspot.singularity.SingularityRequest;
1415
import com.hubspot.singularity.SingularityRequestWithState;
1516
import com.hubspot.singularity.SingularityTaskId;
@@ -31,7 +32,7 @@ public SingularityAuthorizer(RequestManager requestManager, boolean authEnabled)
3132
this.authEnabled = authEnabled;
3233
}
3334

34-
static boolean groupsIntersect(Set<String> a, Set<String> b) {
35+
public static boolean groupsIntersect(Set<String> a, Set<String> b) {
3536
return !Sets.intersection(a, b).isEmpty();
3637
}
3738

@@ -67,6 +68,13 @@ protected abstract void checkForAuthorization(
6768
Optional<SingularityUserFacingAction> action
6869
);
6970

71+
public void checkForAuthorization(
72+
SingularityRequest request,
73+
SingularityDeploy deploy,
74+
SingularityUser user,
75+
SingularityAuthorizationScope scope
76+
) {}
77+
7078
public boolean isAuthorizedForRequest(
7179
SingularityRequest request,
7280
SingularityUser user,
@@ -91,6 +99,16 @@ protected abstract boolean isAuthorizedForRequest(
9199
Optional<SingularityUserFacingAction> action
92100
);
93101

102+
public void checkForAuthorizedChanges(
103+
SingularityRequest newRequest,
104+
Optional<SingularityRequest> oldRequest,
105+
SingularityUser user
106+
) {
107+
if (oldRequest.isPresent()) {
108+
checkForAuthorizedChanges(newRequest, oldRequest.get(), user);
109+
}
110+
}
111+
94112
public abstract void checkForAuthorizedChanges(
95113
SingularityRequest request,
96114
SingularityRequest oldRequest,

‎SingularityService/src/main/java/com/hubspot/singularity/auth/SingularityGroupsScopesAuthorizer.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,7 @@ public class SingularityGroupsScopesAuthorizer extends SingularityAuthorizer {
2727
SingularityGroupsScopesAuthorizer.class
2828
);
2929

30-
private final AuthConfiguration authConfiguration;
30+
protected final AuthConfiguration authConfiguration;
3131
private final ScopesConfiguration scopesConfiguration;
3232
private final SingularityEventListener singularityEventListener;
3333

@@ -227,7 +227,7 @@ private boolean isJita(SingularityUser user) {
227227
return groupsIntersect(authConfiguration.getJitaGroups(), user.getGroups());
228228
}
229229

230-
private boolean isAdmin(SingularityUser user) {
230+
protected boolean isAdmin(SingularityUser user) {
231231
return hasScope(user, SingularityAuthorizationScope.ADMIN);
232232
}
233233

@@ -290,7 +290,7 @@ private void checkForbiddenForGroups(
290290
);
291291
}
292292

293-
private Set<String> getGroups(
293+
protected Set<String> getGroups(
294294
SingularityRequest request,
295295
SingularityAuthorizationScope scope
296296
) {

‎SingularityService/src/main/java/com/hubspot/singularity/config/SingularityConfiguration.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -453,6 +453,8 @@ public class SingularityConfiguration extends Configuration {
453453
private int deployCacheTtlInSeconds = 5;
454454
private int requestCacheTtlInSeconds = 5;
455455

456+
private boolean skipPersistingTooLongTaskIds = false;
457+
456458
public long getAskDriverToKillTasksAgainAfterMillis() {
457459
return askDriverToKillTasksAgainAfterMillis;
458460
}
@@ -2119,4 +2121,12 @@ public int getHistoryPollerConcurrency() {
21192121
public void setHistoryPollerConcurrency(int historyPollerConcurrency) {
21202122
this.historyPollerConcurrency = historyPollerConcurrency;
21212123
}
2124+
2125+
public boolean skipPersistingTooLongTaskIds() {
2126+
return skipPersistingTooLongTaskIds;
2127+
}
2128+
2129+
public void setSkipPersistingTooLongTaskIds(boolean skipPersistingTooLongTaskIds) {
2130+
this.skipPersistingTooLongTaskIds = skipPersistingTooLongTaskIds;
2131+
}
21222132
}

‎SingularityService/src/main/java/com/hubspot/singularity/data/StateManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -338,7 +338,7 @@ public SingularityState generateState(boolean includeRequestIds) {
338338
);
339339
}
340340

341-
private SingularityScheduledTasksInfo getScheduledTasksInfo() {
341+
public SingularityScheduledTasksInfo getScheduledTasksInfo() {
342342
long now = System.currentTimeMillis();
343343
List<SingularityPendingTaskId> allPendingTaskIds = taskManager.getPendingTaskIds();
344344
List<SingularityPendingTaskId> lateTasks = allPendingTaskIds

‎SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityDeployHistoryPersister.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -139,6 +139,7 @@ public void runActionOnPoll() {
139139
if (moveToHistoryOrCheckForPurge(deployHistory, i++)) {
140140
numTransferred.increment();
141141
} else {
142+
LOG.error("Deploy History Persister failed on {}", deployHistory);
142143
persisterSuccess.getAndSet(false);
143144
}
144145
}
@@ -221,7 +222,7 @@ protected boolean moveToHistory(SingularityDeployHistory deployHistory) {
221222
try {
222223
historyManager.saveDeployHistory(deployHistory);
223224
} catch (Throwable t) {
224-
LOG.warn(
225+
LOG.error(
225226
"Failed to persist deploy {}",
226227
SingularityDeployKey.fromDeployMarker(deployHistory.getDeployMarker()),
227228
t

‎SingularityService/src/main/java/com/hubspot/singularity/data/history/SingularityRequestHistoryPersister.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,9 +23,9 @@
2323
import java.util.concurrent.atomic.AtomicBoolean;
2424
import java.util.concurrent.atomic.AtomicInteger;
2525
import java.util.concurrent.atomic.AtomicLong;
26-
import java.util.concurrent.locks.ReadWriteLock;
2726
import java.util.concurrent.locks.ReentrantLock;
2827
import javax.inject.Singleton;
28+
import javax.ws.rs.HEAD;
2929
import org.slf4j.Logger;
3030
import org.slf4j.LoggerFactory;
3131

@@ -177,6 +177,10 @@ public void runActionOnPoll() {
177177
) {
178178
numHistoryTransferred.getAndAdd(requestHistoryParent.history.size());
179179
} else {
180+
LOG.error(
181+
"Request History Persister failed on {}",
182+
requestHistoryParent
183+
);
180184
persisterSuccess.getAndSet(false);
181185
}
182186
},
@@ -227,7 +231,7 @@ protected boolean moveToHistory(SingularityRequestHistoryParent object) {
227231
try {
228232
historyManager.saveRequestHistoryUpdate(requestHistory);
229233
} catch (Throwable t) {
230-
LOG.warn("Failed to persist {} into History", requestHistory, t);
234+
LOG.error("Failed to persist {} into History", requestHistory, t);
231235
return false;
232236
}
233237

0 commit comments

Comments
 (0)