Add single parameter dd.data.jobs.enabled to enable DJM (#6972)

Add single parameter dd.data.jobs.enabled=true to enable data jobs monitoring.

This parameter will change the default to have:
- dd.trace.experimental.long-running.enabled=true
- dd.integration.spark.enabled=true
- dd.integration.spark-executor.enabled=true

Only the default are changed, those parameters can still be changed by explicitly providing them

Author: paul-laffon-dd

dd-java-agent/agent-bootstrap/src/main/java/datadog/trace/bootstrap/Agent.java

@@ -100,7 +100,8 @@ private enum AgentFeature {
         propertyNameToSystemPropertyName(CiVisibilityConfig.CIVISIBILITY_AGENTLESS_ENABLED), false),
     USM(propertyNameToSystemPropertyName(UsmConfig.USM_ENABLED), false),
     TELEMETRY(propertyNameToSystemPropertyName(GeneralConfig.TELEMETRY_ENABLED), true),
-    DEBUGGER(propertyNameToSystemPropertyName(DebuggerConfig.DEBUGGER_ENABLED), false);
+    DEBUGGER(propertyNameToSystemPropertyName(DebuggerConfig.DEBUGGER_ENABLED), false),
+    DATA_JOBS(propertyNameToSystemPropertyName(GeneralConfig.DATA_JOBS_ENABLED), false);
 
     private final String systemProp;
     private final boolean enabledByDefault;
@@ -197,6 +198,18 @@ public static void start(final Instrumentation inst, final URL agentJarURL, Stri
       }
     }
 
+    boolean dataJobsEnabled = isFeatureEnabled(AgentFeature.DATA_JOBS);
+    if (dataJobsEnabled) {
+      log.info("Data Jobs Monitoring enabled, enabling spark integrations");
+
+      setSystemPropertyDefault(
+          propertyNameToSystemPropertyName(TracerConfig.TRACE_LONG_RUNNING_ENABLED), "true");
+      setSystemPropertyDefault(
+          propertyNameToSystemPropertyName("integration.spark.enabled"), "true");
+      setSystemPropertyDefault(
+          propertyNameToSystemPropertyName("integration.spark-executor.enabled"), "true");
+    }
+
     if (!isSupportedAppSecArch()) {
       log.debug(
           "OS and architecture ({}/{}) not supported by AppSec, dd.appsec.enabled will default to false",

dd-trace-api/src/main/java/datadog/trace/api/ConfigDefaults.java

@@ -189,6 +189,8 @@ public final class ConfigDefaults {
   static final boolean DEFAULT_CWS_ENABLED = false;
   static final int DEFAULT_CWS_TLS_REFRESH = 5000;
 
+  static final boolean DEFAULT_DATA_JOBS_ENABLED = false;
+
   static final boolean DEFAULT_DATA_STREAMS_ENABLED = false;
   static final int DEFAULT_DATA_STREAMS_BUCKET_DURATION = 10; // seconds
 

dd-trace-api/src/main/java/datadog/trace/api/config/GeneralConfig.java

@@ -66,6 +66,8 @@ public final class GeneralConfig {
   public static final String AZURE_APP_SERVICES = "azure.app.services";
   public static final String INTERNAL_EXIT_ON_FAILURE = "trace.internal.exit.on.failure";
 
+  public static final String DATA_JOBS_ENABLED = "data.jobs.enabled";
+
   public static final String DATA_STREAMS_ENABLED = "data.streams.enabled";
   public static final String DATA_STREAMS_BUCKET_DURATION_SECONDS =
       "data.streams.bucket_duration.seconds";

internal-api/src/main/java/datadog/trace/api/Config.java

@@ -33,6 +33,7 @@
 import static datadog.trace.api.ConfigDefaults.DEFAULT_COUCHBASE_INTERNAL_SPANS_ENABLED;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_CWS_ENABLED;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_CWS_TLS_REFRESH;
+import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_JOBS_ENABLED;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_BUCKET_DURATION;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_DATA_STREAMS_ENABLED;
 import static datadog.trace.api.ConfigDefaults.DEFAULT_DB_CLIENT_HOST_SPLIT_BY_HOST;
@@ -233,6 +234,7 @@
 import static datadog.trace.api.config.GeneralConfig.APPLICATION_KEY;
 import static datadog.trace.api.config.GeneralConfig.APPLICATION_KEY_FILE;
 import static datadog.trace.api.config.GeneralConfig.AZURE_APP_SERVICES;
+import static datadog.trace.api.config.GeneralConfig.DATA_JOBS_ENABLED;
 import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_BUCKET_DURATION_SECONDS;
 import static datadog.trace.api.config.GeneralConfig.DATA_STREAMS_ENABLED;
 import static datadog.trace.api.config.GeneralConfig.DOGSTATSD_ARGS;
@@ -900,6 +902,8 @@ static class HostNameHolder {
   private final boolean cwsEnabled;
   private final int cwsTlsRefresh;
 
+  private final boolean dataJobsEnabled;
+
   private final boolean dataStreamsEnabled;
   private final float dataStreamsBucketDurationSeconds;
 
@@ -1975,6 +1979,8 @@ PROFILING_DATADOG_PROFILER_ENABLED, isDatadogProfilerSafeInCurrentEnvironment())
     cwsEnabled = configProvider.getBoolean(CWS_ENABLED, DEFAULT_CWS_ENABLED);
     cwsTlsRefresh = configProvider.getInteger(CWS_TLS_REFRESH, DEFAULT_CWS_TLS_REFRESH);
 
+    dataJobsEnabled = configProvider.getBoolean(DATA_JOBS_ENABLED, DEFAULT_DATA_JOBS_ENABLED);
+
     dataStreamsEnabled =
         configProvider.getBoolean(DATA_STREAMS_ENABLED, DEFAULT_DATA_STREAMS_ENABLED);
     dataStreamsBucketDurationSeconds =
@@ -3456,7 +3462,7 @@ public boolean isAxisPromoteResourceName() {
   }
 
   public boolean isDataJobsEnabled() {
-    return instrumenterConfig.isDataJobsEnabled();
+    return dataJobsEnabled;
   }
 
   /** @return A map of tags to be applied only to the local application root span. */
@@ -4572,6 +4578,8 @@ public String toString() {
         + telemetryMetricsEnabled
         + ", appSecScaEnabled="
         + appSecScaEnabled
+        + ", dataJobsEnabled="
+        + dataJobsEnabled
         + '}';
   }
 }

internal-api/src/main/java/datadog/trace/api/InstrumenterConfig.java

@@ -69,7 +69,6 @@
 import java.lang.reflect.Method;
 import java.util.Arrays;
 import java.util.Collection;
-import java.util.Collections;
 import java.util.List;
 import java.util.Map;
 import java.util.Set;
@@ -271,12 +270,6 @@ public boolean isIntegrationShortcutMatchingEnabled(
         integrationNames, "integration.", ".matching.shortcut.enabled", defaultEnabled);
   }
 
-  public boolean isDataJobsEnabled() {
-    // there's no dedicated flag to enabled DJM, it's enough to just enable
-    // spark instrumentation
-    return isIntegrationEnabled(Collections.singletonList("spark"), false);
-  }
-
   public boolean isTraceEnabled() {
     return traceEnabled;
   }