Skip to content

Commit 9821e02

Browse files
ojungjpbempel
ojung
and
jpbempel
authored
Integrate initial list of third party libraries (#6928)
* Add third party filtering * use pattern for split add explicit exclusions of JDK packages in tests * Return Set from config --------- Co-authored-by: jean-philippe bempel <jean-philippe.bempel@datadoghq.com>
1 parent ff3a8ed commit 9821e02

19 files changed

Lines changed: 449 additions & 136 deletions

File tree

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/DebuggerAgent.java‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
import static com.datadog.debugger.agent.ConfigurationAcceptor.Source.REMOTE_CONFIG;
44
import static datadog.trace.util.AgentThreadFactory.AGENT_THREAD_GROUP;
5-
import static java.util.Collections.emptyList;
65

76
import com.datadog.debugger.exception.DefaultExceptionDebugger;
87
import com.datadog.debugger.sink.DebuggerSink;
@@ -63,8 +62,7 @@ public static synchronized void run(
6362
config, diagnosticEndpoint, ddAgentFeaturesDiscovery.supportsDebuggerDiagnostics());
6463
DebuggerSink debuggerSink = new DebuggerSink(config, probeStatusSink);
6564
debuggerSink.start();
66-
// TODO filtering out thirdparty code
67-
ClassNameFiltering classNameFiltering = new ClassNameFiltering(emptyList());
65+
ClassNameFiltering classNameFiltering = new ClassNameFiltering(config);
6866
ConfigurationUpdater configurationUpdater =
6967
new ConfigurationUpdater(
7068
instrumentation,
@@ -104,7 +102,8 @@ public static synchronized void run(
104102
instrumentation,
105103
config,
106104
new SymbolAggregator(
107-
debuggerSink.getSymbolSink(), config.getDebuggerSymbolFlushThreshold()));
105+
debuggerSink.getSymbolSink(), config.getDebuggerSymbolFlushThreshold()),
106+
classNameFiltering);
108107
if (config.isDebuggerSymbolForceUpload()) {
109108
symDBEnablement.startSymbolExtraction();
110109
}

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/agent/ThirdPartyLibraries.java‎

Lines changed: 83 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
package com.datadog.debugger.agent;
2+
3+
import com.squareup.moshi.JsonAdapter;
4+
import com.squareup.moshi.Moshi;
5+
import datadog.trace.api.Config;
6+
import java.io.BufferedReader;
7+
import java.io.IOException;
8+
import java.io.InputStream;
9+
import java.io.InputStreamReader;
10+
import java.util.HashSet;
11+
import java.util.List;
12+
import java.util.Set;
13+
import java.util.stream.Collectors;
14+
import org.slf4j.Logger;
15+
import org.slf4j.LoggerFactory;
16+
17+
public class ThirdPartyLibraries {
18+
19+
public static final ThirdPartyLibraries INSTANCE = new ThirdPartyLibraries();
20+
public static final char[] ALPHABET = "abcdefghijklmnopqrstuvwxyz".toCharArray();
21+
private static final Logger LOGGER = LoggerFactory.getLogger(ThirdPartyLibraries.class);
22+
private static final JsonAdapter<InternalConfig> ADAPTER =
23+
new Moshi.Builder().build().adapter(InternalConfig.class);
24+
private static final String FILE_NAME = "/third_party_libraries.json";
25+
26+
private ThirdPartyLibraries() {}
27+
28+
public Set<String> getExcludes(Config config) {
29+
try (InputStream inputStream = this.getClass().getResourceAsStream(FILE_NAME)) {
30+
InternalConfig defaults = readConfig(inputStream);
31+
Set<String> excludes =
32+
config.getThirdPartyExcludes().stream()
33+
.filter(s -> !s.isEmpty())
34+
.collect(Collectors.toSet());
35+
excludes.addAll(defaults.getPrefixes());
36+
return excludes;
37+
} catch (Exception e) {
38+
LOGGER.error("Failed reading " + FILE_NAME + ". Treating all classes as third party.", e);
39+
return getExcludeAll();
40+
}
41+
}
42+
43+
public Set<String> getIncludes(Config config) {
44+
return config.getThirdPartyIncludes().stream()
45+
.filter(s -> !s.isEmpty())
46+
.collect(Collectors.toSet());
47+
}
48+
49+
// Add a*, b*, c*, ..., z* to the exclude trie in ClassNameFiltering. Simply adding * does not
50+
// work.
51+
private static Set<String> getExcludeAll() {
52+
Set<String> excludeAllPrefixes = new HashSet<>();
53+
for (char c : ALPHABET) {
54+
excludeAllPrefixes.add(String.valueOf(c));
55+
}
56+
return excludeAllPrefixes;
57+
}
58+
59+
private static InternalConfig readConfig(InputStream inputStream) throws IOException {
60+
try (InputStreamReader isr = new InputStreamReader(inputStream);
61+
BufferedReader reader = new BufferedReader(isr)) {
62+
return ADAPTER.fromJson(reader.lines().collect(Collectors.joining(System.lineSeparator())));
63+
}
64+
}
65+
66+
private static class InternalConfig {
67+
private final String version;
68+
private final List<String> prefixes;
69+
70+
public InternalConfig(String version, List<String> prefixes) {
71+
this.version = version;
72+
this.prefixes = prefixes;
73+
}
74+
75+
public String getVersion() {
76+
return version;
77+
}
78+
79+
public List<String> getPrefixes() {
80+
return prefixes;
81+
}
82+
}
83+
}

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/ExceptionProbeManager.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@ public void createProbesForException(String fingerprint, StackTraceElement[] sta
4444
// TODO log?
4545
continue;
4646
}
47-
if (classNameFiltering.apply(stackTraceElement.getClassName())) {
47+
if (classNameFiltering.isExcluded(stackTraceElement.getClassName())) {
4848
continue;
4949
}
5050
Where where =

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/exception/Fingerprinter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ public static String fingerprint(Throwable t, ClassNameFiltering classNameFilter
3434
StackTraceElement[] stackTrace = t.getStackTrace();
3535
for (StackTraceElement stackTraceElement : stackTrace) {
3636
String className = stackTraceElement.getClassName();
37-
if (classNameFiltering.apply(className)) {
37+
if (classNameFiltering.isExcluded(className)) {
3838
continue;
3939
}
4040
digest.update(stackTraceElement.toString().getBytes());

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymDBEnablement.java‎

Lines changed: 13 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -2,8 +2,7 @@
22

33
import static com.datadog.debugger.symbol.JarScanner.trimPrefixes;
44

5-
import com.datadog.debugger.agent.AllowListHelper;
6-
import com.datadog.debugger.agent.Configuration;
5+
import com.datadog.debugger.util.ClassNameFiltering;
76
import com.datadog.debugger.util.MoshiHelper;
87
import com.squareup.moshi.JsonAdapter;
98
import datadog.remoteconfig.ConfigurationChangesListener;
@@ -24,7 +23,6 @@
2423
import java.time.LocalDateTime;
2524
import java.time.ZoneId;
2625
import java.util.Arrays;
27-
import java.util.Collections;
2826
import java.util.HashSet;
2927
import java.util.Set;
3028
import java.util.concurrent.atomic.AtomicBoolean;
@@ -48,14 +46,19 @@ public class SymDBEnablement implements ProductListener {
4846
private final Config config;
4947
private final SymbolAggregator symbolAggregator;
5048
private SymbolExtractionTransformer symbolExtractionTransformer;
49+
private final ClassNameFiltering classNameFiltering;
5150
private volatile long lastUploadTimestamp;
52-
private AtomicBoolean starting = new AtomicBoolean();
51+
private final AtomicBoolean starting = new AtomicBoolean();
5352

5453
public SymDBEnablement(
55-
Instrumentation instrumentation, Config config, SymbolAggregator symbolAggregator) {
54+
Instrumentation instrumentation,
55+
Config config,
56+
SymbolAggregator symbolAggregator,
57+
ClassNameFiltering classNameFiltering) {
5658
this.instrumentation = instrumentation;
5759
this.config = config;
5860
this.symbolAggregator = symbolAggregator;
61+
this.classNameFiltering = classNameFiltering;
5962
}
6063

6164
@Override
@@ -116,14 +119,12 @@ public void startSymbolExtraction() {
116119
Instant.ofEpochMilli(lastUploadTimestamp), ZoneId.systemDefault()));
117120
return;
118121
}
119-
String includes = config.getDebuggerSymbolIncludes();
120-
AllowListHelper allowListHelper = new AllowListHelper(buildFilterList(includes));
121122
symbolAggregator.loadedClassesProcessStarted();
122123
try {
123124
symbolExtractionTransformer =
124-
new SymbolExtractionTransformer(allowListHelper, symbolAggregator);
125+
new SymbolExtractionTransformer(symbolAggregator, classNameFiltering);
125126
instrumentation.addTransformer(symbolExtractionTransformer);
126-
extractSymbolForLoadedClasses(allowListHelper);
127+
extractSymbolForLoadedClasses();
127128
lastUploadTimestamp = System.currentTimeMillis();
128129
} catch (Throwable ex) {
129130
// catch all Throwables because LinkageError is possible (duplicate class definition)
@@ -136,12 +137,12 @@ public void startSymbolExtraction() {
136137
}
137138
}
138139

139-
private void extractSymbolForLoadedClasses(AllowListHelper allowListHelper) {
140+
private void extractSymbolForLoadedClasses() {
140141
Class<?>[] classesToExtract;
141142
try {
142143
classesToExtract =
143144
Arrays.stream(instrumentation.getAllLoadedClasses())
144-
.filter(clazz -> allowListHelper.isAllowed(clazz.getTypeName()))
145+
.filter(clazz -> !classNameFiltering.isExcluded(clazz.getTypeName()))
145146
.filter(instrumentation::isModifiableClass)
146147
.toArray(Class<?>[]::new);
147148
} catch (Throwable ex) {
@@ -173,7 +174,7 @@ private void extractSymbolForLoadedClasses(AllowListHelper allowListHelper) {
173174
.filter(jarEntry -> jarEntry.getName().endsWith(".class"))
174175
.filter(
175176
jarEntry ->
176-
allowListHelper.isAllowed(
177+
!classNameFiltering.isExcluded(
177178
Strings.getClassName(trimPrefixes(jarEntry.getName()))))
178179
.forEach(jarEntry -> parseJarEntry(jarEntry, jarFile, jarPath, baos, buffer));
179180
}
@@ -199,12 +200,4 @@ private void parseJarEntry(
199200
LOGGER.debug("Exception during parsing jarEntry class: {}", jarEntry.getName(), ex);
200201
}
201202
}
202-
203-
private Configuration.FilterList buildFilterList(String includes) {
204-
if (includes == null || includes.isEmpty()) {
205-
return null;
206-
}
207-
String[] includeParts = COMMA_PATTERN.split(includes);
208-
return new Configuration.FilterList(Arrays.asList(includeParts), Collections.emptyList());
209-
}
210203
}

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/symbol/SymbolExtractionTransformer.java‎

Lines changed: 12 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,6 @@
11
package com.datadog.debugger.symbol;
22

3-
import com.datadog.debugger.agent.AllowListHelper;
4-
import com.datadog.debugger.sink.SymbolSink;
5-
import datadog.trace.api.Config;
6-
import datadog.trace.util.Strings;
3+
import com.datadog.debugger.util.ClassNameFiltering;
74
import java.lang.instrument.ClassFileTransformer;
85
import java.security.ProtectionDomain;
96
import org.slf4j.Logger;
@@ -13,20 +10,13 @@ public class SymbolExtractionTransformer implements ClassFileTransformer {
1310

1411
private static final Logger LOGGER = LoggerFactory.getLogger(SymbolExtractionTransformer.class);
1512

16-
private final AllowListHelper allowListHelper;
1713
private final SymbolAggregator symbolAggregator;
18-
19-
public SymbolExtractionTransformer() {
20-
this(
21-
new AllowListHelper(null),
22-
new SymbolAggregator(
23-
new SymbolSink(Config.get()), Config.get().getDebuggerSymbolFlushThreshold()));
24-
}
14+
private final ClassNameFiltering classNameFiltering;
2515

2616
public SymbolExtractionTransformer(
27-
AllowListHelper allowListHelper, SymbolAggregator symbolAggregator) {
28-
this.allowListHelper = allowListHelper;
17+
SymbolAggregator symbolAggregator, ClassNameFiltering classNameFiltering) {
2918
this.symbolAggregator = symbolAggregator;
19+
this.classNameFiltering = classNameFiltering;
3020
}
3121

3222
@Override
@@ -40,25 +30,12 @@ public byte[] transform(
4030
return null;
4131
}
4232
try {
43-
if (allowListHelper.isAllowAll()) {
44-
if (className.startsWith("java/")
45-
|| className.startsWith("javax/")
46-
|| className.startsWith("jdk/")
47-
|| className.startsWith("sun/")
48-
|| className.startsWith("com/sun/")
49-
|| className.startsWith("datadog/")
50-
|| className.startsWith("com/datadog/")) {
51-
return null;
52-
}
53-
} else {
54-
String javaClassName = Strings.getClassName(className);
55-
if (!allowListHelper.isAllowed(javaClassName)) {
56-
return null;
57-
}
58-
if (javaClassName.startsWith("com.datadog.debugger.symbol.")) {
59-
// Don't parse our own classes to avoid duplicate class definition
60-
return null;
61-
}
33+
if (className.startsWith("com/datadog/debugger/symbol/")) {
34+
// Don't parse our own classes to avoid duplicate class definition
35+
return null;
36+
}
37+
if (classNameFiltering.isExcluded(className)) {
38+
return null;
6239
}
6340
symbolAggregator.parseClass(className, classfileBuffer, protectionDomain);
6441
return null;
@@ -68,7 +45,7 @@ public byte[] transform(
6845
}
6946
}
7047

71-
AllowListHelper getAllowListHelper() {
72-
return allowListHelper;
48+
ClassNameFiltering getClassNameFiltering() {
49+
return classNameFiltering;
7350
}
7451
}

‎dd-java-agent/agent-debugger/src/main/java/com/datadog/debugger/util/ClassNameFiltering.java‎

Lines changed: 31 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -1,24 +1,41 @@
11
package com.datadog.debugger.util;
22

3+
import com.datadog.debugger.agent.ThirdPartyLibraries;
4+
import datadog.trace.api.Config;
35
import datadog.trace.util.ClassNameTrie;
4-
import java.util.Arrays;
5-
import java.util.List;
6+
import java.util.Collections;
7+
import java.util.Set;
68

79
/** A class to filter out classes based on their package name. */
810
public class ClassNameFiltering {
9-
// Hardcode filtering out JDK classes
10-
private static final List<String> JDK_FILTER_OUT_PACKAGES =
11-
Arrays.asList("java.", "javax.", "sun.", "com.sun.", "jdk.");
12-
private final ClassNameTrie trie;
13-
14-
public ClassNameFiltering(List<String> packages) {
15-
ClassNameTrie.Builder builder = new ClassNameTrie.Builder();
16-
JDK_FILTER_OUT_PACKAGES.forEach(s -> builder.put(s + "*", 1));
17-
packages.forEach(s -> builder.put(s + "*", 1));
18-
this.trie = builder.buildTrie();
11+
12+
private final ClassNameTrie includeTrie;
13+
private final ClassNameTrie excludeTrie;
14+
15+
public ClassNameFiltering(Config config) {
16+
this(
17+
ThirdPartyLibraries.INSTANCE.getExcludes(config),
18+
ThirdPartyLibraries.INSTANCE.getIncludes(config));
19+
}
20+
21+
public ClassNameFiltering(Set<String> excludes) {
22+
this(excludes, Collections.emptySet());
23+
}
24+
25+
public ClassNameFiltering(Set<String> excludes, Set<String> includes) {
26+
ClassNameTrie.Builder excludeBuilder = new ClassNameTrie.Builder();
27+
excludes.forEach(s -> excludeBuilder.put(s + "*", 1));
28+
this.excludeTrie = excludeBuilder.buildTrie();
29+
ClassNameTrie.Builder includeBuilder = new ClassNameTrie.Builder();
30+
includes.forEach(s -> includeBuilder.put(s + "*", 1));
31+
this.includeTrie = includeBuilder.buildTrie();
32+
}
33+
34+
public boolean isExcluded(String className) {
35+
return includeTrie.apply(className) < 0 && excludeTrie.apply(className) > 0;
1936
}
2037

21-
public boolean apply(String className) {
22-
return trie.apply(className) > 0;
38+
public static ClassNameFiltering allowAll() {
39+
return new ClassNameFiltering(Collections.emptySet(), Collections.emptySet());
2340
}
2441
}

‎dd-java-agent/agent-debugger/src/main/resources/third_party_libraries.json‎

Lines changed: 1 addition & 0 deletions
Large diffs are not rendered by default.

0 commit comments

Comments
 (0)