From 911c215a37988a3b90f465f6e0c4151f157162d1 Mon Sep 17 00:00:00 2001 From: Nikita Ogorodnikov Date: Fri, 16 Oct 2026 11:08:20 +0200 Subject: [PATCH] RUM-19005: Upload all packaged native libraries, skip stripped ones With the new Variant API, upload the native libraries from the variant's MERGED_NATIVE_LIBS artifact: everything packaged into the variant (built by the app, prebuilt jniLibs and AAR dependencies), after packaging excludes and before AGP strips them. The artifact carries the dependency on the tasks producing it, so the upload task no longer looks up native build tasks when it is configured, which could find none if it was configured early. Prebuilt and dependency libraries usually ship already stripped. Only their exported symbols could be symbolicated, and the backend keeps the first file per build ID, name and ABI, so a stripped copy could block the copy with symbols. Read the ELF section names of each .so and skip it when it has no .symtab, .debug_info or .zdebug_info. Files which can't be parsed are still uploaded. Warn when every library found was stripped. Also enable the NDK upload test with gzip disabled, which was missing @Test and never disabled gzip. Co-Authored-By: Claude Opus 5.5 --- .../gradle/plugin/NdkSymbolFileUploadTask.kt | 15 + .../plugin/internal/ElfSymbolsDetector.kt | 202 ++++++++++++ .../internal/variant/NewApiAppVariant.kt | 18 +- .../DdAndroidGradlePluginFunctionalTest.kt | 62 +++- .../plugin/NdkSymbolFileUploadTaskTest.kt | 69 +++- .../plugin/internal/ElfSymbolsDetectorTest.kt | 296 ++++++++++++++++++ .../gradle/plugin/utils/ElfFileBuilder.kt | 102 ++++++ ...build_with_native_and_prebuilt_libs.gradle | 83 +++++ 8 files changed, 829 insertions(+), 18 deletions(-) create mode 100644 dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetector.kt create mode 100644 dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetectorTest.kt create mode 100644 dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/utils/ElfFileBuilder.kt create mode 100644 dd-sdk-android-gradle-plugin/src/test/resources/native_gradle_files/build_with_native_and_prebuilt_libs.gradle diff --git a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTask.kt b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTask.kt index 438d9d9d..54e8f370 100644 --- a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTask.kt +++ b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTask.kt @@ -7,6 +7,7 @@ package com.datadog.gradle.plugin import com.datadog.gradle.plugin.internal.ApiKey +import com.datadog.gradle.plugin.internal.ElfSymbolsDetector import com.datadog.gradle.plugin.internal.Uploader import com.datadog.gradle.plugin.internal.utils.capitalizeChar import com.datadog.gradle.plugin.internal.variant.AppVariant @@ -40,6 +41,7 @@ internal abstract class NdkSymbolFileUploadTask @Inject constructor( override fun getFilesList(): List { val files = mutableListOf() + val strippedFiles = mutableListOf() searchDirectories .flatMap(this::findSoFiles) @@ -57,6 +59,12 @@ internal abstract class NdkSymbolFileUploadTask @Inject constructor( return@forEach } archToFiles.value.forEach { + if (ElfSymbolsDetector.detect(it) == ElfSymbolsDetector.Result.STRIPPED) { + // prebuilt and dependency libraries are often packaged already stripped + logger.info("Native library ${it.absolutePath} has no symbols, ignoring it.") + strippedFiles.add(it) + return@forEach + } files.add( Uploader.UploadFileInfo( KEY_NDK_SYMBOL_FILE, @@ -72,6 +80,13 @@ internal abstract class NdkSymbolFileUploadTask @Inject constructor( } } + if (files.isEmpty() && strippedFiles.isNotEmpty()) { + logger.warn( + "All the native libraries found have no symbols (they are stripped), nothing to upload: " + + strippedFiles.joinToString(", ") { it.absolutePath } + ) + } + return files } diff --git a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetector.kt b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetector.kt new file mode 100644 index 00000000..88990a35 --- /dev/null +++ b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetector.kt @@ -0,0 +1,202 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2020-Present Datadog, Inc. + */ + +package com.datadog.gradle.plugin.internal + +import com.datadog.gradle.plugin.DdAndroidGradlePlugin.Companion.LOGGER +import java.io.File +import java.io.IOException +import java.io.RandomAccessFile +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * Tells whether a native library still has the symbols needed for symbolication, by reading the + * types and names of the sections listed in its ELF section header table. + */ +@Suppress("MagicNumber") +internal object ElfSymbolsDetector { + + enum class Result { + /** The library has a symbol table or debug info. */ + HAS_SYMBOLS, + + /** The library is a valid ELF file without symbol table and debug info. */ + STRIPPED, + + /** The file is not an ELF file, or it can't be read. */ + UNKNOWN + } + + fun detect(file: File): Result { + return try { + RandomAccessFile(file, "r").use { detect(it) } + } catch (e: IOException) { + onReadError(file, e) + } catch (e: SecurityException) { + onReadError(file, e) + } + } + + // region Private + + private fun onReadError(file: File, e: Exception): Result { + LOGGER.info("Unable to read native library ${file.absolutePath} to check its symbols", e) + return Result.UNKNOWN + } + + // any inconsistency in the file structure ends the detection early + @Suppress("ReturnCount") + private fun detect(file: RandomAccessFile): Result { + val header = file.read(0, ELF64_HEADER_SIZE) ?: return Result.UNKNOWN + if (!ELF_MAGIC.indices.all { header.get(it) == ELF_MAGIC[it] }) return Result.UNKNOWN + val is64Bit = when (header.get(EI_CLASS).toInt()) { + ELFCLASS32 -> false + ELFCLASS64 -> true + else -> return Result.UNKNOWN + } + when (header.get(EI_DATA).toInt()) { + ELFDATA2LSB -> header.order(ByteOrder.LITTLE_ENDIAN) + ELFDATA2MSB -> header.order(ByteOrder.BIG_ENDIAN) + else -> return Result.UNKNOWN + } + + val layout = if (is64Bit) ELF64_LAYOUT else ELF32_LAYOUT + val sectionHeadersOffset = header.readAddress(layout.shoffOffset, is64Bit) + val sectionHeaderSize = header.getShort(layout.shentsizeOffset).toUnsignedInt() + val sectionsCount = header.getShort(layout.shnumOffset).toUnsignedInt() + val namesSectionIndex = header.getShort(layout.shstrndxOffset).toUnsignedInt() + // `strip --strip-section-headers` drops the section header table, which implies stripping all the symbols: + // only the dynamic symbols reachable through the program headers are left, as for any stripped library + if (sectionHeadersOffset == 0L && sectionsCount == 0) return Result.STRIPPED + // extended numbering (more than 0xff00 sections) is not supported, such libraries are not expected. + // The section header size is fixed by the ELF class, checking it bounds the section header table size. + if (sectionsCount == 0 || + sectionHeaderSize != layout.sectionHeaderSize || + namesSectionIndex >= sectionsCount + ) { + return Result.UNKNOWN + } + + val sectionHeaders = file.read(sectionHeadersOffset, sectionsCount.toLong() * sectionHeaderSize) + ?.order(header.order()) + ?: return Result.UNKNOWN + return detect(file, sectionHeaders, sectionsCount, namesSectionIndex, layout, is64Bit) + } + + @Suppress("ReturnCount", "LongParameterList") + private fun detect( + file: RandomAccessFile, + sectionHeaders: ByteBuffer, + sectionsCount: Int, + namesSectionIndex: Int, + layout: Layout, + is64Bit: Boolean + ): Result { + // the symbol table is found by its type, as symbolic does, so it doesn't depend on the section names + val hasSymbolTable = (0 until sectionsCount).any { index -> + sectionHeaders.getInt(index * layout.sectionHeaderSize + SH_TYPE_OFFSET) == SHT_SYMTAB + } + if (hasSymbolTable) return Result.HAS_SYMBOLS + // debug info sections are only found by their names, which can't be checked without the names table + if (namesSectionIndex == SHN_UNDEF) return Result.UNKNOWN + + val namesHeaderOffset = namesSectionIndex * layout.sectionHeaderSize + val namesSize = sectionHeaders.readAddress(namesHeaderOffset + layout.shSizeOffset, is64Bit) + if (namesSize > MAX_NAMES_SECTION_SIZE) return Result.UNKNOWN + val names = file.read( + sectionHeaders.readAddress(namesHeaderOffset + layout.shOffsetOffset, is64Bit), + namesSize + ) ?: return Result.UNKNOWN + + val hasDebugInfo = (0 until sectionsCount).any { index -> + val nameOffset = sectionHeaders.getInt(index * layout.sectionHeaderSize).toUnsignedLong() + DEBUG_INFO_SECTION_NAMES.any { names.hasNameAt(nameOffset, it) } + } + return if (hasDebugInfo) Result.HAS_SYMBOLS else Result.STRIPPED + } + + private fun RandomAccessFile.read(offset: Long, size: Long): ByteBuffer? { + val isSizeValid = size in 0..Int.MAX_VALUE + if (!isSizeValid || offset !in 0..length() - size) return null + val bytes = ByteArray(size.toInt()) + seek(offset) + readFully(bytes) + return ByteBuffer.wrap(bytes) + } + + private fun ByteBuffer.readAddress(offset: Int, is64Bit: Boolean): Long { + // a negative value (> Long.MAX_VALUE as unsigned) is rejected later as out of the file bounds + return if (is64Bit) getLong(offset) else getInt(offset).toUnsignedLong() + } + + // compares at most name.size + 1 bytes, so that many sections pointing at a long unterminated + // name don't make the check scan the whole table for each of them + private fun ByteBuffer.hasNameAt(offset: Long, name: ByteArray): Boolean { + if (offset < 0 || offset + name.size >= limit()) return false + val start = offset.toInt() + return name.indices.all { get(start + it) == name[it] } && get(start + name.size) == 0.toByte() + } + + private fun Short.toUnsignedInt(): Int = toInt() and 0xFFFF + + private fun Int.toUnsignedLong(): Long = toLong() and 0xFFFFFFFFL + + private class Layout( + val shoffOffset: Int, + val shentsizeOffset: Int, + val shnumOffset: Int, + val shstrndxOffset: Int, + val sectionHeaderSize: Int, + val shOffsetOffset: Int, + val shSizeOffset: Int + ) + + // endregion + + private val ELF_MAGIC = byteArrayOf(0x7F, 'E'.code.toByte(), 'L'.code.toByte(), 'F'.code.toByte()) + private const val EI_CLASS = 4 + private const val EI_DATA = 5 + private const val ELFCLASS32 = 1 + private const val ELFCLASS64 = 2 + private const val ELFDATA2LSB = 1 + private const val ELFDATA2MSB = 2 + + // the 64-bit header is the bigger one, the 32-bit one fits into it + private const val ELF64_HEADER_SIZE = 64L + + // section names table is usually smaller than 1 KB, anything bigger is not a sane library + private const val MAX_NAMES_SECTION_SIZE = 1024L * 1024L + + private val ELF32_LAYOUT = Layout( + shoffOffset = 0x20, + shentsizeOffset = 0x2E, + shnumOffset = 0x30, + shstrndxOffset = 0x32, + sectionHeaderSize = 0x28, + shOffsetOffset = 0x10, + shSizeOffset = 0x14 + ) + + private val ELF64_LAYOUT = Layout( + shoffOffset = 0x28, + shentsizeOffset = 0x3A, + shnumOffset = 0x3C, + shstrndxOffset = 0x3E, + sectionHeaderSize = 0x40, + shOffsetOffset = 0x18, + shSizeOffset = 0x20 + ) + + private const val SH_TYPE_OFFSET = 4 + private const val SHT_SYMTAB = 2 + private const val SHN_UNDEF = 0 + + // debug info sections removed by stripping, which are needed to symbolicate a native crash. `.gnu_debugdata` + // (MiniDebugInfo) is not listed, as symbolic doesn't read it. + private val DEBUG_INFO_SECTION_NAMES = listOf(".debug_info", ".zdebug_info") + .map { it.toByteArray(Charsets.US_ASCII) } +} diff --git a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/variant/NewApiAppVariant.kt b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/variant/NewApiAppVariant.kt index cf110857..70548756 100644 --- a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/variant/NewApiAppVariant.kt +++ b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/variant/NewApiAppVariant.kt @@ -10,13 +10,11 @@ import com.android.build.api.artifact.SingleArtifact import com.android.build.api.dsl.ApplicationExtension import com.android.build.api.variant.ApplicationVariant import com.android.build.api.variant.VariantOutput -import com.android.build.gradle.tasks.ExternalNativeBuildTask import com.datadog.gradle.plugin.GenerateBuildIdTask import com.datadog.gradle.plugin.InjectBuildIdToAssetsTask import com.datadog.gradle.plugin.MappingFileUploadTask import com.datadog.gradle.plugin.NdkSymbolFileUploadTask import com.datadog.gradle.plugin.internal.CurrentAgpVersion -import com.datadog.gradle.plugin.internal.getSearchObjDirs import com.datadog.gradle.plugin.internal.utils.capitalizeChar import org.gradle.api.Project import org.gradle.api.artifacts.Configuration @@ -75,18 +73,10 @@ internal class NewApiAppVariant( } override fun bindWith(ndkUploadTask: NdkSymbolFileUploadTask) { - // New Variant API doesn't expose native build tasks. AGP builds each distinct native configuration - // once (`buildCMake[]`, attributed to a single representative variant), and the - // `externalNativeBuild` task of each variant depends on the ones it uses, so we follow it. - // Its own output folder is not used: it is either empty or holds copies of the same libraries. - target.tasks.withType(ExternalNativeBuildTask::class.java) - .filter { it.variantName == variant.name } - .forEach { variantNativeBuildTask -> - ndkUploadTask.dependsOn(variantNativeBuildTask) - variantNativeBuildTask.taskDependencies.getDependencies(variantNativeBuildTask) - .filterIsInstance() - .forEach { ndkUploadTask.searchDirectories.from(it.getSearchObjDirs()) } - } + // All the native libraries packaged into this variant (built by the app, prebuilt and coming from + // dependencies), not stripped yet. Libraries which are already stripped are skipped by the task. + // The artifact also brings the dependency on the tasks producing it. + ndkUploadTask.searchDirectories.from(variant.artifacts.get(SingleArtifact.MERGED_NATIVE_LIBS)) } override fun bindWith(mappingFileUploadTask: MappingFileUploadTask) { diff --git a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/DdAndroidGradlePluginFunctionalTest.kt b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/DdAndroidGradlePluginFunctionalTest.kt index 03f5f142..2d336c85 100644 --- a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/DdAndroidGradlePluginFunctionalTest.kt +++ b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/DdAndroidGradlePluginFunctionalTest.kt @@ -6,7 +6,9 @@ package com.datadog.gradle.plugin +import com.datadog.gradle.plugin.internal.ElfSymbolsDetector import com.datadog.gradle.plugin.internal.utils.capitalizeChar +import com.datadog.gradle.plugin.utils.ElfFileBuilder import com.datadog.gradle.plugin.utils.assertj.BuildResultAssert.Companion.assertThat import com.datadog.gradle.plugin.utils.forge.Configurator import com.datadog.gradle.plugin.utils.headHash @@ -1496,13 +1498,70 @@ internal class DdAndroidGradlePluginFunctionalTest { val uploadedFilesPerTask = uploadedNdkSymbolFilesPerTask(result.output) assertThat(uploadedFilesPerTask.keys).containsExactlyInAnyOrderElementsOf(uploadTaskNames) uploadedFilesPerTask.forEach { (taskName, uploadedFiles) -> + val variantName = taskName.removePrefix("uploadNdkSymbolFiles").replaceFirstChar { it.lowercase() } assertThat(uploadedFiles) .describedAs("files uploaded by $taskName") .hasSize(SUPPORTED_ABIS.size) - .allMatch { it.contains("/RelWithDebInfo/") && it.endsWith("/libplaceholder.so") } + .allMatch { it.contains("/merged_native_libs/$variantName/") && it.endsWith("/libplaceholder.so") } } } + @Test + fun `M upload all the packaged native libraries with symbols W upload symbol file`(forge: Forge) { + // Given + // new Variant API is only used starting from AGP 8.4 + buildVersionConfig = LATEST_VERSIONS_TEST_CONFIGURATION + stubGradlePropertiesFile(buildVersionConfig) + stubGradleBuildFromResourceFile( + "lib_module_build.gradle", + libModuleBuildGradleFile, + overwrite = true + ) + stubGradleBuildFromResourceFile( + "native_gradle_files/build_with_native_and_prebuilt_libs.gradle", + appBuildGradleFile + ) + stubNativeFiles() + val prebuiltLibsDir = File(appMainSrcDir, "jniLibs/arm64-v8a").apply { mkdirs() } + File(prebuiltLibsDir, "libprebuilt.so").writeBytes(ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS)) + File(prebuiltLibsDir, "libstripped.so").writeBytes(ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS)) + File(prebuiltLibsDir, "libexcluded.so").writeBytes(ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS)) + + val variant = "${forge.anElementFrom(versions).lowercase()}${forge.anElementFrom(colors)}" + val taskName = resolveNdkSymbolUploadTask(variant) + + gradleRunner { withArguments(":samples:app:assembleRelease") } + .build() + + // When + val result = gradleRunner { + withArguments( + taskName, + "--info", + "-PDD_API_KEY=fakekey", + "-Pdd-emulate-upload-call" + ) + } + .build() + + // Then + val uploadedFiles = uploadedNdkSymbolFilesPerTask(result.output).getValue(taskName) + assertThat(uploadedFiles.map { File(it).name }) + .containsExactlyInAnyOrderElementsOf(List(SUPPORTED_ABIS.size) { "libplaceholder.so" } + "libprebuilt.so") + assertThat(result).containsInOutput("libstripped.so has no symbols, ignoring it.") + + // the libraries built by the app are detected with symbols before AGP strips them, and without after + val intermediatesDir = File(appRootDir, "build/intermediates") + val builtLibs = intermediatesDir.walkTopDown().filter { it.name == "libplaceholder.so" }.toList() + assertThat(builtLibs.filter { it.path.contains("/merged_native_libs/") }) + .isNotEmpty + .allMatch { ElfSymbolsDetector.detect(it) == ElfSymbolsDetector.Result.HAS_SYMBOLS } + assertThat(builtLibs.filter { it.path.contains("/stripped_native_libs/") }) + .isNotEmpty + .allMatch { ElfSymbolsDetector.detect(it) == ElfSymbolsDetector.Result.STRIPPED } + } + + @Test fun `M try to upload the symbol file W upload { using a fake API_KEY, gzip disabled }`(forge: Forge) { // Given stubGradleBuildFromResourceFile( @@ -1526,6 +1585,7 @@ internal class DdAndroidGradlePluginFunctionalTest { "--info", "--stacktrace", "-PDD_API_KEY=fakekey", + "-Pdd-disable-gzip", "-Pdd-emulate-upload-call" ) } diff --git a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTaskTest.kt b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTaskTest.kt index a43a41bf..705e6133 100644 --- a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTaskTest.kt +++ b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTaskTest.kt @@ -11,6 +11,7 @@ import com.datadog.gradle.plugin.internal.ApiKeySource import com.datadog.gradle.plugin.internal.DdAppIdentifier import com.datadog.gradle.plugin.internal.Uploader import com.datadog.gradle.plugin.internal.variant.AppVariant +import com.datadog.gradle.plugin.utils.ElfFileBuilder import com.datadog.gradle.plugin.utils.forge.Configurator import fr.xgouchet.elmyr.Forge import fr.xgouchet.elmyr.annotation.Forgery @@ -36,6 +37,7 @@ import org.mockito.kotlin.doReturn import org.mockito.kotlin.eq import org.mockito.kotlin.verify import org.mockito.kotlin.verifyNoInteractions +import org.mockito.kotlin.verifyNoMoreInteractions import org.mockito.kotlin.whenever import org.mockito.quality.Strictness import java.io.File @@ -441,6 +443,63 @@ internal class NdkSymbolFileUploadTaskTest { verifyNoInteractions(mockUploader) } + @Test + fun `M upload only libraries with symbols W applyTask() { some libraries are stripped }`() { + // Given + whenever(mockRepositoryDetector.detectRepositories(any(), eq(""))) + .doReturn(emptyList()) + val unstrippedSoFile = writeFakeSoFile( + "arm64-v8a", + ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS), + "libunstripped.so" + ) + writeFakeSoFile("arm64-v8a", ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS), "libstripped.so") + + // When + testedTask.applyTask() + + // Then + verify(mockUploader).upload( + fakeSite, + Uploader.UploadFileInfo( + fileKey = NdkSymbolFileUploadTask.KEY_NDK_SYMBOL_FILE, + file = unstrippedSoFile, + encoding = NdkSymbolFileUploadTask.ENCODING, + fileType = NdkSymbolFileUploadTask.TYPE_NDK_SYMBOL_FILE, + fileName = "libunstripped.so", + extraAttributes = mapOf( + "arch" to "arm64" + ) + ), + null, + fakeApiKey.value, + DdAppIdentifier( + serviceName = fakeService, + version = fakeVersion, + versionCode = fakeVersionCode, + variant = fakeVariantName, + buildId = fakeBuildId + ), + null, + useGzip = true, + emulateNetworkCall = false + ) + verifyNoMoreInteractions(mockUploader) + } + + @Test + fun `M do nothing W applyTask() { all libraries are stripped }`() { + // Given + writeFakeSoFile("arm64-v8a", ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS)) + writeFakeSoFile("x86", ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS)) + + // When + testedTask.applyTask() + + // Then + verifyNoInteractions(mockUploader) + } + @Test fun `M do nothing W applyTask() {no mapping file}`() { // Given @@ -452,12 +511,16 @@ internal class NdkSymbolFileUploadTaskTest { verifyNoInteractions(mockUploader) } - private fun writeFakeSoFile(arch: String): File { + private fun writeFakeSoFile( + arch: String, + content: ByteArray = "fake".toByteArray(), + fileName: String = "libfake.so" + ): File { val soTempDir = File(tempDir, "fakeSos/$arch") - val fakeSoFile = File(soTempDir, "libfake.so") + val fakeSoFile = File(soTempDir, fileName) fakeSoFile.parentFile.mkdirs() - fakeSoFile.writeText("fake") + fakeSoFile.writeBytes(content) return fakeSoFile } diff --git a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetectorTest.kt b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetectorTest.kt new file mode 100644 index 00000000..0444e9bf --- /dev/null +++ b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetectorTest.kt @@ -0,0 +1,296 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2020-Present Datadog, Inc. + */ + +package com.datadog.gradle.plugin.internal + +import com.datadog.gradle.plugin.utils.ElfFileBuilder +import com.datadog.gradle.plugin.utils.forge.Configurator +import fr.xgouchet.elmyr.Forge +import fr.xgouchet.elmyr.annotation.StringForgery +import fr.xgouchet.elmyr.junit5.ForgeConfiguration +import fr.xgouchet.elmyr.junit5.ForgeExtension +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Timeout +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.api.extension.Extensions +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.TimeUnit + +@Extensions( + ExtendWith(ForgeExtension::class) +) +@ForgeConfiguration(Configurator::class) +internal class ElfSymbolsDetectorTest { + + @TempDir + lateinit var tempDir: File + + @ParameterizedTest + @MethodSource("elfFormats") + fun `M return HAS_SYMBOLS W detect() { library has symbol table }`(is64Bit: Boolean, byteOrder: ByteOrder) { + // Given + val file = writeFile(ElfFileBuilder.build(listOf(".text", ".symtab", ".strtab"), is64Bit, byteOrder)) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.HAS_SYMBOLS) + } + + @ParameterizedTest + @MethodSource("elfFormats") + fun `M return HAS_SYMBOLS W detect() { library has debug info only }`(is64Bit: Boolean, byteOrder: ByteOrder) { + // Given + val file = writeFile(ElfFileBuilder.build(listOf(".text", ".debug_info"), is64Bit, byteOrder)) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.HAS_SYMBOLS) + } + + @ParameterizedTest + @MethodSource("elfFormats") + fun `M return STRIPPED W detect() { library has no symbols }`(is64Bit: Boolean, byteOrder: ByteOrder) { + // Given + val file = writeFile(ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS, is64Bit, byteOrder)) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + fun `M return UNKNOWN W detect() { not an ELF file }`( + @StringForgery fakeContent: String + ) { + // Given + val file = writeFile(fakeContent.toByteArray()) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return UNKNOWN W detect() { empty file }`() { + // Given + val file = writeFile(ByteArray(0)) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return UNKNOWN W detect() { truncated ELF file }`(forge: Forge) { + // Given + val content = ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS) + val file = writeFile(content.copyOf(forge.anInt(min = 4, max = content.size))) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return UNKNOWN W detect() { unexpected section header size }`(forge: Forge) { + // Given + val content = ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + buffer.putShort(ELF64_SHENTSIZE_OFFSET, forge.anElementFrom(40, 63, 65, 128).toShort()) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return UNKNOWN W detect() { section names table is too big }`() { + // Given + // the names table declares a size bigger than the limit, but which still fits into the file. + // No symbol table, so that the names table has to be read to look for debug info. + val content = ElfFileBuilder.build(listOf(".text", ".debug_info")) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + val sectionHeadersOffset = buffer.getLong(ELF64_SHOFF_OFFSET).toInt() + val namesSectionIndex = buffer.getShort(ELF64_SHSTRNDX_OFFSET).toInt() + val namesSizeOffset = sectionHeadersOffset + namesSectionIndex * ELF64_SECTION_HEADER_SIZE + 0x20 + val namesSize = 2L * 1024L * 1024L + buffer.putLong(namesSizeOffset, namesSize) + val file = writeFile(content.copyOf(content.size + namesSize.toInt())) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return HAS_SYMBOLS W detect() { symbol table without section names table }`() { + // Given + val content = ElfFileBuilder.build(listOf(".text", ".symtab", ".strtab")) + ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN).putShort(ELF64_SHSTRNDX_OFFSET, 0) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.HAS_SYMBOLS) + } + + @Test + fun `M return UNKNOWN W detect() { no symbol table and no section names table }`() { + // Given + // debug info can't be identified without the section names + val content = ElfFileBuilder.build(listOf(".text", ".debug_info")) + ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN).putShort(ELF64_SHSTRNDX_OFFSET, 0) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return STRIPPED W detect() { no section header table }`() { + // Given + // as produced by `strip --strip-section-headers` + val content = ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + buffer.putLong(ELF64_SHOFF_OFFSET, 0) + buffer.putShort(ELF64_SHNUM_OFFSET, 0) + buffer.putShort(ELF64_SHSTRNDX_OFFSET, 0) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + fun `M return UNKNOWN W detect() { extended section numbering }`() { + // Given + // the section header table is there, but its size is stored in the first section header + val content = ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS) + ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN).putShort(ELF64_SHNUM_OFFSET, 0) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return STRIPPED W detect() { only MiniDebugInfo }`() { + // Given + // symbolic doesn't read .gnu_debugdata, so the library is as useful as any stripped one + val file = writeFile(ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS + ".gnu_debugdata")) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + fun `M return STRIPPED W detect() { section named symtab with another type }`() { + // Given + // symbolic only reads the symbol table from a section of type SHT_SYMTAB + val content = ElfFileBuilder.build(listOf(".text", ".symtab", ".dynsym")) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + val sectionHeadersOffset = buffer.getLong(ELF64_SHOFF_OFFSET).toInt() + // sections: null, .text, .symtab, ... + buffer.putInt(sectionHeadersOffset + 2 * ELF64_SECTION_HEADER_SIZE + 4, 1) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + @Timeout(10, unit = TimeUnit.SECONDS) + fun `M return STRIPPED quickly W detect() { many sections share a long unterminated name }`() { + // Given + // every section points at the same name, which runs until the end of the names table + val sectionsCount = 60_000 + val content = ElfFileBuilder.build(List(sectionsCount) { ".text" }) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + val sectionHeadersOffset = buffer.getLong(ELF64_SHOFF_OFFSET).toInt() + val namesSectionIndex = buffer.getShort(ELF64_SHSTRNDX_OFFSET).toInt() and 0xFFFF + val namesHeaderOffset = sectionHeadersOffset + namesSectionIndex * ELF64_SECTION_HEADER_SIZE + val namesOffset = buffer.getLong(namesHeaderOffset + 0x18).toInt() + val namesSize = buffer.getLong(namesHeaderOffset + 0x20).toInt() + content.fill('a'.code.toByte(), namesOffset + 1, namesOffset + namesSize) + for (index in 1..namesSectionIndex) { + buffer.putInt(sectionHeadersOffset + index * ELF64_SECTION_HEADER_SIZE, 1) + } + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + fun `M return UNKNOWN W detect() { file doesn't exist }`() { + // When + val result = ElfSymbolsDetector.detect(File(tempDir, "missing.so")) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + private fun writeFile(content: ByteArray): File { + return File(tempDir, "libfake.so").apply { writeBytes(content) } + } + + companion object { + private const val ELF64_SHOFF_OFFSET = 0x28 + private const val ELF64_SHENTSIZE_OFFSET = 0x3A + private const val ELF64_SHNUM_OFFSET = 0x3C + private const val ELF64_SHSTRNDX_OFFSET = 0x3E + private const val ELF64_SECTION_HEADER_SIZE = 64 + + @JvmStatic + fun elfFormats() = listOf(true, false).flatMap { is64Bit -> + listOf(ByteOrder.LITTLE_ENDIAN, ByteOrder.BIG_ENDIAN).map { Arguments.of(is64Bit, it) } + } + } +} diff --git a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/utils/ElfFileBuilder.kt b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/utils/ElfFileBuilder.kt new file mode 100644 index 00000000..be40b839 --- /dev/null +++ b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/utils/ElfFileBuilder.kt @@ -0,0 +1,102 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2020-Present Datadog, Inc. + */ + +package com.datadog.gradle.plugin.utils + +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * Builds a minimal ELF shared object, containing only the given (empty) sections and the section + * names table, which is enough to tell if a library is stripped or not. + */ +internal object ElfFileBuilder { + + val UNSTRIPPED_SECTIONS = listOf(".text", ".dynsym", ".symtab", ".strtab", ".debug_info") + val STRIPPED_SECTIONS = listOf(".text", ".dynsym", ".dynstr") + + fun build( + sectionNames: List, + is64Bit: Boolean = true, + byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN + ): ByteArray { + val headerSize = if (is64Bit) 64 else 52 + val sectionHeaderSize = if (is64Bit) 64 else 40 + + // section names table: first byte is the empty name of the null section + val namesTable = ByteArrayOutputStream().apply { write(0) } + val allSectionNames = sectionNames + ".shstrtab" + val nameOffsets = allSectionNames.map { name -> + namesTable.size().also { namesTable.write((name + "\u0000").toByteArray(Charsets.US_ASCII)) } + } + val names = namesTable.toByteArray() + val namesOffset = headerSize + val sectionHeadersOffset = namesOffset + names.size + // null section + named sections + names table section + val sectionsCount = sectionNames.size + 2 + + val buffer = ByteBuffer.allocate(sectionHeadersOffset + sectionsCount * sectionHeaderSize) + .order(byteOrder) + buffer.put(byteArrayOf(0x7F, 'E'.code.toByte(), 'L'.code.toByte(), 'F'.code.toByte())) + buffer.put(if (is64Bit) 2 else 1) + buffer.put(if (byteOrder == ByteOrder.LITTLE_ENDIAN) 1 else 2) + buffer.put(1) // EI_VERSION + buffer.position(16) + buffer.putShort(3) // e_type = ET_DYN + buffer.putShort(if (is64Bit) 0xB7 else 0x28) // e_machine = AArch64 / ARM + buffer.putInt(1) // e_version + if (is64Bit) { + buffer.putLong(0) // e_entry + buffer.putLong(0) // e_phoff + buffer.putLong(sectionHeadersOffset.toLong()) // e_shoff + } else { + buffer.putInt(0) + buffer.putInt(0) + buffer.putInt(sectionHeadersOffset) + } + buffer.putInt(0) // e_flags + buffer.putShort(headerSize.toShort()) // e_ehsize + buffer.putShort(0) // e_phentsize + buffer.putShort(0) // e_phnum + buffer.putShort(sectionHeaderSize.toShort()) // e_shentsize + buffer.putShort(sectionsCount.toShort()) // e_shnum + buffer.putShort((sectionsCount - 1).toShort()) // e_shstrndx + + buffer.position(namesOffset) + buffer.put(names) + + // null section header is all zeros + nameOffsets.forEachIndexed { index, nameOffset -> + val isNamesTable = index == nameOffsets.lastIndex + buffer.position(sectionHeadersOffset + (index + 1) * sectionHeaderSize) + buffer.putInt(nameOffset) // sh_name + buffer.putInt(sectionType(allSectionNames[index])) // sh_type + val offset = if (isNamesTable) namesOffset else 0 + val size = if (isNamesTable) names.size else 0 + if (is64Bit) { + buffer.putLong(0) // sh_flags + buffer.putLong(0) // sh_addr + buffer.putLong(offset.toLong()) // sh_offset + buffer.putLong(size.toLong()) // sh_size + } else { + buffer.putInt(0) + buffer.putInt(0) + buffer.putInt(offset) + buffer.putInt(size) + } + } + + return buffer.array() + } + + private fun sectionType(name: String): Int = when (name) { + ".symtab" -> 2 // SHT_SYMTAB + ".strtab", ".dynstr", ".shstrtab" -> 3 // SHT_STRTAB + ".dynsym" -> 11 // SHT_DYNSYM + else -> 1 // SHT_PROGBITS + } +} diff --git a/dd-sdk-android-gradle-plugin/src/test/resources/native_gradle_files/build_with_native_and_prebuilt_libs.gradle b/dd-sdk-android-gradle-plugin/src/test/resources/native_gradle_files/build_with_native_and_prebuilt_libs.gradle new file mode 100644 index 00000000..af966851 --- /dev/null +++ b/dd-sdk-android-gradle-plugin/src/test/resources/native_gradle_files/build_with_native_and_prebuilt_libs.gradle @@ -0,0 +1,83 @@ +plugins { + id("com.android.application") + id("kotlin-android") + id("com.datadoghq.dd-sdk-android-gradle-plugin") +} + +repositories { + google() + mavenCentral() +} + +android { + compileSdkVersion = rootProject.ext.targetSdkVersion + buildToolsVersion = rootProject.ext.buildToolsVersion + + defaultConfig { + applicationId "com.example.variants" + minSdkVersion 23 + targetSdkVersion rootProject.ext.targetSdkVersion + versionCode 1 + versionName "1.0" + multiDexEnabled = true + } + + namespace = "com.example.variants" + + compileOptions { + sourceCompatibility = rootProject.ext.jvmTarget + targetCompatibility = rootProject.ext.jvmTarget + } + + externalNativeBuild { + cmake { + path file('src/main/cpp/CMakeLists.txt') + version '3.22.1' + } + } + + packaging { + jniLibs { + excludes += ["**/libexcluded.so"] + } + } + + flavorDimensions("version", "colour") + productFlavors { + demo { + dimension "version" + applicationIdSuffix ".demo" + versionNameSuffix "-demo" + } + full { + dimension "version" + applicationIdSuffix ".full" + versionNameSuffix "-full" + } + + green { + dimension "colour" + } + blue { + dimension "colour" + } + } +} + +if (Integer.parseInt(rootProject.ext.agpVersion.split("\\.")[0]) >= 9) { + kotlin { + compilerOptions { + jvmTarget = org.jetbrains.kotlin.gradle.dsl.JvmTarget.fromTarget(rootProject.ext.jvmTarget) + } + } +} else { + android { + kotlinOptions { + jvmTarget = rootProject.ext.jvmTarget + } + } +} + +dependencies { + implementation(rootProject.ext.datadogSdkDependency) +}