diff --git a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTask.kt b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTask.kt index 438d9d9d..54e8f370 100644 --- a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTask.kt +++ b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTask.kt @@ -7,6 +7,7 @@ package com.datadog.gradle.plugin import com.datadog.gradle.plugin.internal.ApiKey +import com.datadog.gradle.plugin.internal.ElfSymbolsDetector import com.datadog.gradle.plugin.internal.Uploader import com.datadog.gradle.plugin.internal.utils.capitalizeChar import com.datadog.gradle.plugin.internal.variant.AppVariant @@ -40,6 +41,7 @@ internal abstract class NdkSymbolFileUploadTask @Inject constructor( override fun getFilesList(): List { val files = mutableListOf() + val strippedFiles = mutableListOf() searchDirectories .flatMap(this::findSoFiles) @@ -57,6 +59,12 @@ internal abstract class NdkSymbolFileUploadTask @Inject constructor( return@forEach } archToFiles.value.forEach { + if (ElfSymbolsDetector.detect(it) == ElfSymbolsDetector.Result.STRIPPED) { + // prebuilt and dependency libraries are often packaged already stripped + logger.info("Native library ${it.absolutePath} has no symbols, ignoring it.") + strippedFiles.add(it) + return@forEach + } files.add( Uploader.UploadFileInfo( KEY_NDK_SYMBOL_FILE, @@ -72,6 +80,13 @@ internal abstract class NdkSymbolFileUploadTask @Inject constructor( } } + if (files.isEmpty() && strippedFiles.isNotEmpty()) { + logger.warn( + "All the native libraries found have no symbols (they are stripped), nothing to upload: " + + strippedFiles.joinToString(", ") { it.absolutePath } + ) + } + return files } diff --git a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetector.kt b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetector.kt new file mode 100644 index 00000000..88990a35 --- /dev/null +++ b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetector.kt @@ -0,0 +1,202 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2020-Present Datadog, Inc. + */ + +package com.datadog.gradle.plugin.internal + +import com.datadog.gradle.plugin.DdAndroidGradlePlugin.Companion.LOGGER +import java.io.File +import java.io.IOException +import java.io.RandomAccessFile +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * Tells whether a native library still has the symbols needed for symbolication, by reading the + * types and names of the sections listed in its ELF section header table. + */ +@Suppress("MagicNumber") +internal object ElfSymbolsDetector { + + enum class Result { + /** The library has a symbol table or debug info. */ + HAS_SYMBOLS, + + /** The library is a valid ELF file without symbol table and debug info. */ + STRIPPED, + + /** The file is not an ELF file, or it can't be read. */ + UNKNOWN + } + + fun detect(file: File): Result { + return try { + RandomAccessFile(file, "r").use { detect(it) } + } catch (e: IOException) { + onReadError(file, e) + } catch (e: SecurityException) { + onReadError(file, e) + } + } + + // region Private + + private fun onReadError(file: File, e: Exception): Result { + LOGGER.info("Unable to read native library ${file.absolutePath} to check its symbols", e) + return Result.UNKNOWN + } + + // any inconsistency in the file structure ends the detection early + @Suppress("ReturnCount") + private fun detect(file: RandomAccessFile): Result { + val header = file.read(0, ELF64_HEADER_SIZE) ?: return Result.UNKNOWN + if (!ELF_MAGIC.indices.all { header.get(it) == ELF_MAGIC[it] }) return Result.UNKNOWN + val is64Bit = when (header.get(EI_CLASS).toInt()) { + ELFCLASS32 -> false + ELFCLASS64 -> true + else -> return Result.UNKNOWN + } + when (header.get(EI_DATA).toInt()) { + ELFDATA2LSB -> header.order(ByteOrder.LITTLE_ENDIAN) + ELFDATA2MSB -> header.order(ByteOrder.BIG_ENDIAN) + else -> return Result.UNKNOWN + } + + val layout = if (is64Bit) ELF64_LAYOUT else ELF32_LAYOUT + val sectionHeadersOffset = header.readAddress(layout.shoffOffset, is64Bit) + val sectionHeaderSize = header.getShort(layout.shentsizeOffset).toUnsignedInt() + val sectionsCount = header.getShort(layout.shnumOffset).toUnsignedInt() + val namesSectionIndex = header.getShort(layout.shstrndxOffset).toUnsignedInt() + // `strip --strip-section-headers` drops the section header table, which implies stripping all the symbols: + // only the dynamic symbols reachable through the program headers are left, as for any stripped library + if (sectionHeadersOffset == 0L && sectionsCount == 0) return Result.STRIPPED + // extended numbering (more than 0xff00 sections) is not supported, such libraries are not expected. + // The section header size is fixed by the ELF class, checking it bounds the section header table size. + if (sectionsCount == 0 || + sectionHeaderSize != layout.sectionHeaderSize || + namesSectionIndex >= sectionsCount + ) { + return Result.UNKNOWN + } + + val sectionHeaders = file.read(sectionHeadersOffset, sectionsCount.toLong() * sectionHeaderSize) + ?.order(header.order()) + ?: return Result.UNKNOWN + return detect(file, sectionHeaders, sectionsCount, namesSectionIndex, layout, is64Bit) + } + + @Suppress("ReturnCount", "LongParameterList") + private fun detect( + file: RandomAccessFile, + sectionHeaders: ByteBuffer, + sectionsCount: Int, + namesSectionIndex: Int, + layout: Layout, + is64Bit: Boolean + ): Result { + // the symbol table is found by its type, as symbolic does, so it doesn't depend on the section names + val hasSymbolTable = (0 until sectionsCount).any { index -> + sectionHeaders.getInt(index * layout.sectionHeaderSize + SH_TYPE_OFFSET) == SHT_SYMTAB + } + if (hasSymbolTable) return Result.HAS_SYMBOLS + // debug info sections are only found by their names, which can't be checked without the names table + if (namesSectionIndex == SHN_UNDEF) return Result.UNKNOWN + + val namesHeaderOffset = namesSectionIndex * layout.sectionHeaderSize + val namesSize = sectionHeaders.readAddress(namesHeaderOffset + layout.shSizeOffset, is64Bit) + if (namesSize > MAX_NAMES_SECTION_SIZE) return Result.UNKNOWN + val names = file.read( + sectionHeaders.readAddress(namesHeaderOffset + layout.shOffsetOffset, is64Bit), + namesSize + ) ?: return Result.UNKNOWN + + val hasDebugInfo = (0 until sectionsCount).any { index -> + val nameOffset = sectionHeaders.getInt(index * layout.sectionHeaderSize).toUnsignedLong() + DEBUG_INFO_SECTION_NAMES.any { names.hasNameAt(nameOffset, it) } + } + return if (hasDebugInfo) Result.HAS_SYMBOLS else Result.STRIPPED + } + + private fun RandomAccessFile.read(offset: Long, size: Long): ByteBuffer? { + val isSizeValid = size in 0..Int.MAX_VALUE + if (!isSizeValid || offset !in 0..length() - size) return null + val bytes = ByteArray(size.toInt()) + seek(offset) + readFully(bytes) + return ByteBuffer.wrap(bytes) + } + + private fun ByteBuffer.readAddress(offset: Int, is64Bit: Boolean): Long { + // a negative value (> Long.MAX_VALUE as unsigned) is rejected later as out of the file bounds + return if (is64Bit) getLong(offset) else getInt(offset).toUnsignedLong() + } + + // compares at most name.size + 1 bytes, so that many sections pointing at a long unterminated + // name don't make the check scan the whole table for each of them + private fun ByteBuffer.hasNameAt(offset: Long, name: ByteArray): Boolean { + if (offset < 0 || offset + name.size >= limit()) return false + val start = offset.toInt() + return name.indices.all { get(start + it) == name[it] } && get(start + name.size) == 0.toByte() + } + + private fun Short.toUnsignedInt(): Int = toInt() and 0xFFFF + + private fun Int.toUnsignedLong(): Long = toLong() and 0xFFFFFFFFL + + private class Layout( + val shoffOffset: Int, + val shentsizeOffset: Int, + val shnumOffset: Int, + val shstrndxOffset: Int, + val sectionHeaderSize: Int, + val shOffsetOffset: Int, + val shSizeOffset: Int + ) + + // endregion + + private val ELF_MAGIC = byteArrayOf(0x7F, 'E'.code.toByte(), 'L'.code.toByte(), 'F'.code.toByte()) + private const val EI_CLASS = 4 + private const val EI_DATA = 5 + private const val ELFCLASS32 = 1 + private const val ELFCLASS64 = 2 + private const val ELFDATA2LSB = 1 + private const val ELFDATA2MSB = 2 + + // the 64-bit header is the bigger one, the 32-bit one fits into it + private const val ELF64_HEADER_SIZE = 64L + + // section names table is usually smaller than 1 KB, anything bigger is not a sane library + private const val MAX_NAMES_SECTION_SIZE = 1024L * 1024L + + private val ELF32_LAYOUT = Layout( + shoffOffset = 0x20, + shentsizeOffset = 0x2E, + shnumOffset = 0x30, + shstrndxOffset = 0x32, + sectionHeaderSize = 0x28, + shOffsetOffset = 0x10, + shSizeOffset = 0x14 + ) + + private val ELF64_LAYOUT = Layout( + shoffOffset = 0x28, + shentsizeOffset = 0x3A, + shnumOffset = 0x3C, + shstrndxOffset = 0x3E, + sectionHeaderSize = 0x40, + shOffsetOffset = 0x18, + shSizeOffset = 0x20 + ) + + private const val SH_TYPE_OFFSET = 4 + private const val SHT_SYMTAB = 2 + private const val SHN_UNDEF = 0 + + // debug info sections removed by stripping, which are needed to symbolicate a native crash. `.gnu_debugdata` + // (MiniDebugInfo) is not listed, as symbolic doesn't read it. + private val DEBUG_INFO_SECTION_NAMES = listOf(".debug_info", ".zdebug_info") + .map { it.toByteArray(Charsets.US_ASCII) } +} diff --git a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/variant/NewApiAppVariant.kt b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/variant/NewApiAppVariant.kt index cf110857..70548756 100644 --- a/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/variant/NewApiAppVariant.kt +++ b/dd-sdk-android-gradle-plugin/src/main/kotlin/com/datadog/gradle/plugin/internal/variant/NewApiAppVariant.kt @@ -10,13 +10,11 @@ import com.android.build.api.artifact.SingleArtifact import com.android.build.api.dsl.ApplicationExtension import com.android.build.api.variant.ApplicationVariant import com.android.build.api.variant.VariantOutput -import com.android.build.gradle.tasks.ExternalNativeBuildTask import com.datadog.gradle.plugin.GenerateBuildIdTask import com.datadog.gradle.plugin.InjectBuildIdToAssetsTask import com.datadog.gradle.plugin.MappingFileUploadTask import com.datadog.gradle.plugin.NdkSymbolFileUploadTask import com.datadog.gradle.plugin.internal.CurrentAgpVersion -import com.datadog.gradle.plugin.internal.getSearchObjDirs import com.datadog.gradle.plugin.internal.utils.capitalizeChar import org.gradle.api.Project import org.gradle.api.artifacts.Configuration @@ -75,18 +73,10 @@ internal class NewApiAppVariant( } override fun bindWith(ndkUploadTask: NdkSymbolFileUploadTask) { - // New Variant API doesn't expose native build tasks. AGP builds each distinct native configuration - // once (`buildCMake[]`, attributed to a single representative variant), and the - // `externalNativeBuild` task of each variant depends on the ones it uses, so we follow it. - // Its own output folder is not used: it is either empty or holds copies of the same libraries. - target.tasks.withType(ExternalNativeBuildTask::class.java) - .filter { it.variantName == variant.name } - .forEach { variantNativeBuildTask -> - ndkUploadTask.dependsOn(variantNativeBuildTask) - variantNativeBuildTask.taskDependencies.getDependencies(variantNativeBuildTask) - .filterIsInstance() - .forEach { ndkUploadTask.searchDirectories.from(it.getSearchObjDirs()) } - } + // All the native libraries packaged into this variant (built by the app, prebuilt and coming from + // dependencies), not stripped yet. Libraries which are already stripped are skipped by the task. + // The artifact also brings the dependency on the tasks producing it. + ndkUploadTask.searchDirectories.from(variant.artifacts.get(SingleArtifact.MERGED_NATIVE_LIBS)) } override fun bindWith(mappingFileUploadTask: MappingFileUploadTask) { diff --git a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/DdAndroidGradlePluginFunctionalTest.kt b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/DdAndroidGradlePluginFunctionalTest.kt index 03f5f142..2d336c85 100644 --- a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/DdAndroidGradlePluginFunctionalTest.kt +++ b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/DdAndroidGradlePluginFunctionalTest.kt @@ -6,7 +6,9 @@ package com.datadog.gradle.plugin +import com.datadog.gradle.plugin.internal.ElfSymbolsDetector import com.datadog.gradle.plugin.internal.utils.capitalizeChar +import com.datadog.gradle.plugin.utils.ElfFileBuilder import com.datadog.gradle.plugin.utils.assertj.BuildResultAssert.Companion.assertThat import com.datadog.gradle.plugin.utils.forge.Configurator import com.datadog.gradle.plugin.utils.headHash @@ -1496,13 +1498,70 @@ internal class DdAndroidGradlePluginFunctionalTest { val uploadedFilesPerTask = uploadedNdkSymbolFilesPerTask(result.output) assertThat(uploadedFilesPerTask.keys).containsExactlyInAnyOrderElementsOf(uploadTaskNames) uploadedFilesPerTask.forEach { (taskName, uploadedFiles) -> + val variantName = taskName.removePrefix("uploadNdkSymbolFiles").replaceFirstChar { it.lowercase() } assertThat(uploadedFiles) .describedAs("files uploaded by $taskName") .hasSize(SUPPORTED_ABIS.size) - .allMatch { it.contains("/RelWithDebInfo/") && it.endsWith("/libplaceholder.so") } + .allMatch { it.contains("/merged_native_libs/$variantName/") && it.endsWith("/libplaceholder.so") } } } + @Test + fun `M upload all the packaged native libraries with symbols W upload symbol file`(forge: Forge) { + // Given + // new Variant API is only used starting from AGP 8.4 + buildVersionConfig = LATEST_VERSIONS_TEST_CONFIGURATION + stubGradlePropertiesFile(buildVersionConfig) + stubGradleBuildFromResourceFile( + "lib_module_build.gradle", + libModuleBuildGradleFile, + overwrite = true + ) + stubGradleBuildFromResourceFile( + "native_gradle_files/build_with_native_and_prebuilt_libs.gradle", + appBuildGradleFile + ) + stubNativeFiles() + val prebuiltLibsDir = File(appMainSrcDir, "jniLibs/arm64-v8a").apply { mkdirs() } + File(prebuiltLibsDir, "libprebuilt.so").writeBytes(ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS)) + File(prebuiltLibsDir, "libstripped.so").writeBytes(ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS)) + File(prebuiltLibsDir, "libexcluded.so").writeBytes(ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS)) + + val variant = "${forge.anElementFrom(versions).lowercase()}${forge.anElementFrom(colors)}" + val taskName = resolveNdkSymbolUploadTask(variant) + + gradleRunner { withArguments(":samples:app:assembleRelease") } + .build() + + // When + val result = gradleRunner { + withArguments( + taskName, + "--info", + "-PDD_API_KEY=fakekey", + "-Pdd-emulate-upload-call" + ) + } + .build() + + // Then + val uploadedFiles = uploadedNdkSymbolFilesPerTask(result.output).getValue(taskName) + assertThat(uploadedFiles.map { File(it).name }) + .containsExactlyInAnyOrderElementsOf(List(SUPPORTED_ABIS.size) { "libplaceholder.so" } + "libprebuilt.so") + assertThat(result).containsInOutput("libstripped.so has no symbols, ignoring it.") + + // the libraries built by the app are detected with symbols before AGP strips them, and without after + val intermediatesDir = File(appRootDir, "build/intermediates") + val builtLibs = intermediatesDir.walkTopDown().filter { it.name == "libplaceholder.so" }.toList() + assertThat(builtLibs.filter { it.path.contains("/merged_native_libs/") }) + .isNotEmpty + .allMatch { ElfSymbolsDetector.detect(it) == ElfSymbolsDetector.Result.HAS_SYMBOLS } + assertThat(builtLibs.filter { it.path.contains("/stripped_native_libs/") }) + .isNotEmpty + .allMatch { ElfSymbolsDetector.detect(it) == ElfSymbolsDetector.Result.STRIPPED } + } + + @Test fun `M try to upload the symbol file W upload { using a fake API_KEY, gzip disabled }`(forge: Forge) { // Given stubGradleBuildFromResourceFile( @@ -1526,6 +1585,7 @@ internal class DdAndroidGradlePluginFunctionalTest { "--info", "--stacktrace", "-PDD_API_KEY=fakekey", + "-Pdd-disable-gzip", "-Pdd-emulate-upload-call" ) } diff --git a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTaskTest.kt b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTaskTest.kt index a43a41bf..705e6133 100644 --- a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTaskTest.kt +++ b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/NdkSymbolFileUploadTaskTest.kt @@ -11,6 +11,7 @@ import com.datadog.gradle.plugin.internal.ApiKeySource import com.datadog.gradle.plugin.internal.DdAppIdentifier import com.datadog.gradle.plugin.internal.Uploader import com.datadog.gradle.plugin.internal.variant.AppVariant +import com.datadog.gradle.plugin.utils.ElfFileBuilder import com.datadog.gradle.plugin.utils.forge.Configurator import fr.xgouchet.elmyr.Forge import fr.xgouchet.elmyr.annotation.Forgery @@ -36,6 +37,7 @@ import org.mockito.kotlin.doReturn import org.mockito.kotlin.eq import org.mockito.kotlin.verify import org.mockito.kotlin.verifyNoInteractions +import org.mockito.kotlin.verifyNoMoreInteractions import org.mockito.kotlin.whenever import org.mockito.quality.Strictness import java.io.File @@ -441,6 +443,63 @@ internal class NdkSymbolFileUploadTaskTest { verifyNoInteractions(mockUploader) } + @Test + fun `M upload only libraries with symbols W applyTask() { some libraries are stripped }`() { + // Given + whenever(mockRepositoryDetector.detectRepositories(any(), eq(""))) + .doReturn(emptyList()) + val unstrippedSoFile = writeFakeSoFile( + "arm64-v8a", + ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS), + "libunstripped.so" + ) + writeFakeSoFile("arm64-v8a", ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS), "libstripped.so") + + // When + testedTask.applyTask() + + // Then + verify(mockUploader).upload( + fakeSite, + Uploader.UploadFileInfo( + fileKey = NdkSymbolFileUploadTask.KEY_NDK_SYMBOL_FILE, + file = unstrippedSoFile, + encoding = NdkSymbolFileUploadTask.ENCODING, + fileType = NdkSymbolFileUploadTask.TYPE_NDK_SYMBOL_FILE, + fileName = "libunstripped.so", + extraAttributes = mapOf( + "arch" to "arm64" + ) + ), + null, + fakeApiKey.value, + DdAppIdentifier( + serviceName = fakeService, + version = fakeVersion, + versionCode = fakeVersionCode, + variant = fakeVariantName, + buildId = fakeBuildId + ), + null, + useGzip = true, + emulateNetworkCall = false + ) + verifyNoMoreInteractions(mockUploader) + } + + @Test + fun `M do nothing W applyTask() { all libraries are stripped }`() { + // Given + writeFakeSoFile("arm64-v8a", ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS)) + writeFakeSoFile("x86", ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS)) + + // When + testedTask.applyTask() + + // Then + verifyNoInteractions(mockUploader) + } + @Test fun `M do nothing W applyTask() {no mapping file}`() { // Given @@ -452,12 +511,16 @@ internal class NdkSymbolFileUploadTaskTest { verifyNoInteractions(mockUploader) } - private fun writeFakeSoFile(arch: String): File { + private fun writeFakeSoFile( + arch: String, + content: ByteArray = "fake".toByteArray(), + fileName: String = "libfake.so" + ): File { val soTempDir = File(tempDir, "fakeSos/$arch") - val fakeSoFile = File(soTempDir, "libfake.so") + val fakeSoFile = File(soTempDir, fileName) fakeSoFile.parentFile.mkdirs() - fakeSoFile.writeText("fake") + fakeSoFile.writeBytes(content) return fakeSoFile } diff --git a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetectorTest.kt b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetectorTest.kt new file mode 100644 index 00000000..0444e9bf --- /dev/null +++ b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/internal/ElfSymbolsDetectorTest.kt @@ -0,0 +1,296 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2020-Present Datadog, Inc. + */ + +package com.datadog.gradle.plugin.internal + +import com.datadog.gradle.plugin.utils.ElfFileBuilder +import com.datadog.gradle.plugin.utils.forge.Configurator +import fr.xgouchet.elmyr.Forge +import fr.xgouchet.elmyr.annotation.StringForgery +import fr.xgouchet.elmyr.junit5.ForgeConfiguration +import fr.xgouchet.elmyr.junit5.ForgeExtension +import org.assertj.core.api.Assertions.assertThat +import org.junit.jupiter.api.Test +import org.junit.jupiter.api.Timeout +import org.junit.jupiter.api.extension.ExtendWith +import org.junit.jupiter.api.extension.Extensions +import org.junit.jupiter.api.io.TempDir +import org.junit.jupiter.params.ParameterizedTest +import org.junit.jupiter.params.provider.Arguments +import org.junit.jupiter.params.provider.MethodSource +import java.io.File +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.TimeUnit + +@Extensions( + ExtendWith(ForgeExtension::class) +) +@ForgeConfiguration(Configurator::class) +internal class ElfSymbolsDetectorTest { + + @TempDir + lateinit var tempDir: File + + @ParameterizedTest + @MethodSource("elfFormats") + fun `M return HAS_SYMBOLS W detect() { library has symbol table }`(is64Bit: Boolean, byteOrder: ByteOrder) { + // Given + val file = writeFile(ElfFileBuilder.build(listOf(".text", ".symtab", ".strtab"), is64Bit, byteOrder)) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.HAS_SYMBOLS) + } + + @ParameterizedTest + @MethodSource("elfFormats") + fun `M return HAS_SYMBOLS W detect() { library has debug info only }`(is64Bit: Boolean, byteOrder: ByteOrder) { + // Given + val file = writeFile(ElfFileBuilder.build(listOf(".text", ".debug_info"), is64Bit, byteOrder)) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.HAS_SYMBOLS) + } + + @ParameterizedTest + @MethodSource("elfFormats") + fun `M return STRIPPED W detect() { library has no symbols }`(is64Bit: Boolean, byteOrder: ByteOrder) { + // Given + val file = writeFile(ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS, is64Bit, byteOrder)) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + fun `M return UNKNOWN W detect() { not an ELF file }`( + @StringForgery fakeContent: String + ) { + // Given + val file = writeFile(fakeContent.toByteArray()) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return UNKNOWN W detect() { empty file }`() { + // Given + val file = writeFile(ByteArray(0)) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return UNKNOWN W detect() { truncated ELF file }`(forge: Forge) { + // Given + val content = ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS) + val file = writeFile(content.copyOf(forge.anInt(min = 4, max = content.size))) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return UNKNOWN W detect() { unexpected section header size }`(forge: Forge) { + // Given + val content = ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + buffer.putShort(ELF64_SHENTSIZE_OFFSET, forge.anElementFrom(40, 63, 65, 128).toShort()) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return UNKNOWN W detect() { section names table is too big }`() { + // Given + // the names table declares a size bigger than the limit, but which still fits into the file. + // No symbol table, so that the names table has to be read to look for debug info. + val content = ElfFileBuilder.build(listOf(".text", ".debug_info")) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + val sectionHeadersOffset = buffer.getLong(ELF64_SHOFF_OFFSET).toInt() + val namesSectionIndex = buffer.getShort(ELF64_SHSTRNDX_OFFSET).toInt() + val namesSizeOffset = sectionHeadersOffset + namesSectionIndex * ELF64_SECTION_HEADER_SIZE + 0x20 + val namesSize = 2L * 1024L * 1024L + buffer.putLong(namesSizeOffset, namesSize) + val file = writeFile(content.copyOf(content.size + namesSize.toInt())) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return HAS_SYMBOLS W detect() { symbol table without section names table }`() { + // Given + val content = ElfFileBuilder.build(listOf(".text", ".symtab", ".strtab")) + ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN).putShort(ELF64_SHSTRNDX_OFFSET, 0) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.HAS_SYMBOLS) + } + + @Test + fun `M return UNKNOWN W detect() { no symbol table and no section names table }`() { + // Given + // debug info can't be identified without the section names + val content = ElfFileBuilder.build(listOf(".text", ".debug_info")) + ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN).putShort(ELF64_SHSTRNDX_OFFSET, 0) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return STRIPPED W detect() { no section header table }`() { + // Given + // as produced by `strip --strip-section-headers` + val content = ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + buffer.putLong(ELF64_SHOFF_OFFSET, 0) + buffer.putShort(ELF64_SHNUM_OFFSET, 0) + buffer.putShort(ELF64_SHSTRNDX_OFFSET, 0) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + fun `M return UNKNOWN W detect() { extended section numbering }`() { + // Given + // the section header table is there, but its size is stored in the first section header + val content = ElfFileBuilder.build(ElfFileBuilder.UNSTRIPPED_SECTIONS) + ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN).putShort(ELF64_SHNUM_OFFSET, 0) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + @Test + fun `M return STRIPPED W detect() { only MiniDebugInfo }`() { + // Given + // symbolic doesn't read .gnu_debugdata, so the library is as useful as any stripped one + val file = writeFile(ElfFileBuilder.build(ElfFileBuilder.STRIPPED_SECTIONS + ".gnu_debugdata")) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + fun `M return STRIPPED W detect() { section named symtab with another type }`() { + // Given + // symbolic only reads the symbol table from a section of type SHT_SYMTAB + val content = ElfFileBuilder.build(listOf(".text", ".symtab", ".dynsym")) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + val sectionHeadersOffset = buffer.getLong(ELF64_SHOFF_OFFSET).toInt() + // sections: null, .text, .symtab, ... + buffer.putInt(sectionHeadersOffset + 2 * ELF64_SECTION_HEADER_SIZE + 4, 1) + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + @Timeout(10, unit = TimeUnit.SECONDS) + fun `M return STRIPPED quickly W detect() { many sections share a long unterminated name }`() { + // Given + // every section points at the same name, which runs until the end of the names table + val sectionsCount = 60_000 + val content = ElfFileBuilder.build(List(sectionsCount) { ".text" }) + val buffer = ByteBuffer.wrap(content).order(ByteOrder.LITTLE_ENDIAN) + val sectionHeadersOffset = buffer.getLong(ELF64_SHOFF_OFFSET).toInt() + val namesSectionIndex = buffer.getShort(ELF64_SHSTRNDX_OFFSET).toInt() and 0xFFFF + val namesHeaderOffset = sectionHeadersOffset + namesSectionIndex * ELF64_SECTION_HEADER_SIZE + val namesOffset = buffer.getLong(namesHeaderOffset + 0x18).toInt() + val namesSize = buffer.getLong(namesHeaderOffset + 0x20).toInt() + content.fill('a'.code.toByte(), namesOffset + 1, namesOffset + namesSize) + for (index in 1..namesSectionIndex) { + buffer.putInt(sectionHeadersOffset + index * ELF64_SECTION_HEADER_SIZE, 1) + } + val file = writeFile(content) + + // When + val result = ElfSymbolsDetector.detect(file) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.STRIPPED) + } + + @Test + fun `M return UNKNOWN W detect() { file doesn't exist }`() { + // When + val result = ElfSymbolsDetector.detect(File(tempDir, "missing.so")) + + // Then + assertThat(result).isEqualTo(ElfSymbolsDetector.Result.UNKNOWN) + } + + private fun writeFile(content: ByteArray): File { + return File(tempDir, "libfake.so").apply { writeBytes(content) } + } + + companion object { + private const val ELF64_SHOFF_OFFSET = 0x28 + private const val ELF64_SHENTSIZE_OFFSET = 0x3A + private const val ELF64_SHNUM_OFFSET = 0x3C + private const val ELF64_SHSTRNDX_OFFSET = 0x3E + private const val ELF64_SECTION_HEADER_SIZE = 64 + + @JvmStatic + fun elfFormats() = listOf(true, false).flatMap { is64Bit -> + listOf(ByteOrder.LITTLE_ENDIAN, ByteOrder.BIG_ENDIAN).map { Arguments.of(is64Bit, it) } + } + } +} diff --git a/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/utils/ElfFileBuilder.kt b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/utils/ElfFileBuilder.kt new file mode 100644 index 00000000..be40b839 --- /dev/null +++ b/dd-sdk-android-gradle-plugin/src/test/kotlin/com/datadog/gradle/plugin/utils/ElfFileBuilder.kt @@ -0,0 +1,102 @@ +/* + * Unless explicitly stated otherwise all files in this repository are licensed under the Apache License Version 2.0. + * This product includes software developed at Datadog (https://www.datadoghq.com/). + * Copyright 2020-Present Datadog, Inc. + */ + +package com.datadog.gradle.plugin.utils + +import java.io.ByteArrayOutputStream +import java.nio.ByteBuffer +import java.nio.ByteOrder + +/** + * Builds a minimal ELF shared object, containing only the given (empty) sections and the section + * names table, which is enough to tell if a library is stripped or not. + */ +internal object ElfFileBuilder { + + val UNSTRIPPED_SECTIONS = listOf(".text", ".dynsym", ".symtab", ".strtab", ".debug_info") + val STRIPPED_SECTIONS = listOf(".text", ".dynsym", ".dynstr") + + fun build( + sectionNames: List, + is64Bit: Boolean = true, + byteOrder: ByteOrder = ByteOrder.LITTLE_ENDIAN + ): ByteArray { + val headerSize = if (is64Bit) 64 else 52 + val sectionHeaderSize = if (is64Bit) 64 else 40 + + // section names table: first byte is the empty name of the null section + val namesTable = ByteArrayOutputStream().apply { write(0) } + val allSectionNames = sectionNames + ".shstrtab" + val nameOffsets = allSectionNames.map { name -> + namesTable.size().also { namesTable.write((name + "\u0000").toByteArray(Charsets.US_ASCII)) } + } + val names = namesTable.toByteArray() + val namesOffset = headerSize + val sectionHeadersOffset = namesOffset + names.size + // null section + named sections + names table section + val sectionsCount = sectionNames.size + 2 + + val buffer = ByteBuffer.allocate(sectionHeadersOffset + sectionsCount * sectionHeaderSize) + .order(byteOrder) + buffer.put(byteArrayOf(0x7F, 'E'.code.toByte(), 'L'.code.toByte(), 'F'.code.toByte())) + buffer.put(if (is64Bit) 2 else 1) + buffer.put(if (byteOrder == ByteOrder.LITTLE_ENDIAN) 1 else 2) + buffer.put(1) // EI_VERSION + buffer.position(16) + buffer.putShort(3) // e_type = ET_DYN + buffer.putShort(if (is64Bit) 0xB7 else 0x28) // e_machine = AArch64 / ARM + buffer.putInt(1) // e_version + if (is64Bit) { + buffer.putLong(0) // e_entry + buffer.putLong(0) // e_phoff + buffer.putLong(sectionHeadersOffset.toLong()) // e_shoff + } else { + buffer.putInt(0) + buffer.putInt(0) + buffer.putInt(sectionHeadersOffset) + } + buffer.putInt(0) // e_flags + buffer.putShort(headerSize.toShort()) // e_ehsize + buffer.putShort(0) // e_phentsize + buffer.putShort(0) // e_phnum + buffer.putShort(sectionHeaderSize.toShort()) // e_shentsize + buffer.putShort(sectionsCount.toShort()) // e_shnum + buffer.putShort((sectionsCount - 1).toShort()) // e_shstrndx + + buffer.position(namesOffset) + buffer.put(names) + + // null section header is all zeros + nameOffsets.forEachIndexed { index, nameOffset -> + val isNamesTable = index == nameOffsets.lastIndex + buffer.position(sectionHeadersOffset + (index + 1) * sectionHeaderSize) + buffer.putInt(nameOffset) // sh_name + buffer.putInt(sectionType(allSectionNames[index])) // sh_type + val offset = if (isNamesTable) namesOffset else 0 + val size = if (isNamesTable) names.size else 0 + if (is64Bit) { + buffer.putLong(0) // sh_flags + buffer.putLong(0) // sh_addr + buffer.putLong(offset.toLong()) // sh_offset + buffer.putLong(size.toLong()) // sh_size + } else { + buffer.putInt(0) + buffer.putInt(0) + buffer.putInt(offset) + buffer.putInt(size) + } + } + + return buffer.array() + } + + private fun sectionType(name: String): Int = when (name) { + ".symtab" -> 2 // SHT_SYMTAB + ".strtab", ".dynstr", ".shstrtab" -> 3 // SHT_STRTAB + ".dynsym" -> 11 // SHT_DYNSYM + else -> 1 // SHT_PROGBITS + } +} diff --git a/dd-sdk-android-gradle-plugin/src/test/resources/native_gradle_files/build_with_native_and_prebuilt_libs.gradle b/dd-sdk-android-gradle-plugin/src/test/resources/native_gradle_files/build_with_native_and_prebuilt_libs.gradle new file mode 100644 index 00000000..af966851 --- /dev/null +++ b/dd-sdk-android-gradle-plugin/src/test/resources/native_gradle_files/build_with_native_and_prebuilt_libs.gradle @@ -0,0 +1,83 @@ +plugins { + id("com.android.application") + id("kotlin-android") + id("com.datadoghq.dd-sdk-android-gradle-plugin") +} + +repositories { + google() + mavenCentral() +} + +android { + compileSdkVersion = rootProject.ext.targetSdkVersion + buildToolsVersion = rootProject.ext.buildToolsVersion + + defaultConfig { + applicationId "com.example.variants" + minSdkVersion 23 + targetSdkVersion rootProject.ext.targetSdkVersion + versionCode 1 + versionName "1.0" + multiDexEnabled = true + } + + namespace = "com.example.variants" + + compileOptions { + sourceCompatibility = rootProject.ext.jvmTarget + targetCompatibility = rootProject.ext.jvmTarget + } + + externalNativeBuild { + cmake { + path file('src/main/cpp/CMakeLists.txt') + version '3.22.1' + } + } + + packaging { + jniLibs { + excludes += ["**/libexcluded.so"] + } + } + + flavorDimensions("version", "colour") + productFlavors { + demo { + dimension "version" + applicationIdSuffix ".demo" + versionNameSuffix "-demo" + } + full { + dimension "version" + applicationIdSuffix ".full" + versionNameSuffix "-full" + } + + green { + dimension "colour" + } + blue { + dimension "colour" + } + } +} + +if (Integer.parseInt(rootProject.ext.agpVersion.split("\\.")[0]) >= 9) { + kotlin { + compilerOptions { + jvmTarget = org.jetbrains.kotlin.gradle.dsl.JvmTarget.fromTarget(rootProject.ext.jvmTarget) + } + } +} else { + android { + kotlinOptions { + jvmTarget = rootProject.ext.jvmTarget + } + } +} + +dependencies { + implementation(rootProject.ext.datadogSdkDependency) +}