Use the C++ singlejar in ZipFilterAction instead of legacy Java impl

ZipFilterAction currently depends on ZipCombiner and several other utility classes from the now-deleted Bazel Java singlejar tool (it was rewritten in C++ a while ago). ZipFilterAction doesn't actually need the advanced filtering and callback capabilities of ZipCombiner, since it only cares about a single zip archive at a time.

This change refactors ZipFilterAction to use the C++ singlejar under the hood. Internal testing suggests a 25% wall time reduction in zip filtering is achievable.

PiperOrigin-RevId: 775670271
Change-Id: I5534b5b1c98f06ad3905a06d86ca339286c3a7c2

Author: ted-xie

.bazelrc

@@ -1,4 +1,5 @@
 common --java_runtime_version=17
+common --tool_java_language_version=17
 
 # Workaround for a rules_java + bazel < 8.3.0 issue. It should only be relevant
 # for bazel@HEAD and rolling releases.

MODULE.bazel

@@ -6,7 +6,7 @@ module(
 
 bazel_dep(name = "platforms", version = "0.0.5")
 bazel_dep(name = "rules_license", version = "1.0.0")
-bazel_dep(name = "rules_java", version = "8.12.0")
+bazel_dep(name = "rules_java", version = "8.13.0")
 bazel_dep(name = "rules_cc", version = "0.1.1")
 bazel_dep(name = "rules_shell", version = "0.1.2")
 

examples/basicapp/.bazelrc

@@ -3,6 +3,8 @@ common:core_library_desugaring --desugar_java8_libs
 
 # Flags to enable mobile-install v3
 mobile-install --mode=skylark --mobile_install_aspect=@rules_android//mobile_install:mi.bzl --mobile_install_supported_rules=android_binary
+# Required to build Android builder tools
+common --tool_java_language_version=17
 # Required to invoke the Studio deployer jar
 common --tool_java_runtime_version=17
 

examples/basicapp/java/com/basicapp/BUILD

@@ -9,7 +9,6 @@ android_binary(
 android_library(
     name = "basic_lib",
     srcs = ["BasicActivity.java"],
-    javacopts = ["-source 11 -target 11"],
     manifest = "AndroidManifest.xml",
     resource_files = glob(["res/**"]),
 )

prereqs.bzl

@@ -25,9 +25,9 @@ def rules_android_prereqs(dev_mode = False):
         http_archive,
         name = "rules_java",
         urls = [
-            "https://github.com/bazelbuild/rules_java/releases/download/8.12.0/rules_java-8.12.0.tar.gz",
+            "https://github.com/bazelbuild/rules_java/releases/download/8.13.0/rules_java-8.13.0.tar.gz",
         ],
-        sha256 = "1558508fc6c348d7f99477bd21681e5746936f15f0436b5f4233e30832a590f9",
+        sha256 = "b6c6d92ca9dbb77de31fb6c6a794d20427072663ce41c2b047902ffcc123e3ef",
     )
 
     maybe(

src/tools/java/com/google/devtools/build/android/BUILD

@@ -19,30 +19,22 @@ java_binary(
 
 java_binary(
     name = "ZipFilterAction",
+    data = [
+        "//tools/jdk:singlejar",
+    ],
     # Memory consumption of SingleJar is about 250 bytes per entry in the output file. Unfortunately,
     # the JVM tends to kill the process with an OOM long before we're at the limit. In the most
     # recent example, 400 MB of memory was enough for about 500,000 entries.
-    jvm_flags = ["-Xmx1600m"],
+    jvm_flags = [
+        "-Xmx1600m",
+        "-Drunfiles.path=$$0.runfiles",
+        "-Dsinglejar.path=$(rlocationpath //tools/jdk:singlejar)",
+    ],
     main_class = "com.google.devtools.build.android.ZipFilterAction",
     visibility = ["//tools/android:__pkg__"],
     runtime_deps = [":android_builder_lib"],
 )
 
-run_singlejar(
-    name = "singlejar_jar_gen",
-    srcs = ["@android_tools//:all_android_tools_deploy.jar"],
-    out = "singlejar.jar",
-    include_prefixes = [
-        "com/google/devtools/build/singlejar/",
-        "com/google/devtools/build/zip/",
-    ],
-)
-
-java_import(
-    name = "singlejar_jar",
-    jars = [":singlejar_jar_gen"],
-)
-
 run_singlejar(
     name = "databinding_exec_jar_gen",
     srcs = ["@android_tools//:all_android_tools_deploy.jar"],
@@ -108,7 +100,6 @@ java_library(
         ":android_databinding_wrapper_lib",
         ":android_options_utils",
         ":dependency_info",
-        ":singlejar_jar",
         "//src/tools/java/com/google/devtools/build/android/junctions",
         "//src/tools/java/com/google/devtools/build/android/proto:resources_java_proto",
         "//src/tools/java/com/google/devtools/build/android/proto:serialize_format_java_pb",
@@ -154,8 +145,6 @@ java_library(
     ],
 )
 
-
-
 java_library(
     name = "android_common_30_1_3",
     # Do not sort: Deps order matters until android_tools.jar deps are deleted (b/393172052).

src/tools/java/com/google/devtools/build/android/ZipFilterAction.java

@@ -13,6 +13,8 @@
 // limitations under the License.
 package com.google.devtools.build.android;
 
+import static java.nio.charset.StandardCharsets.UTF_8;
+
 import com.beust.jcommander.IStringConverter;
 import com.beust.jcommander.IValueValidator;
 import com.beust.jcommander.JCommander;
@@ -23,16 +25,18 @@
 import com.google.common.base.Joiner;
 import com.google.common.base.Stopwatch;
 import com.google.common.collect.ImmutableList;
-import com.google.common.collect.ImmutableMap;
 import com.google.common.collect.ImmutableSetMultimap;
 import com.google.common.collect.Multimap;
-import com.google.devtools.build.singlejar.ZipCombiner;
-import com.google.devtools.build.singlejar.ZipCombiner.OutputMode;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
-import java.io.OutputStream;
+import java.io.InputStreamReader;
 import java.nio.file.FileSystems;
 import java.nio.file.Files;
 import java.nio.file.Path;
+import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Enumeration;
 import java.util.LinkedHashSet;
@@ -43,6 +47,7 @@
 import java.util.regex.Pattern;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipFile;
+import java.util.zip.ZipOutputStream;
 
 /**
  * Action to filter entries out of a Zip file.
@@ -71,6 +76,23 @@
  * </pre>
  */
 public class ZipFilterAction {
+  /** A copy of Bazel's singlejar.ZipCombiner#OutputMode enum. */
+  enum OutputMode {
+
+    /** Output entries using any method. */
+    DONT_CARE,
+
+    /**
+     * Output all entries using DEFLATE method, except directory entries. It is always more
+     * efficient to store directory entries uncompressed.
+     */
+    FORCE_DEFLATE,
+
+    /** Output all entries using STORED method. */
+    FORCE_STORED,
+  }
+
+  record GenerateExcludeListResult(int sawErrors, ArrayList<String> excludeList) {}
 
   private static final Logger logger = Logger.getLogger(ZipFilterAction.class.getName());
 
@@ -200,45 +222,133 @@ public static void main(String[] args) throws IOException {
     System.exit(run(args));
   }
 
-  static int run(String[] args) throws IOException {
-    Options options = new Options();
-    new JCommander(options).parse(args);
-    logger.fine(
-        String.format(
-            "Creating filter from entries of type %s, in zip files %s.",
-            options.filterTypes, options.filterZips));
-
-    final Stopwatch timer = Stopwatch.createStarted();
+  static GenerateExcludeListResult generateExcludeList(Options options, Stopwatch timer)
+      throws IOException {
     Multimap<String, Long> entriesToOmit =
         getEntriesToOmit(options.filterZips, options.filterTypes);
     final String explicitFilter =
         options.explicitFilters.isEmpty()
             ? ""
             : String.format(".*(%s).*", Joiner.on("|").join(options.explicitFilters));
     logger.fine(String.format("Filter created in %dms", timer.elapsed(TimeUnit.MILLISECONDS)));
+    ArrayList<String> excludeList = new ArrayList<>();
 
-    ImmutableMap.Builder<String, Long> inputEntries = ImmutableMap.builder();
-    try (ZipFile zf = new ZipFile(options.inputZip.toFile())) {
+    int sawErrors = 0;
+    try (ZipFile zf = new ZipFile(options.inputZip.toFile());
+        ZipOutputStream zos =
+            new ZipOutputStream(new FileOutputStream(options.outputZip.toString()))) {
       Enumeration<? extends ZipEntry> entries = zf.entries();
 
       while (entries.hasMoreElements()) {
         ZipEntry entry = entries.nextElement();
-        inputEntries.put(entry.getName(), entry.getCrc());
+        if (entry.getName().matches(explicitFilter)) {
+          excludeList.add(entry.getName());
+        } else if (entriesToOmit.containsEntry(entry.getName(), entry.getCrc())) {
+          excludeList.add(entry.getName());
+        } else if (entriesToOmit.containsKey(entry.getName())) {
+          // entriesToOmit contains the filename, but a different CRC was observed.
+          if (options.hashMismatchCheckMode == HashMismatchCheckMode.IGNORE) {
+            // Just add it to the excluded entry list.
+            excludeList.add(entry.getName());
+          } else {
+            if (options.hashMismatchCheckMode == HashMismatchCheckMode.ERROR) {
+              logger.severe(
+                  String.format(
+                      "ERROR: Requested to filter entries of name "
+                          + "'%s'; name matches but the hash does not.\n",
+                      entry.getName()));
+              sawErrors = 1;
+              excludeList.add(entry.getName());
+            } else {
+              logger.severe(
+                  String.format(
+                      "WARNING: Requested to filter entries of name "
+                          + "'%s'; name matches but the hash does not. Copying anyway.\n",
+                      entry.getName()));
+            }
+          }
+        }
       }
     }
+    return new GenerateExcludeListResult(sawErrors, excludeList);
+  }
 
-    ZipFilterEntryFilter entryFilter =
-        new ZipFilterEntryFilter(
-            explicitFilter,
-            entriesToOmit,
-            inputEntries.buildOrThrow(),
-            options.hashMismatchCheckMode);
+  @SuppressWarnings("RuntimeExec")
+  static int run(String[] args) throws IOException {
+    Options options = new Options();
+    new JCommander(options).parse(args);
+    logger.fine(
+        String.format(
+            "Creating filter from entries of type %s, in zip files %s.",
+            options.filterTypes, options.filterZips));
+
+    String compressionStrategy = "--dont_change_compression";
+    if (options.outputMode == OutputMode.FORCE_STORED) {
+      compressionStrategy = "--compression";
+    } else if (options.outputMode == OutputMode.FORCE_DEFLATE) {
+      throw new IllegalArgumentException("FORCE_DEFLATE is not supported.");
+    }
+
+    String singleJarPath =
+        Path.of(System.getProperty("runfiles.path"), System.getProperty("singlejar.path"))
+            .toString();
+    ImmutableList.Builder<String> singleJarArgsBuilder = ImmutableList.builder();
+    singleJarArgsBuilder
+        .add("--sources")
+        .add(options.inputZip.toString())
+        .add("--output")
+        .add(options.outputZip.toString())
+        .add(compressionStrategy)
+        .add("--exclude_build_data")
+        .add("--normalize");
+
+    final Stopwatch timer = Stopwatch.createStarted();
+    GenerateExcludeListResult excludeListResult = generateExcludeList(options, timer);
+    int sawErrors = excludeListResult.sawErrors();
+    ArrayList<String> excludeList = excludeListResult.excludeList();
+
+    if (!excludeList.isEmpty()) {
+      singleJarArgsBuilder.add("--exclude_zip_entries").addAll(excludeList);
+    }
+
+    ImmutableList<String> singleJarArgs = singleJarArgsBuilder.build();
+    // Dump the singlejar args into a params file
+    File paramsFile = File.createTempFile("singlejar_params", ".txt");
+    try (BufferedWriter writer = Files.newBufferedWriter(paramsFile.toPath(), UTF_8)) {
+      for (String arg : singleJarArgs) {
+        writer.write(arg + "\n");
+      }
+    }
 
-    try (OutputStream out = Files.newOutputStream(options.outputZip);
-        ZipCombiner combiner = new ZipCombiner(options.outputMode, entryFilter, out)) {
-      combiner.addZip(options.inputZip.toFile());
+    boolean singleJarError = false;
+    Process singleJarProcess =
+        Runtime.getRuntime().exec(new String[] {singleJarPath, "@" + paramsFile.getAbsolutePath()});
+    try {
+      int singlejarExitCode = singleJarProcess.waitFor();
+      if (singlejarExitCode != 0) {
+        sawErrors = 1;
+        singleJarError = true;
+        logger.severe(
+            String.format(
+                "ERROR: singlejar failed with exit code %d. See logs for details.",
+                singlejarExitCode));
+      }
+    } catch (InterruptedException e) {
+      singleJarError = true;
+      logger.severe(String.format("ERROR: singlejar was interrupted: %s", e.getMessage()));
+      sawErrors = 1;
+    }
+    // Dump out the singlejar stderr if there was an issue.
+    if (singleJarError) {
+      InputStreamReader isr = new InputStreamReader(singleJarProcess.getErrorStream(), UTF_8);
+      BufferedReader br = new BufferedReader(isr);
+      String line;
+      while ((line = br.readLine()) != null) {
+        System.out.println("  [singlejar stderr] " + line);
+      }
     }
     logger.fine(String.format("Filtering completed in %dms", timer.elapsed(TimeUnit.MILLISECONDS)));
-    return entryFilter.sawErrors() ? 1 : 0;
+
+    return sawErrors;
   }
 }