SOLR-17971: Fix REBALANCELEADERS wait when the Overseer is disabled

changelog/unreleased/solr-17971-rebalance-leaders-distributed.yml

@@ -0,0 +1,7 @@
+title: "REBALANCELEADERS no longer waits until timeout or leaves async ids behind when the Overseer is disabled."
+type: fixed
+authors:
+  - name: Amrit Sarkar
+links:
+  - name: SOLR-17971
+    url: https://issues.apache.org/jira/browse/SOLR-17971

solr/core/src/java/org/apache/solr/cloud/ZkController.java

@@ -2510,6 +2510,24 @@ public boolean clearAsyncId(String asyncId) throws KeeperException {
     }
   }
 
+  /**
+   * Removes the stored response of a completed or failed async operation and releases its asyncId,
+   * in both Overseer and distributed modes. An asyncId with no stored response is left claimed,
+   * since its task may still be running or pending.
+   *
+   * @return True if a stored response was found and removed.
+   */
+  public boolean removeStoredAsyncResponse(String asyncId) throws Exception {
+    if (distributedCommandRunner.isPresent()) {
+      return distributedCommandRunner.get().deleteSingleAsyncId(asyncId);
+    }
+    if (overseerCompletedMap.remove(asyncId) || overseerFailureMap.remove(asyncId)) {
+      clearAsyncId(asyncId);
+      return true;
+    }
+    return false;
+  }
+
   public Overseer getOverseer() {
     return overseer;
   }

solr/core/src/java/org/apache/solr/handler/admin/RebalanceLeaders.java

@@ -122,7 +122,7 @@ class RebalanceLeaders {
     coreContainer = collectionsHandler.getCoreContainer();
   }
 
-  void execute() throws KeeperException, InterruptedException {
+  void execute() throws Exception {
     DocCollection dc = checkParams();
 
     int max = req.getParams().getInt(MAX_AT_ONCE_PROP, Integer.MAX_VALUE);
@@ -471,8 +471,7 @@ private void rejoinElectionQueue(
   // mean failure!
   //
 
-  private boolean waitAsyncRequests(final int maxWaitSecs, Boolean waitForAll)
-      throws KeeperException, InterruptedException {
+  private boolean waitAsyncRequests(final int maxWaitSecs, Boolean waitForAll) throws Exception {
 
     if (asyncRequests.size() == 0) {
       return true;
@@ -482,15 +481,7 @@ private boolean waitAsyncRequests(final int maxWaitSecs, Boolean waitForAll)
       Iterator<String> iter = asyncRequests.iterator();
       boolean foundChange = false;
       while (iter.hasNext()) {
-        String asyncId = iter.next();
-        if (coreContainer.getZkController().getOverseerFailureMap().contains(asyncId)) {
-          coreContainer.getZkController().getOverseerFailureMap().remove(asyncId);
-          coreContainer.getZkController().clearAsyncId(asyncId);
-          iter.remove();
-          foundChange = true;
-        } else if (coreContainer.getZkController().getOverseerCompletedMap().contains(asyncId)) {
-          coreContainer.getZkController().getOverseerCompletedMap().remove(asyncId);
-          coreContainer.getZkController().clearAsyncId(asyncId);
+        if (coreContainer.getZkController().removeStoredAsyncResponse(iter.next())) {
           iter.remove();
           foundChange = true;
         }

solr/core/src/java/org/apache/solr/handler/admin/api/ClusterCommands.java

@@ -129,20 +129,7 @@ public DeleteClusterCommandStatusResponse deleteClusterCommandStatus(String requ
 
     final DeleteClusterCommandStatusResponse response =
         instantiateJerseyResponse(DeleteClusterCommandStatusResponse.class);
-    final ZkController zkController = coreContainer.getZkController();
-
-    if (zkController.getDistributedCommandRunner().isEmpty()) {
-      if (zkController.getOverseerCompletedMap().remove(requestId)) {
-        zkController.clearAsyncId(requestId);
-        response.status = "successfully removed stored response for [" + requestId + "]";
-      } else if (zkController.getOverseerFailureMap().remove(requestId)) {
-        zkController.clearAsyncId(requestId);
-        response.status = "successfully removed stored response for [" + requestId + "]";
-      } else {
-        // Don't call zkController.clearAsyncId for this, since it could be a running/pending task
-        response.status = "[" + requestId + "] not found in stored responses";
-      }
-    } else if (zkController.getDistributedCommandRunner().get().deleteSingleAsyncId(requestId)) {
+    if (coreContainer.getZkController().removeStoredAsyncResponse(requestId)) {
       response.status = "successfully removed stored response for [" + requestId + "]";
     } else {
       response.status = "[" + requestId + "] not found in stored responses";

solr/core/src/test/org/apache/solr/cloud/TestRebalanceLeaders.java

@@ -157,6 +157,46 @@ public void testRebalanceLeaders() throws Exception {
     checkPreferredsAreLeaders();
   }
 
+  // Sub-command async ids must be found and cleared in both Overseer and distributed modes.
+  @Test
+  public void testRebalanceLeadersClearsAsyncIds() throws Exception {
+    DocCollection docCollection =
+        cluster.getSolrClient().getClusterState().getCollection(COLLECTION_NAME);
+    for (Slice slice : docCollection.getSlices()) {
+      Replica nonLeader =
+          slice.getReplicas().stream()
+              .filter(rep -> rep.getName().equals(slice.getLeader().getName()) == false)
+              .findFirst()
+              .orElseThrow();
+      setProp(slice, nonLeader, "preferredLeader");
+    }
+
+    int maxWaitSeconds = 30;
+    long start = System.nanoTime();
+    CollectionAdminResponse resp =
+        CollectionAdminRequest.rebalanceLeaders(COLLECTION_NAME)
+            .setMaxWaitSeconds(maxWaitSeconds)
+            .process(cluster.getSolrClient());
+    long elapsedSeconds = TimeUnit.NANOSECONDS.toSeconds(System.nanoTime() - start);
+    assertTrue(resp.getResponse().get("Summary").toString().contains("Success"));
+
+    for (String path :
+        List.of(
+            "/distributedapi/async/persistent",
+            "/overseer/collection-map-completed",
+            "/overseer/collection-map-failure")) {
+      if (zkClient().exists(path)) {
+        List<String> leftover =
+            zkClient().getChildren(path, null).stream()
+                .filter(id -> id.startsWith("mn-rebalanceleaders_"))
+                .toList();
+        assertTrue("Async ids not cleared from " + path + ": " + leftover, leftover.isEmpty());
+      }
+    }
+    assertTrue(
+        "Waited " + elapsedSeconds + "s, hit maxWaitSeconds", elapsedSeconds < maxWaitSeconds);
+  }
+
   // Ensure that the property is set on only one replica per slice when changing a unique property
   // on an individual
   // replica.