This is an automated email from the ASF dual-hosted git repository. fjy pushed a commit to branch 0.15.1-incubating in repository https://gitbox.apache.org/repos/asf/incubator-druid.git
The following commit(s) were added to refs/heads/0.15.1-incubating by this push: new 6888cfb fix issue with CuratorLoadQueuePeon shutting down executors it does not own (#8140) (#8151) 6888cfb is described below commit 6888cfbfdb0b18123e5850e8f42dbe957323fb32 Author: Clint Wylie <cwy...@apache.org> AuthorDate: Thu Jul 25 10:12:30 2019 -0700 fix issue with CuratorLoadQueuePeon shutting down executors it does not own (#8140) (#8151) * fix issue with CuratorLoadQueuePeon shutting down executors it does not own * use lifecycled executors * maybe this --- .../server/coordinator/CuratorLoadQueuePeon.java | 2 -- .../coordinator/CuratorDruidCoordinatorTest.java | 22 ++++++++++++++++++---- .../java/org/apache/druid/cli/CliCoordinator.java | 12 +++++++++--- 3 files changed, 27 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/coordinator/CuratorLoadQueuePeon.java b/server/src/main/java/org/apache/druid/server/coordinator/CuratorLoadQueuePeon.java index a4d5d94..6a526e8 100644 --- a/server/src/main/java/org/apache/druid/server/coordinator/CuratorLoadQueuePeon.java +++ b/server/src/main/java/org/apache/druid/server/coordinator/CuratorLoadQueuePeon.java @@ -334,8 +334,6 @@ public class CuratorLoadQueuePeon extends LoadQueuePeon queuedSize.set(0L); failedAssignCount.set(0); - processingExecutor.shutdown(); - callBackExecutor.shutdown(); } private void entryRemoved(SegmentHolder segmentHolder, String path) diff --git a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java index 9cd0206..1cea98e 100644 --- a/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java +++ b/server/src/test/java/org/apache/druid/server/coordinator/CuratorDruidCoordinatorTest.java @@ -73,7 +73,9 @@ import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ConcurrentMap; import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; +import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicReference; @@ -117,6 +119,9 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase private final ObjectMapper jsonMapper; private final ZkPathsConfig zkPathsConfig; + private ScheduledExecutorService peonExec = Execs.scheduledSingleThreaded("Master-PeonExec--%d"); + private ExecutorService callbackExec = Execs.multiThreaded(4, "LoadQueuePeon-callbackexec--%d"); + public CuratorDruidCoordinatorTest() { jsonMapper = TestHelper.makeJsonMapper(); @@ -186,16 +191,16 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase curator, SOURCE_LOAD_PATH, objectMapper, - Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_src_scheduled-%d"), - Execs.singleThreaded("coordinator_test_load_queue_peon_src-%d"), + peonExec, + callbackExec, druidCoordinatorConfig ); destinationLoadQueuePeon = new CuratorLoadQueuePeon( curator, DESTINATION_LOAD_PATH, objectMapper, - Execs.scheduledSingleThreaded("coordinator_test_load_queue_peon_dest_scheduled-%d"), - Execs.singleThreaded("coordinator_test_load_queue_peon_dest-%d"), + peonExec, + callbackExec, druidCoordinatorConfig ); druidNode = new DruidNode("hey", "what", false, 1234, null, true, false); @@ -261,6 +266,15 @@ public class CuratorDruidCoordinatorTest extends CuratorTestBase public final TestRule timeout = new DeadlockDetectingTimeout(60, TimeUnit.SECONDS); @Test + public void testStopDoesntKillPoolItDoesntOwn() throws Exception + { + setupView(); + sourceLoadQueuePeon.stop(); + Assert.assertFalse(peonExec.isShutdown()); + Assert.assertFalse(callbackExec.isShutdown()); + } + + @Test public void testMoveSegment() throws Exception { segmentViewInitLatch = new CountDownLatch(1); diff --git a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java index 98bb958..834b417 100644 --- a/services/src/main/java/org/apache/druid/cli/CliCoordinator.java +++ b/services/src/main/java/org/apache/druid/cli/CliCoordinator.java @@ -47,7 +47,9 @@ import org.apache.druid.guice.annotations.CoordinatorIndexingServiceHelper; import org.apache.druid.guice.annotations.EscalatedGlobal; import org.apache.druid.guice.http.JettyHttpClientModule; import org.apache.druid.java.util.common.concurrent.Execs; +import org.apache.druid.java.util.common.concurrent.ExecutorServices; import org.apache.druid.java.util.common.concurrent.ScheduledExecutorFactory; +import org.apache.druid.java.util.common.lifecycle.Lifecycle; import org.apache.druid.java.util.common.logger.Logger; import org.apache.druid.java.util.http.client.HttpClient; import org.apache.druid.metadata.MetadataRuleManager; @@ -248,7 +250,8 @@ public class CliCoordinator extends ServerRunnable ScheduledExecutorFactory factory, DruidCoordinatorConfig config, @EscalatedGlobal HttpClient httpClient, - ZkPathsConfig zkPaths + ZkPathsConfig zkPaths, + Lifecycle lifecycle ) { boolean useHttpLoadQueuePeon = "http".equalsIgnoreCase(config.getLoadQueuePeonType()); @@ -256,9 +259,12 @@ public class CliCoordinator extends ServerRunnable if (useHttpLoadQueuePeon) { callBackExec = Execs.singleThreaded("LoadQueuePeon-callbackexec--%d"); } else { - callBackExec = Execs.multiThreaded(config.getNumCuratorCallBackThreads(), "LoadQueuePeon" - + "-callbackexec--%d"); + callBackExec = Execs.multiThreaded( + config.getNumCuratorCallBackThreads(), + "LoadQueuePeon-callbackexec--%d" + ); } + ExecutorServices.manageLifecycle(lifecycle, callBackExec); return new LoadQueueTaskMaster( curator, jsonMapper, --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@druid.apache.org For additional commands, e-mail: commits-h...@druid.apache.org