This is an automated email from the ASF dual-hosted git repository. jmckenzie pushed a commit to branch trunk in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/trunk by this push: new 3e0b94565a Add support to generate a One-Shot heap dump on unhandled exceptions 3e0b94565a is described below commit 3e0b94565acc64e903d73af3a14b23c875abc5b3 Author: Josh McKenzie <jmcken...@apache.org> AuthorDate: Tue Aug 2 16:02:03 2022 -0400 Add support to generate a One-Shot heap dump on unhandled exceptions Patch by Caleb Rackliffe; reviewed by Josh McKenzie, David Capwell, and Jon Meredith for CASSANDRA-17795 Co-authored-by: Caleb Rackliffe <calebrackli...@gmail.com> Co-authored-by: Josh McKenzie <jmcken...@apache.org> --- CHANGES.txt | 1 + build.xml | 1 + conf/cassandra.yaml | 10 +++ ide/idea/workspace.xml | 6 +- src/java/org/apache/cassandra/config/Config.java | 3 + .../cassandra/config/DatabaseDescriptor.java | 79 +++++++++++++++++++++ .../org/apache/cassandra/service/StorageProxy.java | 12 ++++ .../cassandra/service/StorageProxyMBean.java | 3 + src/java/org/apache/cassandra/utils/HeapUtils.java | 82 +++++++++++++++++++++- .../cassandra/utils/JVMStabilityInspector.java | 3 + test/conf/cassandra.yaml | 3 + .../org/apache/cassandra/utils/HeapUtilsTest.java | 63 +++++++++++++++++ .../org/apache/cassandra/tools/BulkLoaderTest.java | 26 ++++--- 13 files changed, 277 insertions(+), 15 deletions(-) diff --git a/CHANGES.txt b/CHANGES.txt index 3aaaf8b38e..dd09b25c56 100644 --- a/CHANGES.txt +++ b/CHANGES.txt @@ -1,4 +1,5 @@ 4.2 + * Add support to generate a One-Shot heap dump on unhandled exceptions (CASSANDRA-17795) * Rate-limit new client connection auth setup to avoid overwhelming bcrypt (CASSANDRA-17812) * DataOutputBuffer#scratchBuffer can use off-heap or on-heap memory as a means to control memory allocations (CASSANDRA-16471) * Add ability to read the TTLs and write times of the elements of a collection and/or UDT (CASSANDRA-8877) diff --git a/build.xml b/build.xml index ca346c9f28..82a805d5d8 100644 --- a/build.xml +++ b/build.xml @@ -1500,6 +1500,7 @@ more aggressively rather than waiting. See CASSANDRA-14922 for more details. --> <jvmarg value="-XX:SoftRefLRUPolicyMSPerMB=0" /> + <jvmarg value="-XX:HeapDumpPath=build/test" /> <jvmarg value="-Dcassandra.test.driver.connection_timeout_ms=${test.driver.connection_timeout_ms}"/> <jvmarg value="-Dcassandra.test.driver.read_timeout_ms=${test.driver.read_timeout_ms}"/> <jvmarg value="-Dcassandra.memtable_row_overhead_computation_step=100"/> diff --git a/conf/cassandra.yaml b/conf/cassandra.yaml index 98d70a035f..21e3f78c10 100644 --- a/conf/cassandra.yaml +++ b/conf/cassandra.yaml @@ -107,6 +107,16 @@ auto_hints_cleanup_enabled: false # parameters: # - +# Directory where Cassandra should store results of a One-Shot troubleshooting heapdump for uncaught exceptions. +# Note: this value can be overridden by the -XX:HeapDumpPath JVM env param with a relative local path for testing if +# so desired. +# If not set, the default directory is $CASSANDRA_HOME/heapdump +# heap_dump_path: /var/lib/cassandra/heapdump + +# Enable / disable automatic dump of heap on first uncaught exception +# If not set, the default value is false +# dump_heap_on_uncaught_exception: true + # Enable / disable persistent hint windows. # # If set to false, a hint will be stored only in case a respective node diff --git a/ide/idea/workspace.xml b/ide/idea/workspace.xml index e35ba90ac7..321edd8024 100644 --- a/ide/idea/workspace.xml +++ b/ide/idea/workspace.xml @@ -143,7 +143,7 @@ <configuration default="true" type="Application" factoryName="Application"> <extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" /> <option name="MAIN_CLASS_NAME" value="" /> - <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -DQT_SHRINKS=0 -ea -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true" /> + <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -DQT_SHRINKS=0 -ea -Dcassandra.reads.thresholds.coordinator.defensive_checks_enabled=true -XX:HeapDumpPath=build/test" /> <option name="PROGRAM_PARAMETERS" value="" /> <option name="WORKING_DIRECTORY" value="" /> <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" /> @@ -167,7 +167,7 @@ <option name="MAIN_CLASS_NAME" value="" /> <option name="METHOD_NAME" value="" /> <option name="TEST_OBJECT" value="class" /> - <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.skip_sync=true -ea -XX:MaxMet [...] + <option name="VM_PARAMETERS" value="-Dcassandra.config=file://$PROJECT_DIR$/test/conf/cassandra.yaml -Dlogback.configurationFile=file://$PROJECT_DIR$/test/conf/logback-test.xml -Dcassandra.logdir=$PROJECT_DIR$/build/test/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dlegacy-sstable-root=$PROJECT_DIR$/test/data/legacy-sstables -Dinvalid-legacy-sstable-root=$PROJECT_DIR$/test/data/invalid-legacy-sstables -Dcassandra.ring_delay_ms=1000 -Dcassandra.skip_sync=true -ea -XX:MaxMet [...] <option name="PARAMETERS" value="" /> <fork_mode value="class" /> <option name="WORKING_DIRECTORY" value="" /> @@ -187,7 +187,7 @@ <configuration default="false" name="Cassandra" type="Application" factoryName="Application"> <extension name="coverage" enabled="false" merge="false" sample_coverage="true" runner="idea" /> <option name="MAIN_CLASS_NAME" value="org.apache.cassandra.service.CassandraDaemon" /> - <option name="VM_PARAMETERS" value="-Dcassandra-foreground=yes -Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=7199 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate= [...] + <option name="VM_PARAMETERS" value="-Dcassandra-foreground=yes -Dcassandra.config=file://$PROJECT_DIR$/conf/cassandra.yaml -Dcassandra.storagedir=$PROJECT_DIR$/data -Dlogback.configurationFile=file://$PROJECT_DIR$/conf/logback.xml -Dcassandra.logdir=$PROJECT_DIR$/data/logs -Djava.library.path=$PROJECT_DIR$/lib/sigar-bin -Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.port=7199 -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate= [...] <option name="PROGRAM_PARAMETERS" value="" /> <option name="WORKING_DIRECTORY" value="file://$PROJECT_DIR$" /> <option name="ALTERNATIVE_JRE_PATH_ENABLED" value="false" /> diff --git a/src/java/org/apache/cassandra/config/Config.java b/src/java/org/apache/cassandra/config/Config.java index bdca0a7df1..f14bb3544a 100644 --- a/src/java/org/apache/cassandra/config/Config.java +++ b/src/java/org/apache/cassandra/config/Config.java @@ -1179,4 +1179,7 @@ public class Config logger.info("Node configuration:[{}]", Joiner.on("; ").join(configMap.entrySet())); } + + public volatile boolean dump_heap_on_uncaught_exception = false; + public String heap_dump_path = "heapdump"; } diff --git a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java index 0a9036f632..e000eeea26 100644 --- a/src/java/org/apache/cassandra/config/DatabaseDescriptor.java +++ b/src/java/org/apache/cassandra/config/DatabaseDescriptor.java @@ -18,6 +18,8 @@ package org.apache.cassandra.config; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.RuntimeMXBean; import java.net.Inet4Address; import java.net.Inet6Address; import java.net.InetAddress; @@ -25,6 +27,8 @@ import java.net.NetworkInterface; import java.net.SocketException; import java.net.UnknownHostException; import java.nio.file.FileStore; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.ArrayList; import java.util.Collection; import java.util.Comparator; @@ -32,11 +36,13 @@ import java.util.Enumeration; import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.Set; import java.util.UUID; import java.util.concurrent.TimeUnit; import java.util.function.Function; import java.util.function.Supplier; +import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -906,6 +912,9 @@ public class DatabaseDescriptor conf.paxos_state_purging = PaxosStatePurging.legacy; logInitializationOutcome(logger); + + if (conf.dump_heap_on_uncaught_exception && DatabaseDescriptor.getHeapDumpPath() == null) + throw new ConfigurationException(String.format("Invalid configuration. Heap dump is enabled but cannot create heap dump output path: %s.", conf.heap_dump_path != null ? conf.heap_dump_path : "null")); } @VisibleForTesting @@ -1582,6 +1591,13 @@ public class DatabaseDescriptor throw new ConfigurationException("cdc_raw_directory must be specified", false); FileUtils.createDirectory(conf.cdc_raw_directory); } + + boolean created = maybeCreateHeapDumpPath(); + if (!created && conf.dump_heap_on_uncaught_exception) + { + logger.error(String.format("cassandra.yaml:dump_heap_on_uncaught_exception is enabled but unable to create heap dump path %s. Disabling.", conf.heap_dump_path != null ? conf.heap_dump_path : "null")); + conf.dump_heap_on_uncaught_exception = false; + } } catch (ConfigurationException e) { @@ -4397,4 +4413,67 @@ public class DatabaseDescriptor { conf.min_tracked_partition_tombstone_count = value; } + + public static boolean getDumpHeapOnUncaughtException() + { + return conf.dump_heap_on_uncaught_exception; + } + + /** + * @return Whether the path exists (be it created now or already prior) + */ + private static boolean maybeCreateHeapDumpPath() + { + if (!conf.dump_heap_on_uncaught_exception) + return false; + + Path heap_dump_path = getHeapDumpPath(); + if (heap_dump_path == null) + { + logger.warn("Neither -XX:HeapDumpPath nor cassandra.yaml:heap_dump_path are set; unable to create a directory to hold the output."); + return false; + } + if (PathUtils.exists(Paths.get(conf.heap_dump_path))) + return true; + return PathUtils.createDirectoryIfNotExists(Paths.get(conf.heap_dump_path)); + } + + /** + * As this is at its heart a debug operation (getting a one-shot heapdump from an uncaught exception), we support + * both the more evolved cassandra.yaml approach but also the -XX param to override it on a one-off basis so you don't + * have to change the full config of a node or a cluster in order to get a heap dump from a single node that's + * misbehaving. + * @return the absolute path of the -XX param if provided, else the heap_dump_path in cassandra.yaml + */ + public static Path getHeapDumpPath() + { + RuntimeMXBean runtimeMxBean = ManagementFactory.getRuntimeMXBean(); + Optional<String> pathArg = runtimeMxBean.getInputArguments().stream().filter(s -> s.startsWith("-XX:HeapDumpPath=")).findFirst(); + + if (pathArg.isPresent()) + { + Pattern HEAP_DUMP_PATH_SPLITTER = Pattern.compile("HeapDumpPath="); + String fullHeapPathString = HEAP_DUMP_PATH_SPLITTER.split(pathArg.get())[1]; + Path absolutePath = Paths.get(fullHeapPathString).toAbsolutePath(); + Path basePath = fullHeapPathString.endsWith(".hprof") ? absolutePath.subpath(0, absolutePath.getNameCount() - 1) : absolutePath; + return Paths.get("/").resolve(basePath); + } + if (conf.heap_dump_path == null) + throw new ConfigurationException("Attempted to get heap dump path without -XX:HeapDumpPath or cassandra.yaml:heap_dump_path set."); + return Paths.get(conf.heap_dump_path); + } + + public static void setDumpHeapOnUncaughtException(boolean enabled) + { + conf.dump_heap_on_uncaught_exception = enabled; + boolean pathExists = maybeCreateHeapDumpPath(); + + if (enabled && !pathExists) + { + logger.error("Attempted to enable heap dump but cannot create the requested path. Disabling."); + conf.dump_heap_on_uncaught_exception = false; + } + else + logger.info("Setting dump_heap_on_uncaught_exception to {}", enabled); + } } diff --git a/src/java/org/apache/cassandra/service/StorageProxy.java b/src/java/org/apache/cassandra/service/StorageProxy.java index e89bdae717..2bf140fc29 100644 --- a/src/java/org/apache/cassandra/service/StorageProxy.java +++ b/src/java/org/apache/cassandra/service/StorageProxy.java @@ -3167,4 +3167,16 @@ public class StorageProxy implements StorageProxyMBean { return PaxosState.getDisableCoordinatorLocking(); } + + @Override + public boolean getDumpHeapOnUncaughtException() + { + return DatabaseDescriptor.getDumpHeapOnUncaughtException(); + } + + @Override + public void setDumpHeapOnUncaughtException(boolean enabled) + { + DatabaseDescriptor.setDumpHeapOnUncaughtException(enabled); + } } diff --git a/src/java/org/apache/cassandra/service/StorageProxyMBean.java b/src/java/org/apache/cassandra/service/StorageProxyMBean.java index 546143d515..5d7bc69569 100644 --- a/src/java/org/apache/cassandra/service/StorageProxyMBean.java +++ b/src/java/org/apache/cassandra/service/StorageProxyMBean.java @@ -135,4 +135,7 @@ public interface StorageProxyMBean void setPaxosCoordinatorLockingDisabled(boolean disabled); boolean getPaxosCoordinatorLockingDisabled(); + + public boolean getDumpHeapOnUncaughtException(); + public void setDumpHeapOnUncaughtException(boolean enabled); } diff --git a/src/java/org/apache/cassandra/utils/HeapUtils.java b/src/java/org/apache/cassandra/utils/HeapUtils.java index c0910d87fc..504855bbe1 100644 --- a/src/java/org/apache/cassandra/utils/HeapUtils.java +++ b/src/java/org/apache/cassandra/utils/HeapUtils.java @@ -22,15 +22,25 @@ import java.io.BufferedReader; import java.io.IOException; import java.io.InputStreamReader; import java.lang.management.ManagementFactory; +import java.nio.file.FileStore; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.locks.Lock; +import java.util.concurrent.locks.ReentrantLock; +import javax.management.MBeanServer; -import org.apache.cassandra.io.util.File; import org.apache.commons.lang3.ArrayUtils; import org.apache.commons.lang3.text.StrBuilder; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import com.sun.management.HotSpotDiagnosticMXBean; +import org.apache.cassandra.config.DatabaseDescriptor; +import org.apache.cassandra.io.util.File; +import org.apache.cassandra.io.util.PathUtils; + import static org.apache.cassandra.config.CassandraRelevantEnv.JAVA_HOME; +import static org.apache.cassandra.utils.Clock.Global.currentTimeMillis; /** * Utility to log heap histogram. @@ -40,6 +50,8 @@ public final class HeapUtils { private static final Logger logger = LoggerFactory.getLogger(HeapUtils.class); + private static final Lock DUMP_LOCK = new ReentrantLock(); + /** * Generates a HEAP histogram in the log file. */ @@ -73,6 +85,72 @@ public final class HeapUtils } } + /** + * @return full path to the created heap dump file + */ + public static String maybeCreateHeapDump() + { + // Make sure that only one heap dump can be in progress across all threads, and abort for + // threads that cannot immediately acquire the lock, allowing them to fail normally. + if (DUMP_LOCK.tryLock()) + { + try + { + if (DatabaseDescriptor.getDumpHeapOnUncaughtException()) + { + MBeanServer server = ManagementFactory.getPlatformMBeanServer(); + + Path absoluteBasePath = DatabaseDescriptor.getHeapDumpPath(); + // We should never reach this point with this value null as we initialize the bool only after confirming + // the -XX param / .yaml conf is present on initial init and the JMX entry point, but still worth checking. + if (absoluteBasePath == null) + { + DatabaseDescriptor.setDumpHeapOnUncaughtException(false); + throw new RuntimeException("Cannot create heap dump unless -XX:HeapDumpPath or cassandra.yaml:heap_dump_path is specified."); + } + + long maxMemoryBytes = Runtime.getRuntime().maxMemory(); + long freeSpaceBytes = PathUtils.tryGetSpace(absoluteBasePath, FileStore::getUnallocatedSpace); + + // Abort if there isn't enough room on the target disk to dump the entire heap and then copy it. + if (freeSpaceBytes < 2 * maxMemoryBytes) + throw new RuntimeException("Cannot allocated space for a heap dump snapshot. There are only " + freeSpaceBytes + " bytes free at " + absoluteBasePath + '.'); + + HotSpotDiagnosticMXBean mxBean = ManagementFactory.newPlatformMXBeanProxy(server, "com.sun.management:type=HotSpotDiagnostic", HotSpotDiagnosticMXBean.class); + String filename = String.format("pid%s-epoch%s.hprof", HeapUtils.getProcessId().toString(), currentTimeMillis()); + String fullPath = Paths.get(absoluteBasePath.toString(), filename).toString(); + + logger.info("Writing heap dump to {} on partition w/ {} free bytes...", absoluteBasePath, freeSpaceBytes); + mxBean.dumpHeap(fullPath, false); + logger.info("Heap dump written to {}", fullPath); + + // Disable further heap dump creations until explicitly re-enabled. + DatabaseDescriptor.setDumpHeapOnUncaughtException(false); + + return fullPath; + } + else + { + logger.debug("Heap dump creation on uncaught exceptions is disabled."); + } + } + catch (Throwable e) + { + logger.warn("Unable to create heap dump.", e); + } + finally + { + DUMP_LOCK.unlock(); + } + } + else + { + logger.debug("Heap dump creation is already in progress. Request aborted."); + } + + return null; + } + /** * Retrieve the path to the JCMD executable. * @return the path to the JCMD executable or null if it cannot be found. diff --git a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java index 1d3c09f371..ff86396af9 100644 --- a/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java +++ b/src/java/org/apache/cassandra/utils/JVMStabilityInspector.java @@ -133,6 +133,9 @@ public final class JVMStabilityInspector isUnstable = true; } + // Anything other than an OOM, we should try and heap dump to capture what's going on if configured to do so + HeapUtils.maybeCreateHeapDump(); + if (t instanceof InterruptedException) throw new UncheckedInterruptedException((InterruptedException) t); diff --git a/test/conf/cassandra.yaml b/test/conf/cassandra.yaml index 80de239390..53bc206664 100644 --- a/test/conf/cassandra.yaml +++ b/test/conf/cassandra.yaml @@ -54,6 +54,9 @@ drop_compact_storage_enabled: true file_cache_enabled: true auto_hints_cleanup_enabled: true +heap_dump_path: build/test +dump_heap_on_uncaught_exception: true + read_thresholds_enabled: true coordinator_read_size_warn_threshold: 1024KiB coordinator_read_size_fail_threshold: 4096KiB diff --git a/test/long/org/apache/cassandra/utils/HeapUtilsTest.java b/test/long/org/apache/cassandra/utils/HeapUtilsTest.java new file mode 100644 index 0000000000..18c91c44e1 --- /dev/null +++ b/test/long/org/apache/cassandra/utils/HeapUtilsTest.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.cassandra.utils; + +import java.nio.file.Paths; + +import org.junit.BeforeClass; +import org.junit.Test; + +import org.apache.cassandra.config.DatabaseDescriptor; + +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; + +public class HeapUtilsTest +{ + @BeforeClass + public static void setup() + { + DatabaseDescriptor.daemonInitialization(); + } + + @Test + public void shouldDumpHeapWithPathArgSpecified() + { + DatabaseDescriptor.setDumpHeapOnUncaughtException(true); + String path = HeapUtils.maybeCreateHeapDump(); + assertNotNull(path); + assertTrue(Paths.get(path).toFile().exists()); + assertFalse(DatabaseDescriptor.getDumpHeapOnUncaughtException()); + + // After the first dump, subsequent requests should be no-ops... + path = HeapUtils.maybeCreateHeapDump(); + assertNull(path); + + // ...until creation is manually re-enabled. + DatabaseDescriptor.setDumpHeapOnUncaughtException(true); + assertTrue(DatabaseDescriptor.getDumpHeapOnUncaughtException()); + assertNotNull(DatabaseDescriptor.getHeapDumpPath()); + path = HeapUtils.maybeCreateHeapDump(); + assertNotNull(path); + assertTrue(Paths.get(path).toFile().exists()); + assertFalse(DatabaseDescriptor.getDumpHeapOnUncaughtException()); + } +} \ No newline at end of file diff --git a/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java b/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java index d9d759a287..2d9acf7cef 100644 --- a/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java +++ b/test/unit/org/apache/cassandra/tools/BulkLoaderTest.java @@ -64,7 +64,9 @@ public class BulkLoaderTest extends OfflineToolUtils "globalEventExecutor-[1-9]-[1-9]", "globalEventExecutor-[1-9]-[1-9]", "Shutdown-checker", - "cluster[0-9]-connection-reaper-[0-9]" }); + "cluster[0-9]-connection-reaper-[0-9]", + "Attach Listener", + "process reaper"}); assertSchemaNotLoaded(); assertCLSMNotLoaded(); assertSystemKSNotLoaded(); @@ -92,11 +94,13 @@ public class BulkLoaderTest extends OfflineToolUtils "globalEventExecutor-[1-9]-[1-9]", "globalEventExecutor-[1-9]-[1-9]", "Shutdown-checker", - "cluster[0-9]-connection-reaper-[0-9]" }); - assertSchemaNotLoaded(); - assertCLSMNotLoaded(); - assertSystemKSNotLoaded(); - assertKeyspaceNotLoaded(); + "cluster[0-9]-connection-reaper-[0-9]", + "Attach Listener", + "process reaper"}); + assertSchemaNotLoaded(); + assertCLSMNotLoaded(); + assertSystemKSNotLoaded(); + assertKeyspaceNotLoaded(); assertServerNotLoaded(); } @@ -120,10 +124,12 @@ public class BulkLoaderTest extends OfflineToolUtils "globalEventExecutor-[1-9]-[1-9]", "globalEventExecutor-[1-9]-[1-9]", "Shutdown-checker", - "cluster[0-9]-connection-reaper-[0-9]" }); - assertSchemaNotLoaded(); - assertCLSMNotLoaded(); - assertSystemKSNotLoaded(); + "cluster[0-9]-connection-reaper-[0-9]", + "Attach Listener", + "process reaper"}); + assertSchemaNotLoaded(); + assertCLSMNotLoaded(); + assertSystemKSNotLoaded(); assertKeyspaceNotLoaded(); assertServerNotLoaded(); } --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@cassandra.apache.org For additional commands, e-mail: commits-h...@cassandra.apache.org