This is an automated email from the ASF dual-hosted git repository.
lhotari pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pulsar.git
The following commit(s) were added to refs/heads/master by this push:
new b189ea79722 [fix][test] Fix resource leaks in pulsar-metadata
bookkeeper.replication tests (#21483)
b189ea79722 is described below
commit b189ea79722f23358f7463b066261334ddce18a1
Author: Lari Hotari <[email protected]>
AuthorDate: Tue Oct 31 14:52:02 2023 +0200
[fix][test] Fix resource leaks in pulsar-metadata bookkeeper.replication
tests (#21483)
---
pulsar-metadata/pom.xml | 7 ++++++-
.../replication/AuditorCheckAllLedgersTaskTest.java | 10 +++++-----
.../replication/AuditorLedgerCheckerTest.java | 8 ++++----
.../AuditorPlacementPolicyCheckTaskTest.java | 9 ++++++---
.../replication/AuditorReplicasCheckTaskTest.java | 6 +++---
.../replication/BookKeeperClusterTestCase.java | 20 ++++++++++++++++++++
.../replication/BookieLedgerIndexTest.java | 4 ++--
7 files changed, 46 insertions(+), 18 deletions(-)
diff --git a/pulsar-metadata/pom.xml b/pulsar-metadata/pom.xml
index f14f9fab96c..ac39594b1dd 100644
--- a/pulsar-metadata/pom.xml
+++ b/pulsar-metadata/pom.xml
@@ -29,6 +29,11 @@
<relativePath>..</relativePath>
</parent>
+ <properties>
+ <!-- workaround for OOM error 134 in CI -->
+ <testForkCount>2</testForkCount>
+ </properties>
+
<artifactId>pulsar-metadata</artifactId>
<name>Pulsar Metadata</name>
<dependencies>
@@ -149,7 +154,7 @@
</execution>
</executions>
</plugin>
-
+
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-jar-plugin</artifactId>
diff --git
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java
index 6b58c72af07..795e91c7572 100644
---
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java
+++
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorCheckAllLedgersTaskTest.java
@@ -65,15 +65,15 @@ public class AuditorCheckAllLedgersTaskTest extends
BookKeeperClusterTestCase {
@Override
public void setUp() throws Exception {
super.setUp();
- final BookKeeper bookKeeper = new BookKeeper(baseClientConf);
+ final BookKeeper bookKeeper = registerCloseable(new
BookKeeper(baseClientConf));
admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new
ClientConfiguration(baseClientConf));
String ledgersRoot = "/ledgers";
String storeUri = metadataServiceUri.replaceAll("zk://",
"").replaceAll("/ledgers", "");
- MetadataStoreExtended store = MetadataStoreExtended.create(storeUri,
- MetadataStoreConfig.builder().fsyncEnable(false).build());
+ MetadataStoreExtended store =
registerCloseable(MetadataStoreExtended.create(storeUri,
+ MetadataStoreConfig.builder().fsyncEnable(false).build()));
LayoutManager layoutManager = new PulsarLayoutManager(store,
ledgersRoot);
- PulsarLedgerManagerFactory ledgerManagerFactory = new
PulsarLedgerManagerFactory();
+ PulsarLedgerManagerFactory ledgerManagerFactory =
registerCloseable(new PulsarLedgerManagerFactory());
ClientConfiguration conf = new ClientConfiguration();
conf.setZkLedgersRootPath(ledgersRoot);
@@ -86,7 +86,7 @@ public class AuditorCheckAllLedgersTaskTest extends
BookKeeperClusterTestCase {
acquireConcurrentOpenLedgerOperationsTimeoutMSec);
}
- @AfterMethod
+ @AfterMethod(alwaysRun = true)
@Override
public void tearDown() throws Exception {
if (ledgerManager != null) {
diff --git
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
index ec5f77f7946..a1954831abf 100644
---
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
+++
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorLedgerCheckerTest.java
@@ -136,14 +136,14 @@ public class AuditorLedgerCheckerTest extends
BookKeeperClusterTestCase {
String ledgersRoot = "/ledgers";
String storeUri = metadataServiceUri.replaceAll("zk://",
"").replaceAll("/ledgers", "");
- MetadataStoreExtended store = MetadataStoreExtended.create(storeUri,
- MetadataStoreConfig.builder().fsyncEnable(false).build());
+ MetadataStoreExtended store =
registerCloseable(MetadataStoreExtended.create(storeUri,
+ MetadataStoreConfig.builder().fsyncEnable(false).build()));
LayoutManager layoutManager = new PulsarLayoutManager(store,
ledgersRoot);
- PulsarLedgerManagerFactory ledgerManagerFactory = new
PulsarLedgerManagerFactory();
+ PulsarLedgerManagerFactory ledgerManagerFactory =
registerCloseable(new PulsarLedgerManagerFactory());
ClientConfiguration conf = new ClientConfiguration();
conf.setZkLedgersRootPath(ledgersRoot);
ledgerManagerFactory.initialize(conf, layoutManager, 1);
- urLedgerMgr = ledgerManagerFactory.newLedgerUnderreplicationManager();
+ urLedgerMgr =
registerCloseable(ledgerManagerFactory.newLedgerUnderreplicationManager());
urLedgerMgr.setCheckAllLedgersCTime(System.currentTimeMillis());
baseClientConf.setMetadataServiceUri(
diff --git
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java
index 8b9c0b14302..f5d6576229e 100644
---
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java
+++
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorPlacementPolicyCheckTaskTest.java
@@ -63,16 +63,19 @@ public class AuditorPlacementPolicyCheckTaskTest extends
BookKeeperClusterTestCa
super.setUp();
baseClientConf.setMetadataServiceUri(
metadataServiceUri.replaceAll("zk://",
"metadata-store:").replaceAll("/ledgers", ""));
- final BookKeeper bookKeeper = new BookKeeper(baseClientConf);
+ final BookKeeper bookKeeper = registerCloseable(new
BookKeeper(baseClientConf));
admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new
ClientConfiguration(baseClientConf));
- LedgerManagerFactory ledgerManagerFactory =
bookKeeper.getLedgerManagerFactory();
+ LedgerManagerFactory ledgerManagerFactory =
registerCloseable(bookKeeper.getLedgerManagerFactory());
ledgerManager = ledgerManagerFactory.newLedgerManager();
ledgerUnderreplicationManager =
ledgerManagerFactory.newLedgerUnderreplicationManager();
}
- @AfterMethod
+ @AfterMethod(alwaysRun = true)
@Override
public void tearDown() throws Exception {
+ if (admin != null) {
+ admin.close();
+ }
if (ledgerManager != null) {
ledgerManager.close();
}
diff --git
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java
index 62162bd25f4..b48498639e7 100644
---
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java
+++
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/AuditorReplicasCheckTaskTest.java
@@ -63,14 +63,14 @@ public class AuditorReplicasCheckTaskTest extends
BookKeeperClusterTestCase {
super.setUp();
baseClientConf.setMetadataServiceUri(
metadataServiceUri.replaceAll("zk://",
"metadata-store:").replaceAll("/ledgers", ""));
- final BookKeeper bookKeeper = new BookKeeper(baseClientConf);
+ final BookKeeper bookKeeper = registerCloseable(new
BookKeeper(baseClientConf));
admin = new BookKeeperAdmin(bookKeeper, NullStatsLogger.INSTANCE, new
ClientConfiguration(baseClientConf));
- LedgerManagerFactory ledgerManagerFactory =
bookKeeper.getLedgerManagerFactory();
+ LedgerManagerFactory ledgerManagerFactory =
registerCloseable(bookKeeper.getLedgerManagerFactory());
ledgerManager = ledgerManagerFactory.newLedgerManager();
ledgerUnderreplicationManager =
ledgerManagerFactory.newLedgerUnderreplicationManager();
}
- @AfterMethod
+ @AfterMethod(alwaysRun = true)
@Override
public void tearDown() throws Exception {
if (ledgerManager != null) {
diff --git
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java
index 9a8e3ef5a2d..ccbdb8cef64 100644
---
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java
+++
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookKeeperClusterTestCase.java
@@ -115,6 +115,8 @@ public abstract class BookKeeperClusterTestCase {
protected ExecutorService executor;
private final List<Integer> bookiePorts = new ArrayList<>();
+ private final List<AutoCloseable> closeables = new ArrayList<>();
+
SynchronousQueue<Throwable> asyncExceptions = new SynchronousQueue<>();
protected void captureThrowable(Runnable c) {
try {
@@ -187,6 +189,9 @@ public abstract class BookKeeperClusterTestCase {
@AfterTest(alwaysRun = true)
public void tearDown() throws Exception {
+ callCloseables(closeables);
+ closeables.clear();
+
boolean failed = false;
for (Throwable e : asyncExceptions) {
LOG.error("Got async exception: ", e);
@@ -228,6 +233,21 @@ public abstract class BookKeeperClusterTestCase {
}
}
+ protected <T extends AutoCloseable> T registerCloseable(T closeable) {
+ closeables.add(closeable);
+ return closeable;
+ }
+
+ private static void callCloseables(List<AutoCloseable> closeables) {
+ for (int i = closeables.size() - 1; i >= 0; i--) {
+ try {
+ closeables.get(i).close();
+ } catch (Exception e) {
+ LOG.error("Failure in calling close method", e);
+ }
+ }
+ }
+
/**
* Start zookeeper cluster.
*
diff --git
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java
index 1d5cf868cce..425f485e1ad 100644
---
a/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java
+++
b/pulsar-metadata/src/test/java/org/apache/bookkeeper/replication/BookieLedgerIndexTest.java
@@ -91,8 +91,8 @@ public class BookieLedgerIndexTest extends
BookKeeperClusterTestCase {
String ledgersRoot = "/ledgers";
String storeUri = metadataServiceUri.replaceAll("zk://",
"").replaceAll("/ledgers", "");
- MetadataStoreExtended store = MetadataStoreExtended.create(storeUri,
- MetadataStoreConfig.builder().fsyncEnable(false).build());
+ MetadataStoreExtended store =
registerCloseable(MetadataStoreExtended.create(storeUri,
+ MetadataStoreConfig.builder().fsyncEnable(false).build()));
LayoutManager layoutManager = new PulsarLayoutManager(store,
ledgersRoot);
newLedgerManagerFactory = new PulsarLedgerManagerFactory();
