This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0231bd7d05 HDDS-8261. Fail integration tests if RocksDB objects are
leaked (#4590)
0231bd7d05 is described below
commit 0231bd7d0558b229a18d5353723ab1661a80a230
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Fri Apr 21 07:59:55 2023 +0200
HDDS-8261. Fail integration tests if RocksDB objects are leaked (#4590)
---
.../hadoop/hdds/utils/db/TestRDBTableStore.java | 36 +++++++++--------
.../db/managed/ManagedRocksObjectMetrics.java | 9 ++++-
.../hadoop/hdds/scm/server/TestSCMCertStore.java | 43 ++++++++++++---------
.../dev-support/checks/_mvn_unit_report.sh | 18 +++++++--
.../ozone/TestDirectoryDeletingServiceWithFSO.java | 45 +++++++++++++---------
.../hadoop/hdds/scm/TestRatisPipelineLeader.java | 9 +++--
.../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 3 ++
.../TestContainerStateMachineFailureOnRead.java | 9 +++--
.../rpc/TestOzoneClientMultipartUploadWithFSO.java | 5 ++-
.../ozoneimpl/TestOzoneContainerWithTLS.java | 27 ++++++-------
.../ozone/om/TestObjectStoreWithLegacyFS.java | 5 +--
.../ozone/om/TestOzoneManagerHAKeyDeletion.java | 10 +++--
.../ozone/recon/TestReconWithOzoneManagerHA.java | 21 +++++-----
.../src/test/resources/log4j.properties | 1 +
14 files changed, 145 insertions(+), 96 deletions(-)
diff --git
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
index 486d16ff52..b52d5779dc 100644
---
a/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
+++
b/hadoop-hdds/framework/src/test/java/org/apache/hadoop/hdds/utils/db/TestRDBTableStore.java
@@ -428,9 +428,10 @@ public class TestRDBTableStore {
// Remove without next removes first entry.
try (Table<byte[], byte[]> testTable = rdbStore.getTable("Fifth")) {
writeToTable(testTable, 3);
- TableIterator<byte[], ? extends Table.KeyValue<byte[], byte[]>> iterator
=
- testTable.iterator();
- iterator.removeFromDB();
+ try (TableIterator<?, ? extends Table.KeyValue<?, ?>> iterator =
+ testTable.iterator()) {
+ iterator.removeFromDB();
+ }
Assertions.assertNull(testTable.get(bytesOf[1]));
Assertions.assertNotNull(testTable.get(bytesOf[2]));
Assertions.assertNotNull(testTable.get(bytesOf[3]));
@@ -439,10 +440,11 @@ public class TestRDBTableStore {
// Remove after seekToLast removes lastEntry
try (Table<byte[], byte[]> testTable = rdbStore.getTable("Sixth")) {
writeToTable(testTable, 3);
- TableIterator<byte[], ? extends Table.KeyValue<byte[], byte[]>> iterator
=
- testTable.iterator();
- iterator.seekToLast();
- iterator.removeFromDB();
+ try (TableIterator<?, ? extends Table.KeyValue<?, ?>> iterator =
+ testTable.iterator()) {
+ iterator.seekToLast();
+ iterator.removeFromDB();
+ }
Assertions.assertNotNull(testTable.get(bytesOf[1]));
Assertions.assertNotNull(testTable.get(bytesOf[2]));
Assertions.assertNull(testTable.get(bytesOf[3]));
@@ -451,10 +453,11 @@ public class TestRDBTableStore {
// Remove after seek deletes that entry.
try (Table<byte[], byte[]> testTable = rdbStore.getTable("Sixth")) {
writeToTable(testTable, 3);
- TableIterator<byte[], ? extends Table.KeyValue<byte[], byte[]>> iterator
=
- testTable.iterator();
- iterator.seek(bytesOf[3]);
- iterator.removeFromDB();
+ try (TableIterator<byte[], ? extends Table.KeyValue<?, ?>> iterator =
+ testTable.iterator()) {
+ iterator.seek(bytesOf[3]);
+ iterator.removeFromDB();
+ }
Assertions.assertNotNull(testTable.get(bytesOf[1]));
Assertions.assertNotNull(testTable.get(bytesOf[2]));
Assertions.assertNull(testTable.get(bytesOf[3]));
@@ -463,11 +466,12 @@ public class TestRDBTableStore {
// Remove after next() deletes entry that was returned by next.
try (Table<byte[], byte[]> testTable = rdbStore.getTable("Sixth")) {
writeToTable(testTable, 3);
- TableIterator<byte[], ? extends Table.KeyValue<byte[], byte[]>> iterator
=
- testTable.iterator();
- iterator.seek(bytesOf[2]);
- iterator.next();
- iterator.removeFromDB();
+ try (TableIterator<byte[], ? extends Table.KeyValue<?, ?>> iterator =
+ testTable.iterator()) {
+ iterator.seek(bytesOf[2]);
+ iterator.next();
+ iterator.removeFromDB();
+ }
Assertions.assertNotNull(testTable.get(bytesOf[1]));
Assertions.assertNull(testTable.get(bytesOf[2]));
Assertions.assertNotNull(testTable.get(bytesOf[3]));
diff --git
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
index 37d8b2cf4f..8d9d7fe78d 100644
---
a/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
+++
b/hadoop-hdds/managed-rocksdb/src/main/java/org/apache/hadoop/hdds/utils/db/managed/ManagedRocksObjectMetrics.java
@@ -32,7 +32,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
@Metrics(about = "Ozone Manage RocksObject Metrics",
context = OzoneConsts.OZONE)
public class ManagedRocksObjectMetrics {
- static final ManagedRocksObjectMetrics INSTANCE = create();
+ public static final ManagedRocksObjectMetrics INSTANCE = create();
private static final String SOURCE_NAME =
ManagedRocksObjectMetrics.class.getSimpleName();
@@ -54,6 +54,13 @@ public class ManagedRocksObjectMetrics {
totalLeakObjects.incr();
}
+ public void assertNoLeaks() {
+ final long cnt = totalLeakObjects.value();
+ if (cnt > 0) {
+ throw new AssertionError("Found " + cnt + " leaked objects, check logs");
+ }
+ }
+
void increaseManagedObject() {
totalManagedObjects.incr();
}
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
index f607791e4a..ed0a0f26af 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/server/TestSCMCertStore.java
@@ -29,6 +29,8 @@ import
org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateSto
import
org.apache.hadoop.hdds.security.x509.certificate.authority.DefaultCRLApprover;
import org.apache.hadoop.hdds.security.x509.certificate.utils.CertificateCodec;
import org.apache.hadoop.hdds.security.x509.crl.CRLInfo;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.bouncycastle.asn1.x509.CRLReason;
import org.bouncycastle.cert.X509CertificateHolder;
@@ -39,6 +41,7 @@ import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import java.io.IOException;
import java.math.BigInteger;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -48,7 +51,6 @@ import java.security.cert.X509Certificate;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
-import java.util.Iterator;
import java.util.List;
import java.util.Optional;
import java.util.Set;
@@ -175,8 +177,7 @@ public class TestSCMCertStore {
assertFalse(sequenceId.isPresent());
- assertEquals(1L,
- getTableSize(scmMetadataStore.getCRLInfoTable().iterator()));
+ assertEquals(1L, getTableSize(scmMetadataStore.getCRLInfoTable()));
// Generate 3 more certificates and revoke 2 of them
List<BigInteger> newSerialIDs = new ArrayList<>();
@@ -218,17 +219,21 @@ public class TestSCMCertStore {
assertEquals(newCrlInfo.getCrlSequenceID(), sequenceId.get().longValue());
- // Valid certs table should have 1 cert
- assertEquals(1L,
- getTableSize(scmMetadataStore.getValidCertsTable().iterator()));
- // Make sure that the last certificate that was not revoked is the one
- // in the valid certs table.
- assertEquals(newSerialIDs.get(2),
- scmMetadataStore.getValidCertsTable().iterator().next().getKey());
+ Table<BigInteger, X509Certificate> validCertsTable =
+ scmMetadataStore.getValidCertsTable();
+ try (TableIterator<BigInteger, ? extends Table.KeyValue<BigInteger,
+ X509Certificate>> iterator = validCertsTable.iterator()) {
+ assertTrue(iterator.hasNext());
+ // Make sure that the last certificate that was not revoked is the one
+ // in the valid certs table.
+ assertEquals(newSerialIDs.get(2), iterator.next().getKey());
+ // Valid certs table should have 1 cert
+ assertFalse(iterator.hasNext());
+ }
// Revoked certs table should have 3 certs
assertEquals(3L,
- getTableSize(scmMetadataStore.getRevokedCertsV2Table().iterator()));
+ getTableSize(scmMetadataStore.getRevokedCertsV2Table()));
}
@Test
@@ -266,15 +271,17 @@ public class TestSCMCertStore {
"SHA256withRSA")));
}
- private long getTableSize(Iterator iterator) {
- long size = 0;
+ private long getTableSize(Table<?, ?> table) throws IOException {
+ try (TableIterator<?, ?> iterator = table.iterator()) {
+ long size = 0;
- while (iterator.hasNext()) {
- size++;
- iterator.next();
- }
+ while (iterator.hasNext()) {
+ size++;
+ iterator.next();
+ }
- return size;
+ return size;
+ }
}
@Test
diff --git a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
index 18f33dd352..5139dddcd8 100755
--- a/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
+++ b/hadoop-ozone/dev-support/checks/_mvn_unit_report.sh
@@ -26,11 +26,20 @@ _realpath() {
fi
}
+tempfile="${REPORT_DIR}/summary.tmp"
+
## generate summary txt file
find "." -not -path '*/iteration*' -name 'TEST*.xml' -print0 \
| xargs -n1 -0 "grep" -l -E "<failure|<error" \
| awk -F/ '{sub("'"TEST-"'",""); sub(".xml",""); print $NF}' \
- | tee "$REPORT_DIR/summary.txt"
+ > "${tempfile}"
+
+if [[ "${CHECK:-unit}" == "integration" ]]; then
+ find "." -not -path '*/iteration*' -name '*-output.txt' -print0 \
+ | xargs -n1 -0 "grep" -l -E "not closed properly|was not shutdown
properly" \
+ | awk -F/ '{sub("-output.txt",""); print $NF}' \
+ >> "${tempfile}"
+fi
#Copy heap dump and dump leftovers
find "." -not -path '*/iteration*' \
@@ -45,7 +54,7 @@ grep -A1 'Crashed tests' "${REPORT_DIR}/output.log" \
| grep -v -e 'Crashed tests' -e '--' \
| cut -f2- -d' ' \
| sort -u \
- | tee -a "${REPORT_DIR}/summary.txt"
+ >> "${tempfile}"
# Check for tests that started but were not finished
if grep -q 'There was a timeout.*in the fork' "${REPORT_DIR}/output.log"; then
@@ -60,9 +69,12 @@ if grep -q 'There was a timeout.*in the fork'
"${REPORT_DIR}/output.log"; then
| sort -u -k2) \
| grep '^- ' \
| awk '{ print $3 }' \
- | tee -a "${REPORT_DIR}/summary.txt"
+ >> "${tempfile}"
fi
+sort -u "${tempfile}" | tee "${REPORT_DIR}/summary.txt"
+rm "${tempfile}"
+
#Collect of all of the report files of FAILED tests
for failed_test in $(< ${REPORT_DIR}/summary.txt); do
for file in $(find "." -not -path '*/iteration*' \
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
index e1a65783c3..ea93231631 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestDirectoryDeletingServiceWithFSO.java
@@ -33,6 +33,7 @@ import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneClient;
+import org.apache.hadoop.ozone.om.OMMetadataManager;
import org.apache.hadoop.ozone.om.helpers.SnapshotInfo;
import org.apache.hadoop.ozone.om.service.DirectoryDeletingService;
import org.apache.hadoop.ozone.om.service.KeyDeletingService;
@@ -171,9 +172,11 @@ public class TestDirectoryDeletingServiceWithFSO {
assertSubPathsCount(dirDeletingService::getMovedDirsCount, 0);
assertSubPathsCount(dirDeletingService::getMovedFilesCount, 0);
- assertTrue(dirTable.iterator().hasNext());
- assertEquals(root.getName(),
- dirTable.iterator().next().getValue().getName());
+ try (TableIterator<?, ? extends Table.KeyValue<?, OmDirectoryInfo>>
+ iterator = dirTable.iterator()) {
+ assertTrue(iterator.hasNext());
+ assertEquals(root.getName(), iterator.next().getValue().getName());
+ }
assertTrue(dirDeletingService.getRunCount().get() > 1);
}
@@ -480,23 +483,27 @@ public class TestDirectoryDeletingServiceWithFSO {
}
private void cleanupTables() throws IOException {
- cluster.getOzoneManager().getMetadataManager()
- .getDeletedDirTable().iterator().removeFromDB();
- try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
- iterator = cluster.getOzoneManager().getMetadataManager()
- .getFileTable().iterator()) {
- while (iterator.hasNext()) {
- iterator.next();
- iterator.removeFromDB();
- }
+ OMMetadataManager metadataManager =
+ cluster.getOzoneManager().getMetadataManager();
+
+ try (TableIterator<?, ?> it = metadataManager.getDeletedDirTable()
+ .iterator()) {
+ removeAllFromDB(it);
}
- try (TableIterator<String, ? extends Table.KeyValue<String,
- OmDirectoryInfo>> deletedItr = cluster.getOzoneManager()
- .getMetadataManager().getDirectoryTable().iterator()) {
- while (deletedItr.hasNext()) {
- deletedItr.next();
- deletedItr.removeFromDB();
- }
+ try (TableIterator<?, ?> it = metadataManager.getFileTable().iterator()) {
+ removeAllFromDB(it);
+ }
+ try (TableIterator<?, ?> it = metadataManager.getDirectoryTable()
+ .iterator()) {
+ removeAllFromDB(it);
+ }
+ }
+
+ private static void removeAllFromDB(TableIterator<?, ?> iterator)
+ throws IOException {
+ while (iterator.hasNext()) {
+ iterator.next();
+ iterator.removeFromDB();
}
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java
index 2feff4e199..00b7d3535c 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/hdds/scm/TestRatisPipelineLeader.java
@@ -96,15 +96,16 @@ public class TestRatisPipelineLeader {
}
}, 200, 20000);
// Verify client connects to Leader without NotLeaderException
- XceiverClientRatis xceiverClientRatis =
- XceiverClientRatis.newXceiverClientRatis(ratisPipeline, conf);
final Logger log = LoggerFactory.getLogger(
"org.apache.ratis.grpc.server.GrpcClientProtocolService");
GenericTestUtils.setLogLevel(log, Level.DEBUG);
GenericTestUtils.LogCapturer logCapturer =
GenericTestUtils.LogCapturer.captureLogs(log);
- xceiverClientRatis.connect();
- ContainerProtocolCalls.createContainer(xceiverClientRatis, 1L, null);
+ try (XceiverClientRatis xceiverClientRatis =
+ XceiverClientRatis.newXceiverClientRatis(ratisPipeline, conf)) {
+ xceiverClientRatis.connect();
+ ContainerProtocolCalls.createContainer(xceiverClientRatis, 1L, null);
+ }
logCapturer.stopCapturing();
Assertions.assertFalse(
logCapturer.getOutput().contains(
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
index d0e6423c00..5c63f98a15 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
import org.apache.hadoop.hdds.utils.IOUtils;
+import org.apache.hadoop.hdds.utils.db.managed.ManagedRocksObjectMetrics;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
@@ -461,6 +462,8 @@ public class MiniOzoneClusterImpl implements
MiniOzoneCluster {
FileUtils.deleteDirectory(baseDir);
ContainerCache.getInstance(conf).shutdownCache();
DefaultMetricsSystem.shutdown();
+
+ ManagedRocksObjectMetrics.INSTANCE.assertNoLeaks();
} catch (IOException e) {
LOG.error("Exception while shutting down the cluster.", e);
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailureOnRead.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailureOnRead.java
index 7e0b2aae90..d84ede8602 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailureOnRead.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailureOnRead.java
@@ -164,10 +164,11 @@ public class TestContainerStateMachineFailureOnRead {
Assert.assertTrue(dnToStop.isPresent());
cluster.shutdownHddsDatanode(dnToStop.get().getDatanodeDetails());
// Verify healthy pipeline before creating key
- XceiverClientRatis xceiverClientRatis =
- XceiverClientRatis.newXceiverClientRatis(ratisPipeline, conf);
- xceiverClientRatis.connect();
- TestOzoneContainer.createContainerForTesting(xceiverClientRatis, 100L);
+ try (XceiverClientRatis xceiverClientRatis =
+ XceiverClientRatis.newXceiverClientRatis(ratisPipeline, conf)) {
+ xceiverClientRatis.connect();
+ TestOzoneContainer.createContainerForTesting(xceiverClientRatis, 100L);
+ }
OmKeyLocationInfo omKeyLocationInfo;
OzoneOutputStream key = objectStore.getVolume(volumeName)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
index 6bea20df33..90ac291817 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneClientMultipartUploadWithFSO.java
@@ -440,8 +440,9 @@ public class TestOzoneClientMultipartUploadWithFSO {
Assert.assertNotNull(omMultipartCommitUploadPartInfo);
byte[] fileContent = new byte[data.length];
- OzoneInputStream inputStream = bucket.readKey(keyName);
- inputStream.read(fileContent);
+ try (OzoneInputStream inputStream = bucket.readKey(keyName)) {
+ inputStream.read(fileContent);
+ }
StringBuilder sb = new StringBuilder(data.length);
// Combine all parts data, and check is it matching with get key data.
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
index 186a3bea14..22d55992cf 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/ozoneimpl/TestOzoneContainerWithTLS.java
@@ -181,19 +181,20 @@ public class TestOzoneContainerWithTLS {
//Set scmId and manually start ozone container.
container.start(UUID.randomUUID().toString());
- XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf,
- Collections.singletonList(caClient.getCACertificate()));
-
- if (containerTokenEnabled) {
- secretManager.start(caClient);
- client.connect();
- createSecureContainer(client, containerId,
- secretManager.generateToken(
- UserGroupInformation.getCurrentUser().getUserName(),
- ContainerID.valueOf(containerId)));
- } else {
- client.connect();
- createContainer(client, containerId);
+ try (XceiverClientGrpc client = new XceiverClientGrpc(pipeline, conf,
+ Collections.singletonList(caClient.getCACertificate()))) {
+
+ if (containerTokenEnabled) {
+ secretManager.start(caClient);
+ client.connect();
+ createSecureContainer(client, containerId,
+ secretManager.generateToken(
+ UserGroupInformation.getCurrentUser().getUserName(),
+ ContainerID.valueOf(containerId)));
+ } else {
+ client.connect();
+ createContainer(client, containerId);
+ }
}
} finally {
if (container != null) {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java
index 8409131261..3803dc70c7 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreWithLegacyFS.java
@@ -151,9 +151,8 @@ public class TestObjectStoreWithLegacyFS {
String dbKey, int expectedCnt, String keyName) {
int countKeys = 0;
int matchingKeys = 0;
- try {
- TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
- itr = keyTable.iterator();
+ try (TableIterator<String, ? extends Table.KeyValue<String, OmKeyInfo>>
+ itr = keyTable.iterator()) {
itr.seek(dbKey);
while (itr.hasNext()) {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java
index ca24d5dce7..f90ddf7b05 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestOzoneManagerHAKeyDeletion.java
@@ -17,7 +17,10 @@
package org.apache.hadoop.ozone.om;
+import org.apache.hadoop.hdds.utils.db.Table;
+import org.apache.hadoop.hdds.utils.db.TableIterator;
import org.apache.hadoop.ozone.client.OzoneBucket;
+import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
import org.apache.hadoop.ozone.om.service.KeyDeletingService;
import org.apache.ozone.test.GenericTestUtils;
import org.junit.jupiter.api.Test;
@@ -71,9 +74,10 @@ public class TestOzoneManagerHAKeyDeletion extends
TestOzoneManagerHA {
getCluster().getOzoneManagersList().forEach((om) -> {
try {
GenericTestUtils.waitFor(() -> {
- try {
- return !om.getMetadataManager().getDeletedTable().iterator()
- .hasNext();
+ Table<String, RepeatedOmKeyInfo> deletedTable =
+ om.getMetadataManager().getDeletedTable();
+ try (TableIterator<?, ?> iterator = deletedTable.iterator()) {
+ return !iterator.hasNext();
} catch (Exception ex) {
return false;
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
index 91a4347798..3179728e75 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/recon/TestReconWithOzoneManagerHA.java
@@ -142,16 +142,17 @@ public class TestReconWithOzoneManagerHA {
ReconContainerMetadataManager reconContainerMetadataManager =
cluster.getReconServer().getReconContainerMetadataManager();
- TableIterator iterator =
- reconContainerMetadataManager.getContainerTableIterator();
- String reconKeyPrefix = null;
- while (iterator.hasNext()) {
- Table.KeyValue<ContainerKeyPrefix, Integer> keyValue =
- (Table.KeyValue<ContainerKeyPrefix, Integer>) iterator.next();
- reconKeyPrefix = keyValue.getKey().getKeyPrefix();
+ try (TableIterator iterator =
+ reconContainerMetadataManager.getContainerTableIterator()) {
+ String reconKeyPrefix = null;
+ while (iterator.hasNext()) {
+ Table.KeyValue<ContainerKeyPrefix, Integer> keyValue =
+ (Table.KeyValue<ContainerKeyPrefix, Integer>) iterator.next();
+ reconKeyPrefix = keyValue.getKey().getKeyPrefix();
+ }
+ Assert.assertEquals("Container data should be synced to recon.",
+ String.format("/%s/%s/%s", VOL_NAME, VOL_NAME, keyPrefix),
+ reconKeyPrefix);
}
- Assert.assertEquals("Container data should be synced to recon.",
- String.format("/%s/%s/%s", VOL_NAME, VOL_NAME, keyPrefix),
- reconKeyPrefix);
}
}
diff --git a/hadoop-ozone/integration-test/src/test/resources/log4j.properties
b/hadoop-ozone/integration-test/src/test/resources/log4j.properties
index b8ad21d6c7..009160888a 100644
--- a/hadoop-ozone/integration-test/src/test/resources/log4j.properties
+++ b/hadoop-ozone/integration-test/src/test/resources/log4j.properties
@@ -19,3 +19,4 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601}
[%t] %-5p %c{2} (%F:%
log4j.logger.org.apache.hadoop.security.ShellBasedUnixGroupsMapping=ERROR
log4j.logger.org.apache.hadoop.util.NativeCodeLoader=ERROR
+log4j.logger.org.apache.hadoop.hdds.utils.db.managed=TRACE
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
