This is an automated email from the ASF dual-hosted git repository.
szetszwo pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 5e8b806f05 HDDS-8382. Upgrade Ratis to 2.5.1 (#4664)
5e8b806f05 is described below
commit 5e8b806f058a74dc854164118947c703977f94a6
Author: Doroszlai, Attila <[email protected]>
AuthorDate: Tue May 9 18:34:34 2023 +0200
HDDS-8382. Upgrade Ratis to 2.5.1 (#4664)
Co-authored-by: Tsz-Wo Nicholas Sze <[email protected]>
---
.../background/BlockDeletingService.java | 2 +-
.../client/rpc/TestContainerStateMachine.java | 12 ++++++--
.../rpc/TestContainerStateMachineFailures.java | 25 +++++++++++------
.../client/rpc/TestValidateBCSIDOnRestart.java | 4 ++-
.../impl/StatemachineImplTestUtil.java | 32 ++++++++++++++++++++++
pom.xml | 4 +--
6 files changed, 63 insertions(+), 16 deletions(-)
diff --git
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
index afea6cdc7a..7a2e6fd040 100644
---
a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
+++
b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/keyvalue/statemachine/background/BlockDeletingService.java
@@ -220,7 +220,7 @@ public class BlockDeletingService extends BackgroundService
{
long minReplicatedIndex =
ratisServer.getMinReplicatedIndex(pipelineID);
long containerBCSID = containerData.getBlockCommitSequenceId();
- if (minReplicatedIndex >= 0 && minReplicatedIndex < containerBCSID) {
+ if (minReplicatedIndex < containerBCSID) {
LOG.warn("Close Container log Index {} is not replicated across
all"
+ " the servers in the pipeline {} as the min replicated "
+ "index is {}. Deletion is not allowed in this container "
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
index 739adbca79..e925a9cbd3 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachine.java
@@ -54,6 +54,7 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_CONTAINER_REPORT_INTERV
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
+import org.apache.ratis.statemachine.impl.StatemachineImplTestUtil;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
@@ -181,7 +182,7 @@ public class TestContainerStateMachine {
(ContainerStateMachine) TestHelper.getStateMachine(cluster);
SimpleStateMachineStorage storage =
(SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
- Assert.assertNull(storage.findLatestSnapshot());
+ Assert.assertNull(StatemachineImplTestUtil.findLatestSnapshot(storage));
// Write 10 keys. Num snapshots should be equal to config value.
for (int i = 1; i <= 10; i++) {
@@ -202,7 +203,7 @@ public class TestContainerStateMachine {
stateMachine =
(ContainerStateMachine) TestHelper.getStateMachine(cluster);
storage = (SimpleStateMachineStorage)
stateMachine.getStateMachineStorage();
- Path parentPath = storage.findLatestSnapshot().getFile().getPath();
+ Path parentPath = getSnapshotPath(storage);
int numSnapshots = parentPath.getParent().toFile().listFiles().length;
Assert.assertTrue(Math.abs(ratisServerConfiguration
.getNumSnapshotsRetained() - numSnapshots) <= 1);
@@ -222,10 +223,15 @@ public class TestContainerStateMachine {
stateMachine =
(ContainerStateMachine) TestHelper.getStateMachine(cluster);
storage = (SimpleStateMachineStorage)
stateMachine.getStateMachineStorage();
- parentPath = storage.findLatestSnapshot().getFile().getPath();
+ parentPath = getSnapshotPath(storage);
numSnapshots = parentPath.getParent().toFile().listFiles().length;
Assert.assertTrue(Math.abs(ratisServerConfiguration
.getNumSnapshotsRetained() - numSnapshots) <= 1);
}
+ static Path getSnapshotPath(SimpleStateMachineStorage storage)
+ throws IOException {
+ return StatemachineImplTestUtil.findLatestSnapshot(storage)
+ .getFile().getPath();
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
index 03350bd4ba..42c16fb807 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestContainerStateMachineFailures.java
@@ -92,6 +92,7 @@ import org.apache.ratis.server.storage.FileInfo;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
import static org.hamcrest.core.Is.is;
+import org.apache.ratis.statemachine.impl.StatemachineImplTestUtil;
import org.apache.ratis.thirdparty.com.google.protobuf.ByteString;
import org.junit.Assert;
import static org.junit.Assert.assertThat;
@@ -424,11 +425,11 @@ public class TestContainerStateMachineFailures {
SimpleStateMachineStorage storage =
(SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
stateMachine.takeSnapshot();
- Path parentPath = storage.findLatestSnapshot().getFile().getPath();
+ final FileInfo snapshot = getSnapshotFileInfo(storage);
+ final Path parentPath = snapshot.getPath();
// Since the snapshot threshold is set to 1, since there are
// applyTransactions, we should see snapshots
Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
- FileInfo snapshot = storage.findLatestSnapshot().getFile();
Assert.assertNotNull(snapshot);
long containerID = omKeyLocationInfo.getContainerID();
// delete the container db file
@@ -470,7 +471,7 @@ public class TestContainerStateMachineFailures {
if (snapshot.getPath().toFile().exists()) {
// Make sure the latest snapshot is same as the previous one
try {
- FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
+ final FileInfo latestSnapshot = getSnapshotFileInfo(storage);
Assert.assertTrue(snapshot.getPath().equals(latestSnapshot.getPath()));
} catch (Throwable e) {
Assert.assertFalse(snapshot.getPath().toFile().exists());
@@ -512,10 +513,10 @@ public class TestContainerStateMachineFailures {
omKeyLocationInfo.getPipeline());
SimpleStateMachineStorage storage =
(SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
- Path parentPath = storage.findLatestSnapshot().getFile().getPath();
+ final FileInfo snapshot = getSnapshotFileInfo(storage);
+ final Path parentPath = snapshot.getPath();
stateMachine.takeSnapshot();
Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
- FileInfo snapshot = storage.findLatestSnapshot().getFile();
Assert.assertNotNull(snapshot);
long containerID = omKeyLocationInfo.getContainerID();
Pipeline pipeline = cluster.getStorageContainerLocationClient()
@@ -548,7 +549,7 @@ public class TestContainerStateMachineFailures {
} finally {
xceiverClientManager.releaseClient(xceiverClient, false);
}
- FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
+ final FileInfo latestSnapshot = getSnapshotFileInfo(storage);
Assert.assertFalse(snapshot.getPath().equals(latestSnapshot.getPath()));
}
@@ -589,12 +590,12 @@ public class TestContainerStateMachineFailures {
omKeyLocationInfo.getPipeline());
SimpleStateMachineStorage storage =
(SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
- Path parentPath = storage.findLatestSnapshot().getFile().getPath();
+ final FileInfo snapshot = getSnapshotFileInfo(storage);
+ final Path parentPath = snapshot.getPath();
stateMachine.takeSnapshot();
// Since the snapshot threshold is set to 1, since there are
// applyTransactions, we should see snapshots
Assert.assertTrue(parentPath.getParent().toFile().listFiles().length > 0);
- FileInfo snapshot = storage.findLatestSnapshot().getFile();
Assert.assertNotNull(snapshot);
long containerID = omKeyLocationInfo.getContainerID();
Pipeline pipeline = cluster.getStorageContainerLocationClient()
@@ -676,7 +677,8 @@ public class TestContainerStateMachineFailures {
} catch (IOException ioe) {
Assert.fail("Exception should not be thrown");
}
- FileInfo latestSnapshot = storage.findLatestSnapshot().getFile();
+
+ final FileInfo latestSnapshot = getSnapshotFileInfo(storage);
Assert.assertFalse(snapshot.getPath().equals(latestSnapshot.getPath()));
r2.run();
@@ -815,4 +817,9 @@ public class TestContainerStateMachineFailures {
Assert.fail("Exception not expected " + e.getMessage());
}
}
+
+ static FileInfo getSnapshotFileInfo(SimpleStateMachineStorage storage)
+ throws IOException {
+ return StatemachineImplTestUtil.findLatestSnapshot(storage).getFile();
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
index fa3d25af56..31175a06b4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestValidateBCSIDOnRestart.java
@@ -60,6 +60,7 @@ import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_PIPELINE_REPORT_INTERVA
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_PIPELINE_DESTROY_TIMEOUT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_STALENODE_INTERVAL;
import org.apache.ratis.statemachine.impl.SimpleStateMachineStorage;
+import org.apache.ratis.statemachine.impl.StatemachineImplTestUtil;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -191,7 +192,8 @@ public class TestValidateBCSIDOnRestart {
SimpleStateMachineStorage storage =
(SimpleStateMachineStorage) stateMachine.getStateMachineStorage();
stateMachine.takeSnapshot();
- Path parentPath = storage.findLatestSnapshot().getFile().getPath();
+ final Path parentPath =
StatemachineImplTestUtil.findLatestSnapshot(storage)
+ .getFile().getPath();
stateMachine.buildMissingContainerSet(parentPath.toFile());
// Since the snapshot threshold is set to 1, since there are
// applyTransactions, we should see snapshots
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java
new file mode 100644
index 0000000000..b134f29cbe
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/ratis/statemachine/impl/StatemachineImplTestUtil.java
@@ -0,0 +1,32 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.ratis.statemachine.impl;
+
+import java.io.File;
+import java.io.IOException;
+
+/**
+ * Helper for integration test using `SimpleStateMachineStorage`.
+ */
+public interface StatemachineImplTestUtil {
+ static SingleFileSnapshotInfo findLatestSnapshot(
+ SimpleStateMachineStorage storage) throws IOException {
+ final File dir = storage.getStateMachineDir();
+ return SimpleStateMachineStorage.findLatestSnapshot(dir.toPath());
+ }
+}
diff --git a/pom.xml b/pom.xml
index f4aec47f80..a49bc772a3 100644
--- a/pom.xml
+++ b/pom.xml
@@ -75,10 +75,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xs
<!-- HDDS Rocks Native dependency version-->
<hdds.rocks.native.version>${hdds.version}</hdds.rocks.native.version>
<!-- Apache Ratis version -->
- <ratis.version>2.4.2-8b8bdda-SNAPSHOT</ratis.version>
+ <ratis.version>2.5.1</ratis.version>
<!-- Apache Ratis thirdparty version -->
- <ratis.thirdparty.version>1.0.3</ratis.thirdparty.version>
+ <ratis.thirdparty.version>1.0.4</ratis.thirdparty.version>
<!-- Apache Ranger plugin version -->
<ranger.version>2.3.0</ranger.version>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
