This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new a9e76d412b HDDS-9826. Fix handling of GroupMismatchException in
XceiverClientRatis.watchForCommit (#5727)
a9e76d412b is described below
commit a9e76d412b37e9996d1883e0bfec7bbdf3d34dd6
Author: Ivan Brusentsev <[email protected]>
AuthorDate: Wed Dec 6 16:13:04 2023 +0300
HDDS-9826. Fix handling of GroupMismatchException in
XceiverClientRatis.watchForCommit (#5727)
---
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 5 +++--
.../hadoop/hdds/scm/client/HddsClientUtils.java | 14 ++++++++++++++
.../hadoop/ozone/client/TestHddsClientUtils.java | 20 ++++++++++++++++++++
.../hadoop/ozone/client/rpc/TestWatchForCommit.java | 11 +++++++++++
4 files changed, 48 insertions(+), 2 deletions(-)
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index cdcf2b3941..4fae9ee13c 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -286,9 +286,10 @@ public final class XceiverClientRatis extends
XceiverClientSpi {
Preconditions.checkState(updated >= index);
return newWatchReply(index, ReplicationLevel.ALL_COMMITTED, updated);
} catch (Exception e) {
- Throwable t = HddsClientUtils.checkForException(e);
LOG.warn("3 way commit failed on pipeline {}", pipeline, e);
- if (t instanceof GroupMismatchException) {
+ Throwable t =
+ HddsClientUtils.containsException(e, GroupMismatchException.class);
+ if (t != null) {
throw e;
}
final RaftClientReply reply = getClient().async()
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
index e7528c2d08..cf6e09d95a 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/client/HddsClientUtils.java
@@ -296,6 +296,20 @@ public final class HddsClientUtils {
return t;
}
+ // This will return the underlying expected exception if it exists
+ // in an exception trace. Otherwise, returns null.
+ public static Throwable containsException(Exception e,
+ Class<? extends Exception> expectedExceptionClass) {
+ Throwable t = e;
+ while (t != null) {
+ if (expectedExceptionClass.isInstance(t)) {
+ return t;
+ }
+ t = t.getCause();
+ }
+ return null;
+ }
+
public static RetryPolicy createRetryPolicy(int maxRetryCount,
long retryInterval) {
// retry with fixed sleep between retries
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
index 37ab91edcb..e88a894f5c 100644
---
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestHddsClientUtils.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.ozone.client;
+import java.io.IOException;
+import java.net.ConnectException;
import java.net.InetSocketAddress;
import java.util.ArrayList;
import java.util.Collection;
@@ -46,6 +48,8 @@ import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_PORT_KEY
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertNull;
+import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.fail;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertThrows;
@@ -322,4 +326,20 @@ public class TestHddsClientUtils {
}
}
}
+
+ @Test
+ void testContainsException() {
+ Exception ex1 = new ConnectException();
+ Exception ex2 = new IOException(ex1);
+ Exception ex3 = new IllegalArgumentException(ex2);
+
+ assertSame(ex1,
+ HddsClientUtils.containsException(ex3, ConnectException.class));
+ assertSame(ex2,
+ HddsClientUtils.containsException(ex3, IOException.class));
+ assertSame(ex3,
+ HddsClientUtils.containsException(ex3,
IllegalArgumentException.class));
+ assertNull(
+ HddsClientUtils.containsException(ex3, IllegalStateException.class));
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
index b9fb0d425b..5a46a0edf4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestWatchForCommit.java
@@ -249,6 +249,8 @@ public class TestWatchForCommit {
@Test
public void testWatchForCommitForRetryfailure() throws Exception {
+ GenericTestUtils.LogCapturer logCapturer =
+ GenericTestUtils.LogCapturer.captureLogs(XceiverClientRatis.LOG);
try (XceiverClientManager clientManager = new XceiverClientManager(conf)) {
ContainerWithPipeline container1 = storageContainerLocationClient
.allocateContainer(HddsProtos.ReplicationType.RATIS,
@@ -268,6 +270,9 @@ public class TestWatchForCommit {
long index = reply.getLogIndex();
cluster.shutdownHddsDatanode(pipeline.getNodes().get(0));
cluster.shutdownHddsDatanode(pipeline.getNodes().get(1));
+ // emulate closing pipeline when SCM detects DEAD datanodes
+ cluster.getStorageContainerManager()
+ .getPipelineManager().closePipeline(pipeline, false);
// again write data with more than max buffer limit. This wi
try {
// just watch for a log index which in not updated in the commitInfo
Map
@@ -283,6 +288,12 @@ public class TestWatchForCommit {
// RuntimeException
Assert.assertFalse(HddsClientUtils
.checkForException(e) instanceof TimeoutException);
+ // client should not attempt to watch with
+ // MAJORITY_COMMITTED replication level, except the grpc IO issue
+ if (!logCapturer.getOutput().contains("Connection refused")) {
+ Assert.assertFalse(
+ e.getMessage().contains("Watch-MAJORITY_COMMITTED"));
+ }
}
clientManager.releaseClient(xceiverClient, false);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
