This is an automated email from the ASF dual-hosted git repository.
nanda pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 06957187eb HDDS-9009. Introduce InvocationType for Replicate
annotation in SCM. (#5057)
06957187eb is described below
commit 06957187eb7392d4976d3d47e3f50731305143a5
Author: Nandakumar <[email protected]>
AuthorDate: Fri Jul 21 13:31:09 2023 +0530
HDDS-9009. Introduce InvocationType for Replicate annotation in SCM. (#5057)
---
.../apache/hadoop/hdds/scm/metadata/Replicate.java | 19 +++++++-
.../certificate/authority/CertificateStore.java | 2 +-
.../hadoop/hdds/scm/ha/SCMHAInvocationHandler.java | 55 ++++++++++------------
.../hdds/scm/security/RootCARotationHandler.java | 2 +-
.../hdds/scm/ha/TestReplicationAnnotation.java | 25 ----------
5 files changed, 45 insertions(+), 58 deletions(-)
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/Replicate.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/Replicate.java
index aeed57cd4a..f2b5ea570d 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/Replicate.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/scm/metadata/Replicate.java
@@ -24,10 +24,27 @@ import java.lang.annotation.RetentionPolicy;
import java.lang.annotation.Target;
/**
- * TODO: Add javadoc.
+ * Indicates that a method will be called via Ratis.
+ * If a method is annotated with this annotation type the call made to this
+ * method will go via Ratis.
+ * <ul><li>
+ * If the InvocationMethod is DIRECT, the call will be directly submitted
+ * to Ratis Server. The current instance has to be the leader for this call to
+ * work.
+ * </li><li>
+ * If the InvocationMethod is CLIENT, the call will be submitted to
+ * Ratis Server via Ratis Client. The current instance need not be the leader.
+ * </li></ul>
*/
@Inherited
@Target(ElementType.METHOD)
@Retention(RetentionPolicy.RUNTIME)
public @interface Replicate {
+ /**
+ * For DIRECT, the call will be directly submitted to Ratis Server.
+ * <br>
+ * For CLIENT, the call will be submitted to Ratis Server via Ratis Client.
+ */
+ enum InvocationType { DIRECT, CLIENT }
+ InvocationType invocationType() default InvocationType.DIRECT;
}
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
index 18b572e7ab..9cc5dee141 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/security/x509/certificate/authority/CertificateStore.java
@@ -59,7 +59,7 @@ public interface CertificateStore {
* @param role - OM/DN/SCM.
* @throws IOException - on Failure.
*/
- @Replicate
+ @Replicate(invocationType = Replicate.InvocationType.CLIENT)
void storeValidCertificate(BigInteger serialID,
X509Certificate certificate, NodeType role)
throws IOException;
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
index 32ad2c2adf..fd10c9d33e 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/ha/SCMHAInvocationHandler.java
@@ -24,8 +24,6 @@ import java.lang.reflect.Method;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
-import com.google.common.base.Preconditions;
-import org.apache.hadoop.hdds.protocol.proto.HddsProtos;
import org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
import org.apache.hadoop.hdds.scm.exceptions.SCMException;
import org.apache.hadoop.hdds.scm.exceptions.SCMException.ResultCodes;
@@ -102,48 +100,45 @@ public class SCMHAInvocationHandler implements
InvocationHandler {
*/
private Object invokeRatis(Method method, Object[] args)
throws SCMException {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Invoking method {} on target {}", method, ratisHandler);
+ }
try {
- return invokeRatisImpl(method, args);
+ switch (method.getAnnotation(Replicate.class).invocationType()) {
+ case CLIENT:
+ return invokeRatisClient(method, args);
+ case DIRECT:
+ default:
+ return invokeRatisServer(method, args);
+ }
} catch (Exception e) {
throw translateException(e);
}
}
- private Object invokeRatisImpl(Method method, Object[] args)
+ private Object invokeRatisServer(Method method, Object[] args)
throws Exception {
- if (LOG.isTraceEnabled()) {
- LOG.trace("Invoking method {} on target {}", method, ratisHandler);
- }
- // TODO: Add metric here to track time taken by Ratis
- Preconditions.checkNotNull(ratisHandler);
SCMRatisRequest scmRatisRequest = SCMRatisRequest.of(requestType,
method.getName(), method.getParameterTypes(), args);
-
- // Scm Cert DB updates should use RaftClient.
- // As rootCA which is primary SCM only can issue certificates to sub-CA.
- // In case primary is not leader SCM, still sub-ca cert DB updates should
go
- // via ratis. So, in this special scenario we use RaftClient.
- // Or rotationPrepareAck which every SCM will send out to confirm that
- // sub CA rotation preparation is done.
- final SCMRatisResponse response;
- if ((method.getName().equals("storeValidCertificate") &&
- args[args.length - 1].equals(HddsProtos.NodeType.SCM)) ||
- method.getName().equals("rotationPrepareAck")) {
- response =
- HASecurityUtils.submitScmRequestToRatis(
- ratisHandler.getDivision().getGroup(),
- ratisHandler.getGrpcTlsConfig(),
- scmRatisRequest.encode());
-
- } else {
- response = ratisHandler.submitRequest(
- scmRatisRequest);
+ final SCMRatisResponse response = ratisHandler.submitRequest(
+ scmRatisRequest);
+ if (response.isSuccess()) {
+ return response.getResult();
}
+ throw response.getException();
+ }
+ private Object invokeRatisClient(Method method, Object[] args)
+ throws Exception {
+ final SCMRatisRequest scmRatisRequest = SCMRatisRequest.of(requestType,
+ method.getName(), method.getParameterTypes(), args);
+ final SCMRatisResponse response = HASecurityUtils.submitScmRequestToRatis(
+ ratisHandler.getDivision().getGroup(),
+ ratisHandler.getGrpcTlsConfig(),
+ scmRatisRequest.encode());
if (response.isSuccess()) {
return response.getResult();
}
- // Should we unwrap and throw proper exception from here?
throw response.getException();
}
diff --git
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationHandler.java
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationHandler.java
index a91c90e3df..2b726a319f 100644
---
a/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationHandler.java
+++
b/hadoop-hdds/server-scm/src/main/java/org/apache/hadoop/hdds/scm/security/RootCARotationHandler.java
@@ -37,7 +37,7 @@ public interface RootCARotationHandler {
void rotationPrepare(String rootCertId)
throws IOException;
- @Replicate
+ @Replicate(invocationType = Replicate.InvocationType.CLIENT)
void rotationPrepareAck(String rootCertId, String scmCertId, String scmId)
throws IOException;
diff --git
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestReplicationAnnotation.java
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestReplicationAnnotation.java
index 3049e88662..948d99b56a 100644
---
a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestReplicationAnnotation.java
+++
b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/hdds/scm/ha/TestReplicationAnnotation.java
@@ -22,8 +22,6 @@ import
org.apache.hadoop.hdds.protocol.proto.SCMRatisProtocol.RequestType;
import org.apache.hadoop.hdds.scm.AddSCMRequest;
import org.apache.hadoop.hdds.scm.RemoveSCMRequest;
import org.apache.hadoop.hdds.scm.container.ContainerStateManager;
-import
org.apache.hadoop.hdds.security.x509.certificate.authority.CertificateStore;
-import org.apache.hadoop.security.ssl.KeyStoreTestUtil;
import org.apache.ratis.grpc.GrpcTlsConfig;
import org.apache.ratis.protocol.exceptions.NotLeaderException;
import org.apache.ratis.server.RaftServer;
@@ -33,8 +31,6 @@ import org.junit.jupiter.api.Test;
import java.io.IOException;
import java.lang.reflect.Proxy;
-import java.math.BigInteger;
-import java.security.KeyPair;
import java.util.List;
import java.util.concurrent.ExecutionException;
@@ -134,26 +130,5 @@ public class TestReplicationAnnotation {
Assertions.assertNotNull(e.getMessage());
Assertions.assertTrue(e.getMessage().contains("submitRequest is
called"));
}
-
- scmhaInvocationHandler = new SCMHAInvocationHandler(
- RequestType.CERT_STORE, null, scmRatisServer);
-
- CertificateStore certificateStore =
- (CertificateStore) Proxy.newProxyInstance(
- SCMHAInvocationHandler.class.getClassLoader(),
- new Class<?>[]{CertificateStore.class},
- scmhaInvocationHandler);
-
- KeyPair keyPair = KeyStoreTestUtil.generateKeyPair("RSA");
- try {
- certificateStore.storeValidCertificate(BigInteger.valueOf(100L),
- KeyStoreTestUtil.generateCertificate("CN=Test", keyPair, 30,
- "SHA256withRSA"), HddsProtos.NodeType.DATANODE);
- Assertions.fail("Cannot reach here: should have seen a IOException");
- } catch (IOException e) {
- Assertions.assertNotNull(e.getMessage());
- Assertions.assertTrue(e.getMessage().contains("submitRequest is
called"));
- }
-
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
