This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch HDDS-4440-s3-performance
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/HDDS-4440-s3-performance by
this push:
new edec164302 HDDS-6643. Minor fixes for OM gRPC feature (#3347)
edec164302 is described below
commit edec1643028f5cc6c6fba7fefd8d0eeb2b5dd085
Author: Neil Joshi <[email protected]>
AuthorDate: Tue Apr 26 00:57:16 2022 -0600
HDDS-6643. Minor fixes for OM gRPC feature (#3347)
* Remove duplicate ozone.om.grpc config
* Apply max. response length
* Restore proto optionality
* Improve unit test
---
.../ozone/om/protocolPB/GrpcOmTransport.java | 5 ++-
.../hadoop/ozone/TestOzoneConfigurationFields.java | 1 -
.../src/main/proto/OmClientProtocol.proto | 6 ++--
.../hadoop/ozone/om/GrpcOzoneManagerServer.java | 36 +++++-----------------
.../ozone/om/TestGrpcOzoneManagerServer.java | 2 --
.../ozone/protocolPB/TestGrpcOmTransport.java | 2 --
6 files changed, 13 insertions(+), 39 deletions(-)
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
index 6c6aa9e6e5..d32926a88f 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/protocolPB/GrpcOmTransport.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.retry.RetryPolicy;
import org.apache.hadoop.ozone.OzoneConfigKeys;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.exceptions.OMException;
import org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
@@ -84,7 +83,7 @@ public class GrpcOmTransport implements OmTransport {
private ConfigurationSource conf;
private AtomicReference<String> host;
- private int maxSize;
+ private final int maxSize;
private SecurityConfig secConfig;
public static void setCaCerts(List<X509Certificate> x509Certificates) {
@@ -137,7 +136,7 @@ public class GrpcOmTransport implements OmTransport {
NettyChannelBuilder channelBuilder =
NettyChannelBuilder.forAddress(hp.getHost(), hp.getPort())
.usePlaintext()
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE);
+ .maxInboundMessageSize(maxSize);
if (secConfig.isGrpcTlsEnabled()) {
try {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
index a40461dcb3..4d8151f780 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java
@@ -48,7 +48,6 @@ public class TestOzoneConfigurationFields extends
TestConfigurationFieldsBase {
configurationClasses =
new Class[] {OzoneConfigKeys.class, ScmConfigKeys.class,
OMConfigKeys.class, HddsConfigKeys.class,
- ReconServerConfigKeys.class,
ReconConfigKeys.class, ReconServerConfigKeys.class,
SCMHTTPServerConfig.class,
SCMHTTPServerConfig.ConfigStrings.class,
diff --git
a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
index b7b1f38f18..1bad21582f 100644
--- a/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
+++ b/hadoop-ozone/interface-client/src/main/proto/OmClientProtocol.proto
@@ -1360,9 +1360,9 @@ message UpdateGetS3SecretRequest {
This will be used by OM to authenticate S3 gateway requests on a per request
basis.
*/
message S3Authentication {
- required string stringToSign = 1;
- required string signature = 2;
- required string accessId = 3;
+ optional string stringToSign = 1;
+ optional string signature = 2;
+ optional string accessId = 3;
}
/**
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
index b083378fab..2231c28c85 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/GrpcOzoneManagerServer.java
@@ -22,13 +22,10 @@ import java.util.OptionalInt;
import java.util.concurrent.TimeUnit;
import org.apache.hadoop.hdds.HddsUtils;
-import org.apache.hadoop.hdds.conf.Config;
-import org.apache.hadoop.hdds.conf.ConfigGroup;
-import org.apache.hadoop.hdds.conf.ConfigTag;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
-import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.ha.ConfUtils;
import
org.apache.hadoop.ozone.protocolPB.OzoneManagerProtocolServerSideTranslatorPB;
+import org.apache.hadoop.ozone.om.protocolPB.GrpcOmTransport;
import org.apache.hadoop.ozone.security.OzoneDelegationTokenSecretManager;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import
org.apache.hadoop.hdds.security.x509.certificate.client.CertificateClient;
@@ -46,6 +43,8 @@ import org.slf4j.LoggerFactory;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_GRPC_TLS_PROVIDER;
import static org.apache.hadoop.hdds.HddsConfigKeys
.HDDS_GRPC_TLS_PROVIDER_DEFAULT;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH_DEFAULT;
/**
* Separated network server for gRPC transport OzoneManagerService s3g->OM.
@@ -56,6 +55,7 @@ public class GrpcOzoneManagerServer {
private Server server;
private int port = 8981;
+ private final int maxSize;
public GrpcOzoneManagerServer(OzoneConfiguration config,
OzoneManagerProtocolServerSideTranslatorPB
@@ -63,6 +63,8 @@ public class GrpcOzoneManagerServer {
OzoneDelegationTokenSecretManager
delegationTokenMgr,
CertificateClient caClient) {
+ maxSize = config.getInt(OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH,
+ OZONE_OM_GRPC_MAXIMUM_RESPONSE_LENGTH_DEFAULT);
OptionalInt haPort = HddsUtils.getNumberFromConfigKeys(config,
ConfUtils.addKeySuffixes(
OMConfigKeys.OZONE_OM_GRPC_PORT_KEY,
@@ -73,7 +75,7 @@ public class GrpcOzoneManagerServer {
this.port = haPort.getAsInt();
} else {
this.port = config.getObject(
- GrpcOzoneManagerServerConfig.class).
+ GrpcOmTransport.GrpcOmTransportConfig.class).
getPort();
}
@@ -88,7 +90,7 @@ public class GrpcOzoneManagerServer {
OzoneConfiguration omServerConfig,
CertificateClient caClient) {
NettyServerBuilder nettyServerBuilder = NettyServerBuilder.forPort(port)
- .maxInboundMessageSize(OzoneConsts.OZONE_SCM_CHUNK_MAX_SIZE)
+ .maxInboundMessageSize(maxSize)
.addService(new OzoneManagerServiceGrpc(omTranslator,
delegationTokenMgr,
omServerConfig));
@@ -132,29 +134,7 @@ public class GrpcOzoneManagerServer {
LOG.warn("{} couldn't be stopped gracefully",
getClass().getSimpleName());
}
}
-
public int getPort() {
return port;
}
-
- /**
- * GrpcOzoneManagerServer configuration in Java style configuration class.
- */
- @ConfigGroup(prefix = "ozone.om.grpc")
- public static final class GrpcOzoneManagerServerConfig {
- @Config(key = "port", defaultValue = "8981",
- description = "Port used for"
- + " the GrpcOmTransport OzoneManagerServiceGrpc server",
- tags = {ConfigTag.MANAGEMENT})
- private int port;
-
- public int getPort() {
- return port;
- }
-
- public GrpcOzoneManagerServerConfig setPort(int portParam) {
- this.port = portParam;
- return this;
- }
- }
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestGrpcOzoneManagerServer.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestGrpcOzoneManagerServer.java
index e58b7a47ea..687ed21b81 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestGrpcOzoneManagerServer.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/TestGrpcOzoneManagerServer.java
@@ -54,8 +54,6 @@ public class TestGrpcOzoneManagerServer {
try {
server.start();
- } catch (Exception e) {
- e.printStackTrace();
} finally {
server.stop();
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/protocolPB/TestGrpcOmTransport.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/protocolPB/TestGrpcOmTransport.java
index a28f47a809..304a2bfb86 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/protocolPB/TestGrpcOmTransport.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/protocolPB/TestGrpcOmTransport.java
@@ -81,8 +81,6 @@ public class TestGrpcOmTransport {
try {
client.start();
- } catch (Exception e) {
- e.printStackTrace();
} finally {
client.shutdown();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
