This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b9b621d HDDS-3715. Improvement for OzoneFS client to work with Hadoop
2.7.3. (#1036)
b9b621d is described below
commit b9b621d138fa40afa368d180e0aaa529d78961bc
Author: Sammi Chen <[email protected]>
AuthorDate: Wed Jun 10 18:19:57 2020 +0800
HDDS-3715. Improvement for OzoneFS client to work with Hadoop 2.7.3. (#1036)
---
.../apache/hadoop/hdds/scm/XceiverClientGrpc.java | 7 +-
.../apache/hadoop/hdds/scm/XceiverClientRatis.java | 5 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +-
.../hadoop/ozone/om/helpers/OzoneAclUtil.java | 5 +-
.../apache/hadoop/ozone/om/TestKeyManagerImpl.java | 4 +-
.../security/acl/TestOzoneNativeAuthorizer.java | 2 +-
.../fs/ozone/BasicOzoneClientAdapterImpl.java | 122 +++++++++------------
hadoop-ozone/ozonefs-hadoop2/pom.xml | 2 +-
.../hadoop/fs/ozone/Hadoop27RpcTransport.java | 7 +-
.../apache/hadoop/ozone/freon/OmKeyGenerator.java | 2 +-
10 files changed, 71 insertions(+), 87 deletions(-)
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
index ff83dff..28d4491 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientGrpc.java
@@ -51,7 +51,6 @@ import org.apache.hadoop.hdds.tracing.GrpcClientInterceptor;
import org.apache.hadoop.hdds.tracing.TracingUtil;
import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
-import org.apache.hadoop.util.Time;
import java.util.concurrent.TimeoutException;
import com.google.common.annotations.VisibleForTesting;
@@ -434,7 +433,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
final CompletableFuture<ContainerCommandResponseProto> replyFuture =
new CompletableFuture<>();
semaphore.acquire();
- long requestTime = Time.monotonicNowNanos();
+ long requestTime = System.nanoTime();
metrics.incrPendingContainerOpsMetrics(request.getCmdType());
// create a new grpc stream for each non-async call.
@@ -449,7 +448,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
replyFuture.complete(value);
metrics.decrPendingContainerOpsMetrics(request.getCmdType());
metrics.addContainerOpsLatency(request.getCmdType(),
- Time.monotonicNowNanos() - requestTime);
+ System.nanoTime() - requestTime);
semaphore.release();
}
@@ -458,7 +457,7 @@ public class XceiverClientGrpc extends XceiverClientSpi {
replyFuture.completeExceptionally(t);
metrics.decrPendingContainerOpsMetrics(request.getCmdType());
metrics.addContainerOpsLatency(request.getCmdType(),
- Time.monotonicNowNanos() - requestTime);
+ System.nanoTime() - requestTime);
semaphore.release();
}
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
index 7ff5ab1..2fa3342 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/XceiverClientRatis.java
@@ -47,7 +47,6 @@ import org.apache.hadoop.hdds.scm.client.HddsClientUtils;
import org.apache.hadoop.hdds.scm.pipeline.Pipeline;
import org.apache.hadoop.hdds.security.x509.SecurityConfig;
import org.apache.hadoop.hdds.tracing.TracingUtil;
-import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
@@ -301,7 +300,7 @@ public final class XceiverClientRatis extends
XceiverClientSpi {
public XceiverClientReply sendCommandAsync(
ContainerCommandRequestProto request) {
XceiverClientReply asyncReply = new XceiverClientReply(null);
- long requestTime = Time.monotonicNowNanos();
+ long requestTime = System.nanoTime();
CompletableFuture<RaftClientReply> raftClientReply =
sendRequestAsync(request);
metrics.incrPendingContainerOpsMetrics(request.getCmdType());
@@ -315,7 +314,7 @@ public final class XceiverClientRatis extends
XceiverClientSpi {
}
metrics.decrPendingContainerOpsMetrics(request.getCmdType());
metrics.addContainerOpsLatency(request.getCmdType(),
- Time.monotonicNowNanos() - requestTime);
+ System.nanoTime() - requestTime);
}).thenApply(reply -> {
try {
if (!reply.isSuccess()) {
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 4a2d713..8437699 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -477,7 +477,7 @@ public class RpcClient implements ClientProtocol {
* @return listOfAcls
* */
private List<OzoneAcl> getAclList() {
- return OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(),
+ return OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroupNames(),
userRights, groupRights);
}
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java
index fd42fea..d6e1c04 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneAclUtil.java
@@ -26,6 +26,7 @@ import
org.apache.hadoop.ozone.security.acl.IAccessAuthorizer.ACLType;
import org.apache.hadoop.ozone.security.acl.RequestContext;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.BitSet;
import java.util.List;
import java.util.stream.Collectors;
@@ -53,7 +54,7 @@ public final class OzoneAclUtil {
* @return list of OzoneAcls
* */
public static List<OzoneAcl> getAclList(String userName,
- List<String> userGroups, ACLType userRights, ACLType groupRights) {
+ String[] userGroups, ACLType userRights, ACLType groupRights) {
List<OzoneAcl> listOfAcls = new ArrayList<>();
@@ -61,7 +62,7 @@ public final class OzoneAclUtil {
listOfAcls.add(new OzoneAcl(USER, userName, userRights, ACCESS));
if(userGroups != null) {
// Group ACLs of the User.
- userGroups.forEach((group) -> listOfAcls.add(
+ Arrays.asList(userGroups).forEach((group) -> listOfAcls.add(
new OzoneAcl(GROUP, group, groupRights, ACCESS)));
}
return listOfAcls;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
index e108203..7a1cb5b 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestKeyManagerImpl.java
@@ -279,7 +279,7 @@ public class TestKeyManagerImpl {
OmKeyArgs keyArgs = createBuilder()
.setKeyName(KEY_NAME)
.setDataSize(1000)
- .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(),
+ .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(),
ugi.getGroupNames(),
ALL, ALL))
.build();
LambdaTestUtils.intercept(OMException.class,
@@ -1363,7 +1363,7 @@ public class TestKeyManagerImpl {
.setFactor(ReplicationFactor.ONE)
.setDataSize(0)
.setType(ReplicationType.STAND_ALONE)
- .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(),
+ .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(),
ugi.getGroupNames(),
ALL, ALL))
.setVolumeName(VOLUME_NAME);
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
index ac1ad20..219ec97 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/security/acl/TestOzoneNativeAuthorizer.java
@@ -176,7 +176,7 @@ public class TestOzoneNativeAuthorizer {
.setDataSize(0)
.setType(HddsProtos.ReplicationType.STAND_ALONE)
.setAcls(OzoneAclUtil.getAclList(testUgi.getUserName(),
- testUgi.getGroups(), ALL, ALL))
+ testUgi.getGroupNames(), ALL, ALL))
.build();
if (keyName.split(OZONE_URI_DELIMITER).length > 1) {
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
index f388107..bffd679 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java
@@ -92,18 +92,7 @@ public class BasicOzoneClientAdapterImpl implements
OzoneClientAdapter {
*/
public BasicOzoneClientAdapterImpl(String volumeStr, String bucketStr)
throws IOException {
- this(createConf(), volumeStr, bucketStr);
- }
-
- private static OzoneConfiguration createConf() {
- ClassLoader contextClassLoader =
- Thread.currentThread().getContextClassLoader();
- Thread.currentThread().setContextClassLoader(null);
- try {
- return new OzoneConfiguration();
- } finally {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
- }
+ this(new OzoneConfiguration(), volumeStr, bucketStr);
}
public BasicOzoneClientAdapterImpl(OzoneConfiguration conf, String volumeStr,
@@ -116,72 +105,63 @@ public class BasicOzoneClientAdapterImpl implements
OzoneClientAdapter {
ConfigurationSource hadoopConf, String volumeStr, String bucketStr)
throws IOException {
- ClassLoader contextClassLoader =
- Thread.currentThread().getContextClassLoader();
- Thread.currentThread().setContextClassLoader(null);
-
- try {
- OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
- if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
- // When the host name or service id isn't given
- // but ozone.om.service.ids is defined, declare failure.
-
- // This is a safety precaution that prevents the client from
- // accidentally failing over to an unintended OM.
- throw new IllegalArgumentException("Service ID or host name must not"
- + " be omitted when ozone.om.service.ids is defined.");
- }
-
- if (omPort != -1) {
- // When the port number is specified, perform the following check
- if (OmUtils.isOmHAServiceId(conf, omHost)) {
- // If omHost is a service id, it shouldn't use a port
- throw new IllegalArgumentException("Port " + omPort +
- " specified in URI but host '" + omHost + "' is a "
- + "logical (HA) OzoneManager and does not use port
information.");
- }
- } else {
- // When port number is not specified, read it from config
- omPort = OmUtils.getOmRpcPort(conf);
- }
+ OzoneConfiguration conf = OzoneConfiguration.of(hadoopConf);
+ if (omHost == null && OmUtils.isServiceIdsDefined(conf)) {
+ // When the host name or service id isn't given
+ // but ozone.om.service.ids is defined, declare failure.
- SecurityConfig secConfig = new SecurityConfig(conf);
+ // This is a safety precaution that prevents the client from
+ // accidentally failing over to an unintended OM.
+ throw new IllegalArgumentException("Service ID or host name must not"
+ + " be omitted when ozone.om.service.ids is defined.");
+ }
- if (secConfig.isSecurityEnabled()) {
- this.securityEnabled = true;
+ if (omPort != -1) {
+ // When the port number is specified, perform the following check
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // If omHost is a service id, it shouldn't use a port
+ throw new IllegalArgumentException("Port " + omPort +
+ " specified in URI but host '" + omHost + "' is a "
+ + "logical (HA) OzoneManager and does not use port information.");
}
+ } else {
+ // When port number is not specified, read it from config
+ omPort = OmUtils.getOmRpcPort(conf);
+ }
- String replicationTypeConf =
- conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
- OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
-
- int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
- OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+ SecurityConfig secConfig = new SecurityConfig(conf);
- if (OmUtils.isOmHAServiceId(conf, omHost)) {
- // omHost is listed as one of the service ids in the config,
- // thus we should treat omHost as omServiceId
- this.ozoneClient =
- OzoneClientFactory.getRpcClient(omHost, conf);
- } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
- this.ozoneClient =
- OzoneClientFactory.getRpcClient(omHost, omPort, conf);
- } else {
- this.ozoneClient =
- OzoneClientFactory.getRpcClient(conf);
- }
- objectStore = ozoneClient.getObjectStore();
- this.volume = objectStore.getVolume(volumeStr);
- this.bucket = volume.getBucket(bucketStr);
- this.replicationType = ReplicationType.valueOf(replicationTypeConf);
- this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
- this.configuredDnPort = conf.getInt(
- OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
- OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
- } finally {
- Thread.currentThread().setContextClassLoader(contextClassLoader);
+ if (secConfig.isSecurityEnabled()) {
+ this.securityEnabled = true;
}
+ String replicationTypeConf =
+ conf.get(OzoneConfigKeys.OZONE_REPLICATION_TYPE,
+ OzoneConfigKeys.OZONE_REPLICATION_TYPE_DEFAULT);
+
+ int replicationCountConf = conf.getInt(OzoneConfigKeys.OZONE_REPLICATION,
+ OzoneConfigKeys.OZONE_REPLICATION_DEFAULT);
+
+ if (OmUtils.isOmHAServiceId(conf, omHost)) {
+ // omHost is listed as one of the service ids in the config,
+ // thus we should treat omHost as omServiceId
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, conf);
+ } else if (StringUtils.isNotEmpty(omHost) && omPort != -1) {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(omHost, omPort, conf);
+ } else {
+ this.ozoneClient =
+ OzoneClientFactory.getRpcClient(conf);
+ }
+ objectStore = ozoneClient.getObjectStore();
+ this.volume = objectStore.getVolume(volumeStr);
+ this.bucket = volume.getBucket(bucketStr);
+ this.replicationType = ReplicationType.valueOf(replicationTypeConf);
+ this.replicationFactor = ReplicationFactor.valueOf(replicationCountConf);
+ this.configuredDnPort = conf.getInt(
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT,
+ OzoneConfigKeys.DFS_CONTAINER_IPC_PORT_DEFAULT);
}
@Override
diff --git a/hadoop-ozone/ozonefs-hadoop2/pom.xml
b/hadoop-ozone/ozonefs-hadoop2/pom.xml
index 06ea2e2..8fcfec2 100644
--- a/hadoop-ozone/ozonefs-hadoop2/pom.xml
+++ b/hadoop-ozone/ozonefs-hadoop2/pom.xml
@@ -51,7 +51,7 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <version>2.7.7</version>
+ <version>2.7.3</version>
</dependency>
<dependency>
<groupId>log4j</groupId>
diff --git
a/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/Hadoop27RpcTransport.java
b/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/Hadoop27RpcTransport.java
index fc1c97f..76bd76c 100644
---
a/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/Hadoop27RpcTransport.java
+++
b/hadoop-ozone/ozonefs-hadoop2/src/main/java/org/apache/hadoop/fs/ozone/Hadoop27RpcTransport.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs.ozone;
import java.io.IOException;
import java.net.InetSocketAddress;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.hdds.conf.ConfigurationSource;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.apache.hadoop.io.Text;
@@ -58,7 +59,7 @@ public class Hadoop27RpcTransport implements OmTransport {
socket, UserGroupInformation.getCurrentUser(),
ozoneConfiguration,
NetUtils.getDefaultSocketFactory(ozoneConfiguration),
- RPC.getRpcTimeout(ozoneConfiguration), null).getProxy();
+ getRpcTimeout(ozoneConfiguration), null).getProxy();
}
@Override
@@ -79,4 +80,8 @@ public class Hadoop27RpcTransport implements OmTransport {
public void close() throws IOException {
}
+ private int getRpcTimeout(OzoneConfiguration conf) {
+ return conf.getInt(CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_KEY,
+ CommonConfigurationKeys.IPC_CLIENT_RPC_TIMEOUT_DEFAULT);
+ }
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
index 8c37659..9cb22f6 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/freon/OmKeyGenerator.java
@@ -111,7 +111,7 @@ public class OmKeyGenerator extends BaseFreonGenerator
.setFactor(factor)
.setKeyName(generateObjectName(counter))
.setLocationInfoList(new ArrayList<>())
- .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(), ugi.getGroups(),
+ .setAcls(OzoneAclUtil.getAclList(ugi.getUserName(),
ugi.getGroupNames(),
ALL, ALL))
.build();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
