This is an automated email from the ASF dual-hosted git repository. elek pushed a commit to branch HDDS-2181 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 6828f2e3aa7e8a937544e0b70ec844062376f996 Merge: ac4990f 640255a Author: Vivek Ratnavel Subramanian <vivekratnave...@gmail.com> AuthorDate: Thu Oct 10 19:23:16 2019 -0700 Merge remote-tracking branch 'upstream/trunk' into HDDS-2181 .../hadoop/hdds/scm/XceiverClientManager.java | 7 + .../hadoop/hdds/scm/storage/BlockOutputStream.java | 3 +- .../apache/hadoop/hdds/scm/storage/BufferPool.java | 15 + .../hadoop/hdds/scm/ByteStringConversion.java | 62 +++ .../apache/hadoop/hdds/scm/ByteStringHelper.java | 69 ---- .../apache/hadoop/hdds/scm/pipeline/Pipeline.java | 3 +- .../hadoop/hdds/utils/db/cache/CacheKey.java | 11 +- .../hadoop/hdds/utils/db/cache/TableCacheImpl.java | 12 +- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 3 + .../org/apache/hadoop/ozone/lock/ActiveLock.java | 11 +- .../org/apache/hadoop/ozone/lock/LockManager.java | 19 +- .../hadoop/ozone/lock/PooledLockFactory.java | 7 +- .../common/src/main/resources/ozone-default.xml | 11 + .../ozone/container/keyvalue/KeyValueHandler.java | 33 +- .../container/keyvalue/helpers/ChunkUtils.java | 34 +- .../keyvalue/impl/ChunkManagerDummyImpl.java | 6 +- .../container/keyvalue/impl/ChunkManagerImpl.java | 60 ++- .../keyvalue/interfaces/ChunkManager.java | 2 +- .../container/keyvalue/TestChunkManagerImpl.java | 69 ++-- .../client/io/BlockOutputStreamEntryPool.java | 22 +- .../hadoop/ozone/client/io/KeyInputStream.java | 6 +- .../apache/hadoop/ozone/client/rpc/RpcClient.java | 15 +- .../hadoop/ozone/om/S3SecretManagerImpl.java | 4 +- .../ozone/om/ha/OMFailoverProxyProvider.java | 6 +- .../hadoop/ozone/om/helpers/OMRatisHelper.java | 4 +- .../hadoop/ozone/om/lock/OzoneManagerLock.java | 31 +- .../security/OzoneBlockTokenSecretManager.java | 2 +- .../OzoneDelegationTokenSecretManager.java | 6 +- .../security/OzoneDelegationTokenSelector.java | 8 +- .../hadoop/ozone/security/OzoneSecretManager.java | 6 +- .../dev-support/checks/_mvn_unit_report.sh | 5 + .../dist/src/main/compose/ozone-hdfs/docker-config | 46 --- .../dist/src/main/compose/ozone-mr/common-config | 9 - .../src/main/compose/ozone-om-ha/docker-config | 45 --- .../src/main/compose/ozone-recon/docker-config | 47 +-- .../src/main/compose/ozone-topology/docker-config | 49 --- .../dist/src/main/compose/ozone/docker-config | 45 --- .../src/main/compose/ozoneblockade/docker-config | 45 --- .../dist/src/main/compose/ozoneperf/docker-config | 13 - .../src/main/compose/ozones3-haproxy/docker-config | 48 --- .../dist/src/main/compose/ozones3/docker-config | 48 --- .../src/main/compose/ozonescripts/docker-config | 7 +- .../src/main/compose/ozonesecure-mr/docker-config | 46 --- .../src/main/compose/ozonesecure/docker-config | 53 --- .../ozone/container/ContainerTestHelper.java | 11 +- .../common/impl/TestContainerPersistence.java | 53 +-- .../apache/hadoop/ozone/om/BucketManagerImpl.java | 6 +- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 27 +- .../hadoop/ozone/om/OmMetadataManagerImpl.java | 125 ++++-- .../hadoop/ozone/om/OpenKeyCleanupService.java | 4 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 10 +- .../apache/hadoop/ozone/om/PrefixManagerImpl.java | 11 +- .../apache/hadoop/ozone/om/VolumeManagerImpl.java | 16 +- .../ozone/om/ratis/OzoneManagerDoubleBuffer.java | 8 +- .../ozone/om/ratis/OzoneManagerRatisClient.java | 53 +-- .../ozone/om/ratis/OzoneManagerRatisServer.java | 6 +- .../request/bucket/acl/OMBucketSetAclRequest.java | 4 +- .../request/volume/acl/OMVolumeSetAclRequest.java | 6 +- .../OzoneManagerHARequestHandlerImpl.java | 4 +- ...OzoneManagerProtocolServerSideTranslatorPB.java | 4 +- .../protocolPB/OzoneManagerRequestHandler.java | 4 +- .../ozone/security/acl/OzoneNativeAuthorizer.java | 8 +- .../hadoop/ozone/om/TestOmMetadataManager.java | 417 +++++++++++++++++++++ .../ozone/om/request/TestOMRequestUtils.java | 60 ++- .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 +- .../apache/hadoop/ozone/s3/AWSV4AuthParser.java | 10 +- .../hadoop/ozone/s3/OzoneClientProducer.java | 5 +- .../ozone/s3/exception/OS3ExceptionMapper.java | 4 +- 68 files changed, 1040 insertions(+), 873 deletions(-) diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 19976e5,20b7fdf..b451722 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@@ -1670,11 -1674,8 +1670,13 @@@ public class KeyManagerImpl implements } if (keyInfo == null) { - throw new OMException("Key not found, checkAccess failed. Key:" + - objectKey, KEY_NOT_FOUND); + // the key does not exist, but it is a parent "dir" of some key + // let access be determined based on volume/bucket/prefix ACL - LOG.debug("key:{} is non-existent parent, permit access to user:{}", - keyName, context.getClientUgi()); ++ if (LOG.isDebugEnabled()) { ++ LOG.debug("key:{} is non-existent parent, permit access to user:{}", ++ keyName, context.getClientUgi()); ++ } + return true; } boolean hasAccess = OzoneAclUtil.checkAclRight( diff --cc hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java index d974537,0b7c51a..442dc59 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/security/acl/OzoneNativeAuthorizer.java @@@ -80,53 -77,25 +80,53 @@@ public class OzoneNativeAuthorizer impl "configured to work with OzoneObjInfo type only.", INVALID_REQUEST); } + // For CREATE and DELETE acl requests, the parents need to be checked + // for WRITE acl. If Key create request is received, then we need to + // check if user has WRITE acl set on Bucket and Volume. In all other cases + // the parents also need to be checked for the same acl type. + if (isACLTypeCreate || isACLTypeDelete) { + parentContext = RequestContext.newBuilder() + .setClientUgi(context.getClientUgi()) + .setIp(context.getIp()) + .setAclType(context.getAclType()) + .setAclRights(ACLType.WRITE) + .build(); + } else { + parentContext = context; + } + switch (objInfo.getResourceType()) { case VOLUME: - LOG.trace("Checking access for volume:" + objInfo); + LOG.trace("Checking access for volume: {}", objInfo); return volumeManager.checkAccess(objInfo, context); case BUCKET: - LOG.trace("Checking access for bucket:" + objInfo); + LOG.trace("Checking access for bucket: {}", objInfo); - return (bucketManager.checkAccess(objInfo, context) - && volumeManager.checkAccess(objInfo, context)); + // Skip bucket access check for CREATE acl since + // bucket will not exist at the time of creation + boolean bucketAccess = isACLTypeCreate + || bucketManager.checkAccess(objInfo, context); + return (bucketAccess + && volumeManager.checkAccess(objInfo, parentContext)); case KEY: + case OPEN_KEY: - LOG.trace("Checking access for Key:" + objInfo); + LOG.trace("Checking access for Key: {}", objInfo); - return (keyManager.checkAccess(objInfo, context) - && prefixManager.checkAccess(objInfo, context) - && bucketManager.checkAccess(objInfo, context) - && volumeManager.checkAccess(objInfo, context)); + // Skip key access check for CREATE acl since + // key will not exist at the time of creation + boolean keyAccess = isACLTypeCreate + || keyManager.checkAccess(objInfo, context); + return (keyAccess + && prefixManager.checkAccess(objInfo, parentContext) + && bucketManager.checkAccess(objInfo, parentContext) + && volumeManager.checkAccess(objInfo, parentContext)); case PREFIX: - LOG.trace("Checking access for Prefix:" + objInfo); - LOG.trace("Checking access for Prefix: {]", objInfo); - return (prefixManager.checkAccess(objInfo, context) - && bucketManager.checkAccess(objInfo, context) - && volumeManager.checkAccess(objInfo, context)); ++ LOG.trace("Checking access for Prefix: {}", objInfo); + // Skip prefix access check for CREATE acl since + // prefix will not exist at the time of creation + boolean prefixAccess = isACLTypeCreate + || prefixManager.checkAccess(objInfo, context); + return (prefixAccess + && bucketManager.checkAccess(objInfo, parentContext) + && volumeManager.checkAccess(objInfo, parentContext)); default: throw new OMException("Unexpected object type:" + objInfo.getResourceType(), INVALID_REQUEST); --------------------------------------------------------------------- To unsubscribe, e-mail: hdfs-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: hdfs-commits-h...@hadoop.apache.org