This is an automated email from the ASF dual-hosted git repository. bharat pushed a commit to branch HDDS-4017 in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
commit 4501dcc9d692b122a69f0a2fbe886663a8a23dbc Author: Bharat Viswanadham <[email protected]> AuthorDate: Thu Jul 30 15:36:07 2020 -0700 HDDS-4012. Normalize Keypath for lookupKey. --- .../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 40 ++++++++++++++++++++++ .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 11 +++++- .../org/apache/hadoop/ozone/om/OzoneManager.java | 1 - .../hadoop/ozone/om/request/OMClientRequest.java | 2 +- 4 files changed, 51 insertions(+), 3 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java index c4e5435..476c1fc 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java @@ -19,6 +19,7 @@ package org.apache.hadoop.fs.ozone; import org.apache.commons.lang3.RandomStringUtils; +import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; @@ -27,6 +28,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster; import org.apache.hadoop.ozone.TestDataUtil; import org.apache.hadoop.ozone.client.OzoneBucket; import org.apache.hadoop.ozone.client.OzoneVolume; +import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.junit.Assert; @@ -209,6 +211,44 @@ public class TestOzoneFSWithObjectStoreCreate { } + @Test + public void testReadWithNotNormalizedPath() throws Exception { + OzoneVolume ozoneVolume = + cluster.getRpcClient().getObjectStore().getVolume(volumeName); + + OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName); + + String key = "/dir1///dir2/file1/"; + + int length = 10; + byte[] input = new byte[length]; + Arrays.fill(input, (byte)96); + String inputString = new String(input); + + OzoneOutputStream ozoneOutputStream = + ozoneBucket.createKey(key, length); + + ozoneOutputStream.write(input); + ozoneOutputStream.write(input, 0, 10); + ozoneOutputStream.close(); + + // Read the key with given key name. + OzoneInputStream ozoneInputStream = ozoneBucket.readKey(key); + byte[] read = new byte[length]; + ozoneInputStream.read(read, 0, length); + ozoneInputStream.close(); + + Assert.assertEquals(inputString, new String(read)); + + // Read using filesystem. + FSDataInputStream fsDataInputStream = o3fs.open(new Path(key)); + read = new byte[length]; + fsDataInputStream.read(read, 0, length); + ozoneInputStream.close(); + + Assert.assertEquals(inputString, new String(read)); + } + private void checkPath(Path path) { try { o3fs.getFileStatus(path); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index 66ddeb6..4027358 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -94,6 +94,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo; +import org.apache.hadoop.ozone.om.request.OMClientRequest; import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo; import org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager; import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; @@ -165,6 +166,8 @@ public class KeyManagerImpl implements KeyManager { private final KeyProviderCryptoExtension kmsProvider; private final PrefixManager prefixManager; + private final boolean enableFileSystemPaths; + @VisibleForTesting public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient, @@ -208,6 +211,9 @@ public class KeyManagerImpl implements KeyManager { this.listTrashKeysMax = conf.getInt( OZONE_CLIENT_LIST_TRASH_KEYS_MAX, OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT); + this.enableFileSystemPaths = + conf.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, + OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT); this.ozoneManager = om; this.omId = omId; @@ -645,7 +651,10 @@ public class KeyManagerImpl implements KeyManager { Preconditions.checkNotNull(args); String volumeName = args.getVolumeName(); String bucketName = args.getBucketName(); - String keyName = args.getKeyName(); + String keyName = OMClientRequest.validateAndNormalizeKey( + enableFileSystemPaths, args.getKeyName()); + + metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName, bucketName); OmKeyInfo value = null; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 13e47a7..f70bf33 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -2226,7 +2226,6 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl Map<String, String> auditMap = bucket.audit(args.toAuditMap()); args = bucket.update(args); - try { metrics.incNumKeyLookups(); return keyManager.lookupKey(args, getClientAddress()); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java index 0fa9ca1..f0b8cf5 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/OMClientRequest.java @@ -295,7 +295,7 @@ public abstract class OMClientRequest implements RequestAuditor { .normalize().getPath(); } if (!keyName.equals(normalizedKeyName)) { - LOG.debug("Normalized key {} to {} ", keyName, + LOG.info("Normalized key {} to {} ", keyName, normalizedKeyName.substring(1)); } return isValidKeyPath(normalizedKeyName.substring(1)); --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
