This is an automated email from the ASF dual-hosted git repository.
elek pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hadoop-ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 004dd3f HDDS-4102. Normalize Keypath for lookupKey. (#1328)
004dd3f is described below
commit 004dd3ff9d35c0d4b6bcb95370e1bd95b0569008
Author: Bharat Viswanadham <[email protected]>
AuthorDate: Mon Sep 28 11:29:04 2020 -0700
HDDS-4102. Normalize Keypath for lookupKey. (#1328)
---
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 40 ++++++++++++++++++++++
.../hadoop/fs/ozone/TestOzoneFileSystem.java | 6 ++--
.../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 9 ++++-
3 files changed, 51 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
index f288973..e89d1c4 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.ozone.MiniOzoneCluster;
import org.apache.hadoop.ozone.TestDataUtil;
import org.apache.hadoop.ozone.client.OzoneBucket;
import org.apache.hadoop.ozone.client.OzoneVolume;
+import org.apache.hadoop.ozone.client.io.OzoneInputStream;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.exceptions.OMException;
@@ -329,6 +330,45 @@ public class TestOzoneFSWithObjectStoreCreate {
}
}
+
+ @Test
+ public void testReadWithNotNormalizedPath() throws Exception {
+ OzoneVolume ozoneVolume =
+ cluster.getRpcClient().getObjectStore().getVolume(volumeName);
+
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName);
+
+ String key = "/dir1///dir2/file1/";
+
+ int length = 10;
+ byte[] input = new byte[length];
+ Arrays.fill(input, (byte)96);
+ String inputString = new String(input);
+
+ OzoneOutputStream ozoneOutputStream =
+ ozoneBucket.createKey(key, length);
+
+ ozoneOutputStream.write(input);
+ ozoneOutputStream.write(input, 0, 10);
+ ozoneOutputStream.close();
+
+ // Read the key with given key name.
+ OzoneInputStream ozoneInputStream = ozoneBucket.readKey(key);
+ byte[] read = new byte[length];
+ ozoneInputStream.read(read, 0, length);
+ ozoneInputStream.close();
+
+ Assert.assertEquals(inputString, new String(read));
+
+ // Read using filesystem.
+ FSDataInputStream fsDataInputStream = o3fs.open(new Path(key));
+ read = new byte[length];
+ fsDataInputStream.read(read, 0, length);
+ ozoneInputStream.close();
+
+ Assert.assertEquals(inputString, new String(read));
+ }
+
private void checkPath(Path path) {
try {
o3fs.getFileStatus(path);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
index 4e728f7..46c0115 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java
@@ -326,9 +326,9 @@ public class TestOzoneFileSystem {
// Deleting the only child should create the parent dir key if it does
// not exist
- String parentKey = o3fs.pathToKey(parent) + "/";
- OzoneKeyDetails parentKeyInfo = getKey(parent, true);
- assertEquals(parentKey, parentKeyInfo.getName());
+ FileStatus fileStatus = o3fs.getFileStatus(parent);
+ Assert.assertTrue(fileStatus.isDirectory());
+ assertEquals(parent.toString(), fileStatus.getPath().toUri().getPath());
}
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
index 14fc07e..ced055c 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java
@@ -94,6 +94,7 @@ import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil;
import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus;
import org.apache.hadoop.ozone.om.helpers.RepeatedOmKeyInfo;
+import org.apache.hadoop.ozone.om.request.OMClientRequest;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.PartKeyInfo;
import org.apache.hadoop.ozone.security.OzoneBlockTokenSecretManager;
import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer;
@@ -165,6 +166,8 @@ public class KeyManagerImpl implements KeyManager {
private final KeyProviderCryptoExtension kmsProvider;
private final PrefixManager prefixManager;
+ private final boolean enableFileSystemPaths;
+
@VisibleForTesting
public KeyManagerImpl(ScmBlockLocationProtocol scmBlockClient,
@@ -208,6 +211,9 @@ public class KeyManagerImpl implements KeyManager {
this.listTrashKeysMax = conf.getInt(
OZONE_CLIENT_LIST_TRASH_KEYS_MAX,
OZONE_CLIENT_LIST_TRASH_KEYS_MAX_DEFAULT);
+ this.enableFileSystemPaths =
+ conf.getBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
+ OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS_DEFAULT);
this.ozoneManager = om;
this.omId = omId;
@@ -645,7 +651,8 @@ public class KeyManagerImpl implements KeyManager {
Preconditions.checkNotNull(args);
String volumeName = args.getVolumeName();
String bucketName = args.getBucketName();
- String keyName = args.getKeyName();
+ String keyName = OMClientRequest.validateAndNormalizeKey(
+ enableFileSystemPaths, args.getKeyName());
metadataManager.getLock().acquireReadLock(BUCKET_LOCK, volumeName,
bucketName);
OmKeyInfo value = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
