This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 0a558c15e0 HDDS-12758. Error in OmUtils.normalizeKey for key name
starting with `//` (#8225)
0a558c15e0 is described below
commit 0a558c15e039c8db65f03024e829009fa4365567
Author: XiChen <[email protected]>
AuthorDate: Sat Apr 5 13:53:12 2025 +0800
HDDS-12758. Error in OmUtils.normalizeKey for key name starting with `//`
(#8225)
---
.../java/org/apache/hadoop/ozone/OzoneConsts.java | 1 +
.../main/java/org/apache/hadoop/ozone/OmUtils.java | 17 ++++++++++-
.../ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java | 29 ++++++++++++++++++
.../fs/ozone/TestOzoneFSWithObjectStoreCreate.java | 35 ++++++++++++++++++++++
.../ozone/om/request/TestNormalizePaths.java | 3 ++
5 files changed, 84 insertions(+), 1 deletion(-)
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
index d03cc2a22f..11065a27ec 100644
--- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
+++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConsts.java
@@ -170,6 +170,7 @@ public final class OzoneConsts {
*/
public static final String OM_KEY_PREFIX = "/";
+ public static final String DOUBLE_SLASH_OM_KEY_PREFIX = "//";
public static final String OM_USER_PREFIX = "$";
public static final String OM_S3_PREFIX = "S3:";
public static final String OM_S3_CALLER_CONTEXT_PREFIX = "S3Auth:S3G|";
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
index 100ff74ed5..e71e5b912a 100644
--- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
+++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/OmUtils.java
@@ -20,6 +20,7 @@
import static org.apache.hadoop.hdds.HddsUtils.getHostName;
import static org.apache.hadoop.hdds.HddsUtils.getHostNameFromConfigKeys;
import static org.apache.hadoop.hdds.HddsUtils.getPortNumberFromConfigKeys;
+import static org.apache.hadoop.ozone.OzoneConsts.DOUBLE_SLASH_OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OM_SNAPSHOT_INDICATOR;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
@@ -760,7 +761,7 @@ public static String normalizeKey(String keyName,
if (!StringUtils.isBlank(keyName)) {
String normalizedKeyName;
if (keyName.startsWith(OM_KEY_PREFIX)) {
- normalizedKeyName = new Path(keyName).toUri().getPath();
+ normalizedKeyName = new
Path(normalizeLeadingSlashes(keyName)).toUri().getPath();
} else {
normalizedKeyName = new Path(OM_KEY_PREFIX + keyName)
.toUri().getPath();
@@ -778,6 +779,20 @@ public static String normalizeKey(String keyName,
return keyName;
}
+ /**
+ * Normalizes paths by replacing multiple leading slashes with a single
slash.
+ */
+ private static String normalizeLeadingSlashes(String keyName) {
+ if (keyName.startsWith(DOUBLE_SLASH_OM_KEY_PREFIX)) {
+ int index = 0;
+ while (index < keyName.length() && keyName.charAt(index) ==
OM_KEY_PREFIX.charAt(0)) {
+ index++;
+ }
+ return OM_KEY_PREFIX + keyName.substring(index);
+ }
+ return keyName;
+ }
+
/**
* Normalizes a given path up to the bucket level.
*
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
index a4eefb81db..c45b37eed9 100644
---
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/s3/awssdk/v1/AbstractS3SDKV1Tests.java
@@ -101,6 +101,9 @@
import org.apache.hadoop.ozone.client.OzoneClientFactory;
import org.apache.hadoop.ozone.client.OzoneVolume;
import org.apache.hadoop.ozone.client.io.OzoneOutputStream;
+import org.apache.hadoop.ozone.om.helpers.BucketLayout;
+import org.apache.hadoop.ozone.om.helpers.OmBucketInfo;
+import org.apache.hadoop.ozone.om.protocol.OzoneManagerProtocol;
import org.apache.hadoop.ozone.s3.S3ClientFactory;
import org.apache.hadoop.ozone.s3.S3GatewayService;
import org.apache.ozone.test.OzoneTestBase;
@@ -365,6 +368,32 @@ public void testPutObject() {
assertEquals("37b51d194a7513e45b56f6524f2d51f2",
putObjectResult.getETag());
}
+ @Test
+ public void testPutDoubleSlashPrefixObject() throws IOException {
+ final String bucketName = getBucketName();
+ final String keyName = "//dir1";
+ final String content = "bar";
+ OzoneConfiguration conf = cluster.getConf();
+ // Create a FSO bucket for test
+ try (OzoneClient ozoneClient = OzoneClientFactory.getRpcClient(conf)) {
+ ObjectStore store = ozoneClient.getObjectStore();
+ OzoneVolume volume = store.getS3Volume();
+ OmBucketInfo.Builder bucketInfo = new OmBucketInfo.Builder()
+ .setVolumeName(volume.getName())
+ .setBucketName(bucketName)
+ .setBucketLayout(BucketLayout.FILE_SYSTEM_OPTIMIZED);
+ OzoneManagerProtocol ozoneManagerProtocol =
store.getClientProxy().getOzoneManagerClient();
+ ozoneManagerProtocol.createBucket(bucketInfo.build());
+ }
+
+ InputStream is = new
ByteArrayInputStream(content.getBytes(StandardCharsets.UTF_8));
+ PutObjectResult putObjectResult = s3Client.putObject(bucketName, keyName,
is, new ObjectMetadata());
+ assertEquals("37b51d194a7513e45b56f6524f2d51f2",
putObjectResult.getETag());
+
+ S3Object object = s3Client.getObject(bucketName, keyName);
+ assertEquals(content.length(),
object.getObjectMetadata().getContentLength());
+ }
+
@Test
public void testPutObjectEmpty() {
final String bucketName = getBucketName();
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
index 0f5f24fd66..26f06f9abc 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFSWithObjectStoreCreate.java
@@ -27,6 +27,7 @@
import static org.junit.jupiter.api.Assertions.assertNotNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.junit.jupiter.api.Assertions.fail;
import java.io.FileNotFoundException;
import java.io.IOException;
@@ -67,6 +68,8 @@
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.TestInstance;
import org.junit.jupiter.api.Timeout;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
/**
* Class tests create with object store and getFileStatus.
@@ -386,6 +389,38 @@ public void testListKeysWithNotNormalizedPath() throws
Exception {
checkKeyList(ozoneKeyIterator, keys);
}
+ @ParameterizedTest
+ @ValueSource(ints = {2, 3, 4})
+ public void testDoubleSlashPrefixPathNormalization(int slashCount) throws
Exception {
+ OzoneVolume ozoneVolume = client.getObjectStore().getVolume(volumeName);
+ OzoneBucket ozoneBucket = ozoneVolume.getBucket(bucketName);
+ // Generate a path with the specified number of leading slashes
+ StringBuilder keyPrefix = new StringBuilder();
+ for (int i = 0; i < slashCount; i++) {
+ keyPrefix.append('/');
+ }
+ String dirPath = "dir" + slashCount + "/";
+ String keyName = "key" + slashCount;
+ String slashyKey = keyPrefix + dirPath + keyName;
+ String normalizedKey = dirPath + keyName;
+ byte[] data = new byte[10];
+ Arrays.fill(data, (byte)96);
+ ArrayList<String> expectedKeys = new ArrayList<>();
+ expectedKeys.add(dirPath);
+ expectedKeys.add(normalizedKey);
+ TestDataUtil.createKey(ozoneBucket, slashyKey, data);
+
+ try {
+ ozoneBucket.readKey(slashyKey).close();
+ ozoneBucket.readKey(normalizedKey).close();
+ } catch (Exception e) {
+ fail("Should be able to read key " + e.getMessage());
+ }
+
+ Iterator<? extends OzoneKey> it = ozoneBucket.listKeys(dirPath);
+ checkKeyList(it, expectedKeys);
+ }
+
private void checkKeyList(Iterator<? extends OzoneKey > ozoneKeyIterator,
List<String> keys) {
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java
index c37cbfda59..0889a53272 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/TestNormalizePaths.java
@@ -58,6 +58,9 @@ public void testNormalizePathsEnabled() throws Exception {
validateAndNormalizeKey(true, "a/b/c/d/"));
assertEquals("a/b/c/...../d",
validateAndNormalizeKey(true, "////a/b/////c/...../d/"));
+ assertEquals("a/b/c", validateAndNormalizeKey(true, "/a/b/c"));
+ assertEquals("a/b/c", validateAndNormalizeKey(true, "//a/b/c"));
+ assertEquals("a/b/c", validateAndNormalizeKey(true, "///a/b/c"));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
