This is an automated email from the ASF dual-hosted git repository.
adoroszlai pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b267a57831 HDDS-10553. Add test case for creating file with EC
replication config (#6405)
b267a57831 is described below
commit b267a578313da15cb0b5e40fddc4d003c8c222d4
Author: WangYuanben <[email protected]>
AuthorDate: Sun Mar 24 23:03:07 2024 +0800
HDDS-10553. Add test case for creating file with EC replication config
(#6405)
---
hadoop-hdds/docs/content/feature/ErasureCoding.md | 18 ++++++++++++++++++
.../hadoop/fs/ozone/AbstractOzoneFileSystemTest.java | 14 ++++++++++++++
.../fs/ozone/AbstractRootedOzoneFileSystemTest.java | 14 ++++++++++++++
.../apache/hadoop/fs/ozone/OzoneFileSystemTests.java | 16 ++++++++++++++++
4 files changed, 62 insertions(+)
diff --git a/hadoop-hdds/docs/content/feature/ErasureCoding.md
b/hadoop-hdds/docs/content/feature/ErasureCoding.md
index 77866762f6..c4d3739f1d 100644
--- a/hadoop-hdds/docs/content/feature/ErasureCoding.md
+++ b/hadoop-hdds/docs/content/feature/ErasureCoding.md
@@ -174,7 +174,9 @@ the configuration keys
`ozone.server.default.replication.type` and `ozone.server
<name>ozone.server.default.replication.type</name>
<value>EC</value>
</property>
+```
+```XML
<property>
<name>ozone.server.default.replication</name>
<value>RS-6-3-1024k</value>
@@ -208,6 +210,22 @@ We can pass the EC Replication Config while creating the
keys irrespective of bu
ozone sh key put <Ozone Key Object Path> <Local File> --type EC --replication
rs-6-3-1024k
```
+When using ofs/o3fs, we can pass the EC Replication Config by setting the
configuration keys `ozone.replication.type` and `ozone.replication`.
+
+```XML
+<property>
+ <name>ozone.replication.type</name>
+ <value>EC</value>
+</property>
+```
+
+```XML
+<property>
+ <name>ozone.replication</name>
+ <value>rs-3-2-1024k</value>
+</property>
+```
+
In the case bucket already has default EC Replication Config, there is no need
of passing EC Replication Config while creating key.
### Enable Intel ISA-L
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
index b1af31e53f..1cc7ee21a6 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
@@ -110,6 +110,7 @@ import static
org.apache.hadoop.fs.StorageStatistics.CommonStatisticNames.OP_OPE
import static
org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
import static org.apache.hadoop.fs.ozone.Constants.OZONE_DEFAULT_USER;
+import static
org.apache.hadoop.fs.ozone.OzoneFileSystemTests.createKeyWithECReplicationConfiguration;
import static
org.apache.hadoop.hdds.protocol.proto.HddsProtos.ReplicationFactor.ONE;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_ACL_ENABLED;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
@@ -426,6 +427,19 @@ abstract class AbstractOzoneFileSystemTest {
assertTrue(fs.getFileStatus(parent).isDirectory(), "Parent directory does
not appear to be a directory");
}
+ @Test
+ public void testCreateKeyWithECReplicationConfig() throws Exception {
+ Path root = new Path("/" + volumeName + "/" + bucketName);
+ Path testKeyPath = new Path(root, "testKey");
+ createKeyWithECReplicationConfiguration(cluster.getConf(), testKeyPath);
+
+ OzoneKeyDetails key = getKey(testKeyPath, false);
+ assertEquals(HddsProtos.ReplicationType.EC,
+ key.getReplicationConfig().getReplicationType());
+ assertEquals("rs-3-2-1024k",
+ key.getReplicationConfig().getReplication());
+ }
+
@Test
public void testDeleteCreatesFakeParentDir() throws Exception {
Path grandparent = new Path("/testDeleteCreatesFakeParentDir");
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
index 366e8a1905..ff136b3cae 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
@@ -115,6 +115,7 @@ import static
org.apache.hadoop.fs.CommonPathCapabilities.FS_CHECKSUMS;
import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
import static
org.apache.hadoop.fs.contract.ContractTestUtils.assertHasPathCapabilities;
import static org.apache.hadoop.fs.ozone.Constants.LISTING_PAGE_SIZE;
+import static
org.apache.hadoop.fs.ozone.OzoneFileSystemTests.createKeyWithECReplicationConfiguration;
import static org.apache.hadoop.hdds.client.ECReplicationConfig.EcCodec.RS;
import static org.apache.hadoop.ozone.OzoneAcl.AclScope.ACCESS;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_ITERATE_BATCH_SIZE;
@@ -325,6 +326,19 @@ abstract class AbstractRootedOzoneFileSystemTest {
fs.delete(grandparent, true);
}
+ @Test
+ public void testCreateKeyWithECReplicationConfig() throws Exception {
+ String testKeyName = "testKey";
+ Path testKeyPath = new Path(bucketPath, testKeyName);
+ createKeyWithECReplicationConfiguration(cluster.getConf(), testKeyPath);
+
+ OzoneKeyDetails key = getKey(testKeyPath, false);
+ assertEquals(HddsProtos.ReplicationType.EC,
+ key.getReplicationConfig().getReplicationType());
+ assertEquals("rs-3-2-1024k",
+ key.getReplicationConfig().getReplication());
+ }
+
@Test
void testListStatusWithIntermediateDirWithECEnabled()
throws Exception {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
index d729251267..47c584e048 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/OzoneFileSystemTests.java
@@ -21,6 +21,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.contract.ContractTestUtils;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import java.io.IOException;
@@ -30,6 +31,8 @@ import java.util.Set;
import java.util.TreeSet;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
import static org.assertj.core.api.Assertions.assertThat;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -95,4 +98,17 @@ final class OzoneFileSystemTests {
assertEquals(total, iCount);
}
+
+ static void createKeyWithECReplicationConfiguration(OzoneConfiguration
inputConf, Path keyPath)
+ throws IOException {
+ OzoneConfiguration conf = new OzoneConfiguration(inputConf);
+ conf.set(OZONE_REPLICATION, "rs-3-2-1024k");
+ conf.set(OZONE_REPLICATION_TYPE, "EC");
+ URI uri = FileSystem.getDefaultUri(conf);
+ conf.setBoolean(
+ String.format("fs.%s.impl.disable.cache", uri.getScheme()), true);
+ try (FileSystem fileSystem = FileSystem.get(uri, conf)) {
+ ContractTestUtils.touch(fileSystem, keyPath);
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
