This is an automated email from the ASF dual-hosted git repository.
rakeshr pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new d09cdd4 HDDS-4743. [FSO] Add FSO variant of ITestOzoneContractDistcp.
(#2980)
d09cdd4 is described below
commit d09cdd4a585b66e95f32c8a054ee493101ee4de0
Author: Jyotinder Singh <[email protected]>
AuthorDate: Wed Jan 12 17:33:33 2022 +0530
HDDS-4743. [FSO] Add FSO variant of ITestOzoneContractDistcp. (#2980)
---
.../contract/ITestOzoneContractDistCpWithFSO.java | 65 ++++++++++++++++++++++
.../hadoop/fs/ozone/contract/OzoneContract.java | 18 ++++--
2 files changed, 79 insertions(+), 4 deletions(-)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCpWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCpWithFSO.java
new file mode 100644
index 0000000..333ef18
--- /dev/null
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/ITestOzoneContractDistCpWithFSO.java
@@ -0,0 +1,65 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.fs.ozone.contract;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.tools.contract.AbstractContractDistCpTest;
+
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+
+import static org.apache.hadoop.fs.contract.ContractTestUtils.cleanup;
+
+
+/**
+ * Contract test suite covering S3A integration with DistCp.
+ * Uses the block output stream, buffered to disk. This is the
+ * recommended output mechanism for DistCP due to its scalability.
+ * This test suite runs the server in File System Optimized mode.
+ * <p>
+ * Note: It isn't possible to convert this into a parameterized test due to
+ * unrelated failures occurring while trying to handle directories with names
+ * containing '[' and ']' characters.
+ */
+public class ITestOzoneContractDistCpWithFSO
+ extends AbstractContractDistCpTest {
+
+ @BeforeClass
+ public static void createCluster() throws IOException {
+ OzoneContract.createCluster(true);
+ }
+
+ @AfterClass
+ public static void teardownCluster() throws IOException {
+ OzoneContract.destroyCluster();
+ }
+
+ @Override
+ protected OzoneContract createContract(Configuration conf) {
+ return new OzoneContract(conf);
+ }
+
+ @Override
+ protected void deleteTestDirInTeardown() throws IOException {
+ super.deleteTestDirInTeardown();
+ cleanup("TEARDOWN", getLocalFS(), getLocalDir());
+ }
+}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
index 56326b4..f6c9a25 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/OzoneContract.java
@@ -70,6 +70,12 @@ class OzoneContract extends AbstractFSContract {
fsOptimizedServer = fsoServer;
}
+ public static void createCluster(boolean fsoServer) throws IOException {
+ // Set the flag to enable/disable FSO on server.
+ initOzoneConfiguration(fsoServer);
+ createCluster();
+ }
+
public static void createCluster() throws IOException {
OzoneConfiguration conf = new OzoneConfiguration();
DatanodeRatisServerConfig ratisServerConfig =
@@ -87,11 +93,15 @@ class OzoneContract extends AbstractFSContract {
conf.addResource(CONTRACT_XML);
if (fsOptimizedServer){
- conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS,
- true);
+ // Default bucket layout is set to FSO in case of FSO server.
+ conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
+ OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED);
+ } else {
+ // Default bucket layout is set to LEGACY to support Hadoop compatible
+ // FS operations that are incompatible with OBS (default config value).
+ conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
+ BucketLayout.LEGACY.name());
}
- conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
- BucketLayout.LEGACY.name());
cluster = MiniOzoneCluster.newBuilder(conf).setNumDatanodes(5).build();
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
