http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java new file mode 100644 index 0000000..8474e9c --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractConcatLive.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractConcatTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls CONCAT semantics compliance with Hadoop. + */ +public class TestAdlContractConcatLive extends AbstractContractConcatTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testConcatMissingTarget() throws Throwable { + ContractTestUtils.unsupported("BUG : Adl to support expectation from " + + "concat on missing targets."); + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java new file mode 100644 index 0000000..907c50c --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractCreateLive.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls CREATE semantics compliance with Hadoop. + */ +public class TestAdlContractCreateLive extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testOverwriteEmptyDirectory() throws Throwable { + ContractTestUtils + .unsupported("BUG : Adl to support override empty " + "directory."); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java new file mode 100644 index 0000000..30eaec7 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractDeleteLive.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Before; + +/** + * Verify Adls DELETE semantics compliance with Hadoop. + */ +public class TestAdlContractDeleteLive extends AbstractContractDeleteTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java new file mode 100644 index 0000000..e498110 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractMkdirLive.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls MKDIR semantics compliance with Hadoop. + */ +public class TestAdlContractMkdirLive extends AbstractContractMkdirTest { + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new AdlStorageContract(conf); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testMkdirOverParentFile() throws Throwable { + ContractTestUtils.unsupported("Not supported by Adl"); + } + + @Test + public void testNoMkdirOverFile() throws Throwable { + ContractTestUtils.unsupported("Not supported by Adl"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java new file mode 100644 index 0000000..2bb2095 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractOpenLive.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Before; + +/** + * Verify Adls OPEN/READ semantics compliance with Hadoop. + */ +public class TestAdlContractOpenLive extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java new file mode 100644 index 0000000..06063c5 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRenameLive.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls RENAME semantics compliance with Hadoop. + */ +public class TestAdlContractRenameLive extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testRenameFileOverExistingFile() throws Throwable { + ContractTestUtils + .unsupported("BUG : Adl to support full complete POSIX" + "behaviour"); + } + + @Test + public void testRenameFileNonexistentDir() throws Throwable { + ContractTestUtils + .unsupported("BUG : Adl to support create dir is not " + "exist"); + } + + @Test + public void testRenameWithNonEmptySubDir() throws Throwable { + ContractTestUtils.unsupported("BUG : Adl to support non empty dir move."); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java new file mode 100644 index 0000000..bf4e549 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractRootDirLive.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.apache.hadoop.fs.contract.ContractTestUtils; +import org.junit.Before; +import org.junit.Test; + +/** + * Verify Adls root level operation support. + */ +public class TestAdlContractRootDirLive + extends AbstractContractRootDirectoryTest { + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } + + @Test + public void testRmNonEmptyRootDirNonRecursive() throws Throwable { + ContractTestUtils.unsupported( + "BUG : Adl should throw exception instred " + "of returning false."); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java new file mode 100644 index 0000000..0976464 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlContractSeekLive.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Before; + +/** + * Verify Adls OPEN/READ seek operation support. + */ +public class TestAdlContractSeekLive extends AbstractContractSeekTest { + + @Override + protected AbstractFSContract createContract(Configuration configuration) { + return new AdlStorageContract(configuration); + } + + @Before + @Override + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + super.setup(); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java new file mode 100644 index 0000000..8f53400 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlDifferentSizeWritesLive.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; +import java.net.URISyntaxException; +import java.util.Random; + +/** + * Verify different data segment size writes ensure the integrity and + * order of the data. + */ +public class TestAdlDifferentSizeWritesLive { + + public static byte[] getRandomByteArrayData(int size) { + byte[] b = new byte[size]; + Random rand = new Random(); + rand.nextBytes(b); + return b; + } + + @Before + public void setup() throws Exception { + org.junit.Assume + .assumeTrue(AdlStorageConfiguration.isContractTestEnabled()); + } + + @Test + public void testSmallDataWrites() throws IOException { + testDataIntegrity(4 * 1024 * 1024, 1 * 1024); + testDataIntegrity(4 * 1024 * 1024, 7 * 1024); + testDataIntegrity(4 * 1024 * 1024, 10); + testDataIntegrity(2 * 1024 * 1024, 10); + testDataIntegrity(1 * 1024 * 1024, 10); + testDataIntegrity(100, 1); + } + + @Test + public void testMediumDataWrites() throws IOException { + testDataIntegrity(4 * 1024 * 1024, 1 * 1024 * 1024); + testDataIntegrity(7 * 1024 * 1024, 2 * 1024 * 1024); + testDataIntegrity(9 * 1024 * 1024, 2 * 1024 * 1024); + testDataIntegrity(10 * 1024 * 1024, 3 * 1024 * 1024); + } + + private void testDataIntegrity(int totalSize, int chunkSize) + throws IOException { + Path path = new Path("/test/dataIntegrityCheck"); + FileSystem fs = null; + try { + fs = AdlStorageConfiguration.createStorageConnector(); + } catch (URISyntaxException e) { + throw new IllegalStateException("Can not initialize ADL FileSystem. " + + "Please check fs.defaultFS property.", e); + } + byte[] expectedData = getRandomByteArrayData(totalSize); + + FSDataOutputStream out = fs.create(path, true); + int iteration = totalSize / chunkSize; + int reminderIteration = totalSize % chunkSize; + int offset = 0; + for (int i = 0; i < iteration; ++i) { + out.write(expectedData, offset, chunkSize); + offset += chunkSize; + } + + out.write(expectedData, offset, reminderIteration); + out.close(); + + byte[] actualData = new byte[totalSize]; + FSDataInputStream in = fs.open(path); + in.readFully(0, actualData); + in.close(); + Assert.assertArrayEquals(expectedData, actualData); + Assert.assertTrue(fs.delete(path, true)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java new file mode 100644 index 0000000..be27936 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/live/TestAdlFileSystemContractLive.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.hadoop.fs.adl.live; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.FileSystemContractBaseTest; +import org.apache.hadoop.fs.Path; +import org.junit.Test; + +import java.io.IOException; + +/** + * Verify Adls adhere to Hadoop file system semantics. + */ +public class TestAdlFileSystemContractLive extends FileSystemContractBaseTest { + private FileSystem adlStore; + + @Override + protected void setUp() throws Exception { + adlStore = AdlStorageConfiguration.createStorageConnector(); + if (AdlStorageConfiguration.isContractTestEnabled()) { + fs = adlStore; + } + } + + @Override + protected void tearDown() throws Exception { + if (AdlStorageConfiguration.isContractTestEnabled()) { + cleanup(); + adlStore = null; + fs = null; + } + } + + private void cleanup() throws IOException { + adlStore.delete(new Path("/test"), true); + } + + @Override + protected void runTest() throws Throwable { + if (AdlStorageConfiguration.isContractTestEnabled()) { + super.runTest(); + } + } + + public void testGetFileStatus() throws IOException { + if (!AdlStorageConfiguration.isContractTestEnabled()) { + return; + } + + Path testPath = new Path("/test/adltest"); + if (adlStore.exists(testPath)) { + adlStore.delete(testPath, false); + } + + adlStore.create(testPath).close(); + assertTrue(adlStore.delete(testPath, false)); + } + + /** + * The following tests are failing on Azure Data Lake and the Azure Data Lake + * file system code needs to be modified to make them pass. + * A separate work item has been opened for this. + */ + @Test + @Override + public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception { + // BUG : Adl should return exception instead of false. + } + + @Test + @Override + public void testMkdirsWithUmask() throws Exception { + // Support under implementation in Adl + } + +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml new file mode 100644 index 0000000..76a78b0 --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/adls.xml @@ -0,0 +1,140 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<configuration> + <property> + <name>fs.contract.test.root-tests-enabled</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.test.supports-concat</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.rename-returns-false-if-source-missing</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.test.random-seek-count</name> + <value>10</value> + </property> + + <property> + <name>fs.contract.is-case-sensitive</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.rename-returns-true-if-dest-exists</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.rename-returns-true-if-source-missing</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.rename-creates-dest-dirs</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.rename-remove-dest-if-empty-dir</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.supports-settimes</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-append</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-atomic-directory-delete</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-atomic-rename</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-block-locality</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-concat</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-seek</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-seek-on-closed-file</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.rejects-seek-past-eof</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-available-on-closed-file</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-strict-exceptions</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.supports-unix-permissions</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.rename-overwrites-dest</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.supports-append</name> + <value>true</value> + </property> + + <property> + <name>fs.azure.enable.append.support</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-getfilestatus</name> + <value>true</value> + </property> + +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml new file mode 100644 index 0000000..646922e --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/contract-test-options.xml @@ -0,0 +1,61 @@ +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> +<configuration> + <property> + <name>dfs.adls.oauth2.refresh.url</name> + <value> + </value> + </property> + + <property> + <name>dfs.adls.oauth2.credential</name> + <value></value> + </property> + + <property> + <name>dfs.adls.oauth2.client.id</name> + <value></value> + </property> + + <property> + <name>dfs.adls.oauth2.access.token.provider.type</name> + <value>ClientCredential</value> + <description> + Supported provider type: + + "ClientCredential" : Client id and client credentials(Provided + through configuration file) flow. + + "RefreshToken" : Client id and refresh token(Provided + through configuration file)flow. + + "Custom" : Custom AAD token management. + </description> + </property> + + <property> + <name>dfs.adl.test.contract.enable</name> + <value>false</value> + </property> + + <property> + <name>test.fs.adl.name</name> + <value></value> + </property> + + <property> + <name>fs.contract.test.fs</name> + <value>org.apache.hadoop.fs.adl.AdlFileSystem</value> + </property> +</configuration> http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/hadoop-azure-datalake/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/resources/log4j.properties b/hadoop-tools/hadoop-azure-datalake/src/test/resources/log4j.properties new file mode 100644 index 0000000..4cc8f7f --- /dev/null +++ b/hadoop-tools/hadoop-azure-datalake/src/test/resources/log4j.properties @@ -0,0 +1,30 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +log4j.rootLogger=DEBUG,stdout +log4j.appender.stdout=org.apache.log4j.ConsoleAppender +log4j.appender.stdout.layout=org.apache.log4j.PatternLayout +log4j.appender.stdout.layout.ConversionPattern=%d [%t] %-5p %X{file} %c{1} - %m%n +log4j.logger.your.app=* +log4j.additivity.your.app=false +log4j.logger.yourApp=* +log4j.additivity.yourApp=false +log4j.appender.yourApp=org.apache.log4j.ConsoleAppender +log4j.appender.yourApp.layout=org.apache.log4j.PatternLayout +log4j.appender.yourApp.layout.ConversionPattern=%d [%t] %-5p %X{file} %c{1} %m%n +log4j.appender.yourApp.ImmediateFlush=true \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/edf149b9/hadoop-tools/pom.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/pom.xml b/hadoop-tools/pom.xml index 16bab03..86d573a 100644 --- a/hadoop-tools/pom.xml +++ b/hadoop-tools/pom.xml @@ -46,6 +46,7 @@ <module>hadoop-sls</module> <module>hadoop-aws</module> <module>hadoop-azure</module> + <module>hadoop-azure-datalake</module> </modules> <build> --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
