http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java new file mode 100644 index 0000000..6d1c4ae --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRename.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for rename operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractRename extends AbstractContractRenameTest { + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable<Object[]> secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractRename(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java new file mode 100644 index 0000000..46072ad --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractRootDirectory.java @@ -0,0 +1,67 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRootDirectoryTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import org.junit.Ignore; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +/** + * Contract test for root directory operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractRootDirectory extends AbstractContractRootDirectoryTest { + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable<Object[]> secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractRootDirectory(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(secure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } + + @Override + @Ignore("ABFS always return false when non-recursively remove root dir") + public void testRmNonEmptyRootDirNonRecursive() throws Throwable { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java new file mode 100644 index 0000000..1780f6f --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSecureDistCp.java @@ -0,0 +1,44 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; + +/** + * Contract test for secure distCP operation. + */ +public class ITestAbfsFileSystemContractSecureDistCp extends AbstractContractDistCpTest { + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractSecureDistCp() throws Exception { + dependencyInjectedContractTest = new DependencyInjectedContractTest(true); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected ITestAbfsFileSystemContract createContract(Configuration conf) { + return new ITestAbfsFileSystemContract(conf, true); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java new file mode 100644 index 0000000..aeeb042 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSeek.java @@ -0,0 +1,63 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for seek operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractSeek extends AbstractContractSeekTest{ + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable<Object[]> secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractSeek(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java new file mode 100644 index 0000000..ea9392d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAbfsFileSystemContractSetTimes.java @@ -0,0 +1,62 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.contract; + +import java.util.Arrays; + +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSetTimesTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test for setTimes operation. + */ +@RunWith(Parameterized.class) +public class ITestAbfsFileSystemContractSetTimes extends AbstractContractSetTimesTest { + @Parameterized.Parameters(name = "SecureMode={0}") + public static Iterable<Object[]> secure() { + return Arrays.asList(new Object[][] { {true}, {false} }); + } + + private final boolean isSecure; + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAbfsFileSystemContractSetTimes(final boolean secure) throws Exception { + this.isSecure = secure; + dependencyInjectedContractTest = new DependencyInjectedContractTest(this.isSecure); + } + + @Override + public void setup() throws Exception { + dependencyInjectedContractTest.initialize(); + super.setup(); + } + + @Override + protected Configuration createConfiguration() { + return this.dependencyInjectedContractTest.getConfiguration(); + } + + @Override + protected AbstractFSContract createContract(final Configuration conf) { + return new ITestAbfsFileSystemContract(conf, this.isSecure); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java new file mode 100644 index 0000000..e148a05 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/ITestAzureBlobFileSystemBasics.java @@ -0,0 +1,91 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.hadoop.fs.azurebfs.contract; + +import java.io.IOException; + +import org.apache.hadoop.fs.FileSystemContractBaseTest; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +import org.junit.Before; +import org.junit.Ignore; +import org.junit.Test; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertTrue; + +/** + * Basic Contract test for Azure BlobFileSystem. + */ +public class ITestAzureBlobFileSystemBasics extends FileSystemContractBaseTest { + private final DependencyInjectedContractTest dependencyInjectedContractTest; + + public ITestAzureBlobFileSystemBasics() throws Exception { + // If contract tests are running in parallel, some root level tests in this file will fail + // due to the race condition. Hence for this contract test it should be tested in different container + dependencyInjectedContractTest = new DependencyInjectedContractTest(false, false); + } + + @Before + public void setUp() throws Exception { + this.dependencyInjectedContractTest.initialize(); + fs = this.dependencyInjectedContractTest.getFileSystem(); + } + + @Test + public void testListOnFolderWithNoChildren() throws IOException { + assertTrue(fs.mkdirs(path("testListStatus/c/1"))); + + FileStatus[] paths; + paths = fs.listStatus(path("testListStatus")); + assertEquals(1, paths.length); + + // ListStatus on folder with child + paths = fs.listStatus(path("testListStatus/c")); + assertEquals(1, paths.length); + + // Remove the child and listStatus + fs.delete(path("testListStatus/c/1"), true); + paths = fs.listStatus(path("testListStatus/c")); + assertEquals(0, paths.length); + assertTrue(fs.delete(path("testListStatus"), true)); + } + + @Test + public void testListOnfileAndFolder() throws IOException { + Path folderPath = path("testListStatus/folder"); + Path filePath = path("testListStatus/file"); + + assertTrue(fs.mkdirs(folderPath)); + fs.create(filePath); + + FileStatus[] listFolderStatus; + listFolderStatus = fs.listStatus(path("testListStatus")); + assertEquals(filePath, listFolderStatus[0].getPath()); + + //List on file should return absolute path + FileStatus[] listFileStatus = fs.listStatus(filePath); + assertEquals(filePath, listFileStatus[0].getPath()); + } + + @Override + @Ignore("Not implemented in ABFS yet") + public void testMkdirsWithUmask() throws Exception { + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/package-info.java new file mode 100644 index 0000000..f3ff483 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/contract/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +package org.apache.hadoop.fs.azurebfs.contract; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java new file mode 100644 index 0000000..03320d6 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/TestConfigurationValidators.java @@ -0,0 +1,124 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.diagnostics; + +import org.apache.commons.codec.Charsets; +import org.junit.Test; + + +import org.apache.hadoop.fs.azurebfs.DependencyInjectedTest; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.InvalidConfigurationValueException; +import org.apache.commons.codec.binary.Base64; + +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE; + +import static org.junit.Assert.assertEquals; + +/** + * Test configuration validators. + */ +public class TestConfigurationValidators extends DependencyInjectedTest { + + private static final String FAKE_KEY = "FakeKey"; + + public TestConfigurationValidators() throws Exception { + super(); + } + + @Test + public void testIntegerConfigValidator() throws Exception { + IntegerConfigurationBasicValidator integerConfigurationValidator = new IntegerConfigurationBasicValidator( + MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, DEFAULT_READ_BUFFER_SIZE, FAKE_KEY, false); + + assertEquals(MIN_BUFFER_SIZE, (int) integerConfigurationValidator.validate("3072")); + assertEquals(DEFAULT_READ_BUFFER_SIZE, (int) integerConfigurationValidator.validate(null)); + assertEquals(MAX_BUFFER_SIZE, (int) integerConfigurationValidator.validate("104857600")); + } + + @Test(expected = InvalidConfigurationValueException.class) + public void testIntegerConfigValidatorThrowsIfMissingValidValue() throws Exception { + IntegerConfigurationBasicValidator integerConfigurationValidator = new IntegerConfigurationBasicValidator( + MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, DEFAULT_READ_BUFFER_SIZE, FAKE_KEY, true); + integerConfigurationValidator.validate("3072"); + } + + @Test + public void testLongConfigValidator() throws Exception { + LongConfigurationBasicValidator longConfigurationValidator = new LongConfigurationBasicValidator( + MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, DEFAULT_WRITE_BUFFER_SIZE, FAKE_KEY, false); + + assertEquals(DEFAULT_WRITE_BUFFER_SIZE, (long) longConfigurationValidator.validate(null)); + assertEquals(MIN_BUFFER_SIZE, (long) longConfigurationValidator.validate("3072")); + assertEquals(MAX_BUFFER_SIZE, (long) longConfigurationValidator.validate("104857600")); + } + + @Test(expected = InvalidConfigurationValueException.class) + public void testLongConfigValidatorThrowsIfMissingValidValue() throws Exception { + LongConfigurationBasicValidator longConfigurationValidator = new LongConfigurationBasicValidator( + MIN_BUFFER_SIZE, MAX_BUFFER_SIZE, DEFAULT_READ_BUFFER_SIZE, FAKE_KEY, true); + longConfigurationValidator.validate(null); + } + + @Test + public void testBooleanConfigValidator() throws Exception { + BooleanConfigurationBasicValidator booleanConfigurationValidator = new BooleanConfigurationBasicValidator(FAKE_KEY, false, false); + + assertEquals(true, booleanConfigurationValidator.validate("true")); + assertEquals(false, booleanConfigurationValidator.validate("False")); + assertEquals(false, booleanConfigurationValidator.validate(null)); + } + + @Test(expected = InvalidConfigurationValueException.class) + public void testBooleanConfigValidatorThrowsIfMissingValidValue() throws Exception { + BooleanConfigurationBasicValidator booleanConfigurationValidator = new BooleanConfigurationBasicValidator(FAKE_KEY, false, true); + booleanConfigurationValidator.validate("almostTrue"); + } + + @Test + public void testStringConfigValidator() throws Exception { + StringConfigurationBasicValidator stringConfigurationValidator = new StringConfigurationBasicValidator(FAKE_KEY, "value", false); + + assertEquals("value", stringConfigurationValidator.validate(null)); + assertEquals("someValue", stringConfigurationValidator.validate("someValue")); + } + + @Test(expected = InvalidConfigurationValueException.class) + public void testStringConfigValidatorThrowsIfMissingValidValue() throws Exception { + StringConfigurationBasicValidator stringConfigurationValidator = new StringConfigurationBasicValidator(FAKE_KEY, "value", true); + stringConfigurationValidator.validate(null); + } + + @Test + public void testBase64StringConfigValidator() throws Exception { + String encodedVal = new String(new Base64().encode("someValue".getBytes()), Charsets.UTF_8); + Base64StringConfigurationBasicValidator base64StringConfigurationValidator = new Base64StringConfigurationBasicValidator(FAKE_KEY, "", false); + + assertEquals("", base64StringConfigurationValidator.validate(null)); + assertEquals(encodedVal, base64StringConfigurationValidator.validate(encodedVal)); + } + + @Test(expected = InvalidConfigurationValueException.class) + public void testBase64StringConfigValidatorThrowsIfMissingValidValue() throws Exception { + Base64StringConfigurationBasicValidator base64StringConfigurationValidator = new Base64StringConfigurationBasicValidator(FAKE_KEY, "value", true); + base64StringConfigurationValidator.validate("some&%Value"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/package-info.java new file mode 100644 index 0000000..c3434ac --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/diagnostics/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +package org.apache.hadoop.fs.azurebfs.diagnostics; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/package-info.java new file mode 100644 index 0000000..811fdcb --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +package org.apache.hadoop.fs.azurebfs; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsHttpServiceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsHttpServiceImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsHttpServiceImpl.java new file mode 100644 index 0000000..588df20 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestAbfsHttpServiceImpl.java @@ -0,0 +1,122 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.util.Hashtable; + +import org.junit.Assert; +import org.junit.Ignore; +import org.junit.Test; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.DependencyInjectedTest; +import org.apache.hadoop.fs.azurebfs.contracts.services.AbfsHttpService; + +import static org.junit.Assert.assertEquals; + +/** + * Test AbfsHttpServiceImpl. + */ +public class ITestAbfsHttpServiceImpl extends DependencyInjectedTest { + private static final int TEST_DATA = 100; + private static final Path TEST_PATH = new Path("/testfile"); + public ITestAbfsHttpServiceImpl() { + super(); + } + + @Test + public void testReadWriteBytesToFileAndEnsureThreadPoolCleanup() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + testWriteOneByteToFileAndEnsureThreadPoolCleanup(); + + FSDataInputStream inputStream = fs.open(TEST_PATH, 4 * 1024 * 1024); + int i = inputStream.read(); + + assertEquals(TEST_DATA, i); + } + + @Test + public void testWriteOneByteToFileAndEnsureThreadPoolCleanup() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + FSDataOutputStream stream = fs.create(TEST_PATH); + + stream.write(TEST_DATA); + stream.close(); + + FileStatus fileStatus = fs.getFileStatus(TEST_PATH); + assertEquals(1, fileStatus.getLen()); + } + + @Test + @Ignore("JDK7 doesn't support PATCH, so PUT is used. Fix is applied in latest test tenant") + public void testBase64FileSystemProperties() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + final Hashtable<String, String> properties = new Hashtable<>(); + properties.put("key", "{ value: value }"); + AbfsServiceProviderImpl.instance().get(AbfsHttpService.class).setFilesystemProperties( + fs, properties); + Hashtable<String, String> fetchedProperties = AbfsServiceProviderImpl.instance().get(AbfsHttpService.class).getFilesystemProperties(fs); + + Assert.assertEquals(properties, fetchedProperties); + } + + @Test + public void testBase64PathProperties() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + final Hashtable<String, String> properties = new Hashtable<>(); + properties.put("key", "{ value: valueTest }"); + fs.create(TEST_PATH); + AbfsServiceProviderImpl.instance().get(AbfsHttpService.class).setPathProperties( + fs, TEST_PATH, properties); + Hashtable<String, String> fetchedProperties = + AbfsServiceProviderImpl.instance().get(AbfsHttpService.class).getPathProperties(fs, TEST_PATH); + + Assert.assertEquals(properties, fetchedProperties); + } + + @Test (expected = Exception.class) + public void testBase64InvalidFileSystemProperties() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + final Hashtable<String, String> properties = new Hashtable<>(); + properties.put("key", "{ value: valueæ² }"); + AbfsServiceProviderImpl.instance().get(AbfsHttpService.class).setFilesystemProperties( + fs, properties); + Hashtable<String, String> fetchedProperties = AbfsServiceProviderImpl.instance().get(AbfsHttpService.class).getFilesystemProperties(fs); + + Assert.assertEquals(properties, fetchedProperties); + } + + @Test (expected = Exception.class) + public void testBase64InvalidPathProperties() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + final Hashtable<String, String> properties = new Hashtable<>(); + properties.put("key", "{ value: valueTestå © }"); + fs.create(TEST_PATH); + AbfsServiceProviderImpl.instance().get(AbfsHttpService.class).setPathProperties( + fs, TEST_PATH, properties); + Hashtable<String, String> fetchedProperties = + AbfsServiceProviderImpl.instance().get(AbfsHttpService.class).getPathProperties(fs, TEST_PATH); + + Assert.assertEquals(properties, fetchedProperties); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestReadWriteAndSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestReadWriteAndSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestReadWriteAndSeek.java new file mode 100644 index 0000000..1db93cb --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestReadWriteAndSeek.java @@ -0,0 +1,80 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.util.Random; + +import org.apache.hadoop.fs.FSDataInputStream; +import org.apache.hadoop.fs.FSDataOutputStream; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem; +import org.apache.hadoop.fs.azurebfs.DependencyInjectedTest; +import org.apache.hadoop.fs.azurebfs.contracts.services.ConfigurationService; + +import org.junit.Test; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MIN_BUFFER_SIZE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_BUFFER_SIZE; +import static org.junit.Assert.assertArrayEquals; +import static org.junit.Assert.assertNotEquals; + +/** + * Test read, write and seek. + */ +public class ITestReadWriteAndSeek extends DependencyInjectedTest { + private static final Path TEST_PATH = new Path("/testfile"); + public ITestReadWriteAndSeek() { + super(); + } + + @Test + public void testReadAndWriteWithDifferentBufferSizesAndSeek() throws Exception { + testReadWriteAndSeek(MIN_BUFFER_SIZE); + testReadWriteAndSeek(DEFAULT_READ_BUFFER_SIZE); + testReadWriteAndSeek(MAX_BUFFER_SIZE); + } + + private void testReadWriteAndSeek(int bufferSize) throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + final ConfigurationServiceImpl configurationservice = (ConfigurationServiceImpl) AbfsServiceProviderImpl.instance().get(ConfigurationService.class); + + fs.create(TEST_PATH); + + configurationservice.setWriteBufferSize(bufferSize); + configurationservice.setReadBufferSize(bufferSize); + + final FSDataOutputStream stream = fs.create(TEST_PATH); + + final byte[] b = new byte[2 * bufferSize]; + new Random().nextBytes(b); + stream.write(b); + stream.close(); + + final byte[] r = new byte[2 * bufferSize]; + final FSDataInputStream inputStream = fs.open(TEST_PATH); + inputStream.seek(bufferSize); + int result = inputStream.read(r, bufferSize, bufferSize); + assertNotEquals(-1, result); + + inputStream.seek(0); + result = inputStream.read(r, 0, bufferSize); + assertNotEquals(-1, result); + assertArrayEquals(r, b); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestTracingServiceImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestTracingServiceImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestTracingServiceImpl.java new file mode 100644 index 0000000..e90eb22 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/ITestTracingServiceImpl.java @@ -0,0 +1,79 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.log4j.AppenderSkeleton; +import org.apache.log4j.Level; +import org.apache.log4j.Logger; +import org.apache.log4j.spi.LoggingEvent; +import org.junit.Test; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.DependencyInjectedTest; +import org.apache.hadoop.fs.azurebfs.contracts.services.TracingService; +import org.apache.htrace.core.MilliSpan; +import org.apache.htrace.core.TraceScope; + +/** + * Test tracing service. + */ +public class ITestTracingServiceImpl extends DependencyInjectedTest { + private final List<String> messageStorage; + + public ITestTracingServiceImpl() { + super(); + this.messageStorage = new ArrayList<>(); + } + + @Test + public void traceSerializationTest() throws Exception { + Logger rootLogger = Logger.getRootLogger(); + rootLogger.setLevel(Level.TRACE); + rootLogger.addAppender(new AppenderSkeleton() { + @Override + protected void append(LoggingEvent event) { + if (event.getLogger().getName().indexOf("AzureBlobFileSystem") != -1) { + messageStorage.add(event.getRenderedMessage()); + } + } + + @Override + public void close() { + + } + + @Override + public boolean requiresLayout() { + return false; + } + }); + + TracingService tracingService = new TracingServiceImpl(new Configuration()); + TraceScope traceScope = tracingService.traceBegin("Test Scope"); + traceScope.addTimelineAnnotation("Timeline Annotations"); + traceScope.addKVAnnotation("key", "value"); + traceScope.close(); + + // Should not throw exception. + MilliSpan.fromJson(messageStorage.get(0)); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockAbfsHttpClientFactoryImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockAbfsHttpClientFactoryImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockAbfsHttpClientFactoryImpl.java new file mode 100644 index 0000000..2f27b16 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockAbfsHttpClientFactoryImpl.java @@ -0,0 +1,69 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import com.google.common.annotations.VisibleForTesting; +import com.google.inject.Inject; +import com.google.inject.Singleton; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; +import org.apache.hadoop.fs.azurebfs.constants.TestConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.contracts.services.ConfigurationService; +import org.apache.hadoop.fs.azurebfs.utils.UriUtils; +import org.apache.http.client.utils.URIBuilder; + +/** + * Mock AbfsHttpClientFactoryImpl. + */ +@Singleton +@InterfaceAudience.Private +@InterfaceStability.Evolving +public class MockAbfsHttpClientFactoryImpl extends AbfsHttpClientFactoryImpl { + private final ConfigurationService configurationService; + + @Inject + MockAbfsHttpClientFactoryImpl( + final ConfigurationService configurationService) { + super(configurationService); + + this.configurationService = configurationService; + } + + @VisibleForTesting + URIBuilder getURIBuilder(final String hostName, final FileSystem fs) { + final URIBuilder uriBuilder = new URIBuilder(); + + final String testHost = this.configurationService.getConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_HOST_NAME); + final Integer testHostPort = this.configurationService.getConfiguration().getInt(TestConfigurationKeys.FS_AZURE_TEST_HOST_PORT, 80); + final String testAccount = this.configurationService.getConfiguration().get(TestConfigurationKeys.FS_AZURE_TEST_ACCOUNT_NAME); + + String scheme = FileSystemUriSchemes.HTTP_SCHEME; + + uriBuilder.setScheme(scheme); + uriBuilder.setHost(testHost); + uriBuilder.setPort(testHostPort); + + uriBuilder.setPath("/" + UriUtils.extractRawAccountFromAccountName(testAccount) + "/"); + + return uriBuilder; + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockAbfsServiceInjectorImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockAbfsServiceInjectorImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockAbfsServiceInjectorImpl.java new file mode 100644 index 0000000..ff2fb2a --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockAbfsServiceInjectorImpl.java @@ -0,0 +1,50 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import org.apache.hadoop.conf.Configuration; + +/** + * Mock AbfsServiceInjectorImpl. + */ +public class MockAbfsServiceInjectorImpl extends AbfsServiceInjectorImpl { + public MockAbfsServiceInjectorImpl(Configuration configuration) { + super(configuration); + } + + public <T> void replaceInstance(Class<T> tInterface, Object object) { + this.removeInstance(tInterface); + this.removeProvider(tInterface); + this.getInstances().put(tInterface, object); + } + + public <T> void removeInstance(Class<T> tInterface) { + this.getInstances().remove(tInterface); + } + + public <T> void replaceProvider(Class<T> tInterface, Class<? extends T> tClazz) { + this.removeInstance(tInterface); + this.removeProvider(tInterface); + this.getProviders().put(tInterface, tClazz); + } + + public <T> void removeProvider(Class<T> tInterface) { + this.getProviders().remove(tInterface); + } +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockServiceProviderImpl.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockServiceProviderImpl.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockServiceProviderImpl.java new file mode 100644 index 0000000..5992f75 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/MockServiceProviderImpl.java @@ -0,0 +1,36 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import com.google.inject.Guice; +import com.google.inject.Injector; + +/** + * Mock ABFS ServiceProviderImpl. + */ +public final class MockServiceProviderImpl { + public static void create(MockAbfsServiceInjectorImpl abfsServiceInjector) { + Injector injector = Guice.createInjector(abfsServiceInjector); + AbfsServiceProviderImpl.create(injector); + } + + private MockServiceProviderImpl() { + // no-op + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestConfigurationServiceFieldsValidation.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestConfigurationServiceFieldsValidation.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestConfigurationServiceFieldsValidation.java new file mode 100644 index 0000000..688c522 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/TestConfigurationServiceFieldsValidation.java @@ -0,0 +1,149 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.services; + +import java.lang.reflect.Field; + +import org.apache.commons.codec.Charsets; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; +import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.IntegerConfigurationValidatorAnnotation; +import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.BooleanConfigurationValidatorAnnotation; +import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.StringConfigurationValidatorAnnotation; +import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.LongConfigurationValidatorAnnotation; +import org.apache.hadoop.fs.azurebfs.contracts.annotations.ConfigurationValidationAnnotations.Base64StringConfigurationValidatorAnnotation; +import org.apache.hadoop.fs.azurebfs.contracts.exceptions.ConfigurationPropertyNotFoundException; + +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_READ_BUFFER_SIZE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_WRITE_BUFFER_SIZE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_MAX_RETRY_ATTEMPTS; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_BACKOFF_INTERVAL; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_MAX_BACKOFF_INTERVAL; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.DEFAULT_MIN_BACKOFF_INTERVAL; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.MAX_AZURE_BLOCK_SIZE; +import static org.apache.hadoop.fs.azurebfs.constants.FileSystemConfigurations.AZURE_BLOCK_LOCATION_HOST_DEFAULT; + +import org.apache.commons.codec.binary.Base64; +import static org.junit.Assert.assertEquals; + +import org.junit.Test; + +/** + * Test ConfigurationServiceFieldsValidation. + */ +public class TestConfigurationServiceFieldsValidation { + private ConfigurationServiceImpl configService; + + private static final String INT_KEY= "intKey"; + private static final String LONG_KEY= "longKey"; + private static final String STRING_KEY= "stringKey"; + private static final String BASE64_KEY= "base64Key"; + private static final String BOOLEAN_KEY= "booleanKey"; + private static final int DEFAULT_INT = 4194304; + private static final int DEFAULT_LONG = 4194304; + + private static final int TEST_INT = 1234565; + private static final int TEST_LONG = 4194304; + + private final String encodedString; + private final String encodedAccountKey; + + @IntegerConfigurationValidatorAnnotation(ConfigurationKey = INT_KEY, + MinValue = Integer.MIN_VALUE, + MaxValue = Integer.MAX_VALUE, + DefaultValue = DEFAULT_INT) + private int intField; + + @LongConfigurationValidatorAnnotation(ConfigurationKey = LONG_KEY, + MinValue = Long.MIN_VALUE, + MaxValue = Long.MAX_VALUE, + DefaultValue = DEFAULT_LONG) + private int longField; + + @StringConfigurationValidatorAnnotation(ConfigurationKey = STRING_KEY, + DefaultValue = "default") + private String stringField; + + @Base64StringConfigurationValidatorAnnotation(ConfigurationKey = BASE64_KEY, + DefaultValue = "base64") + private String base64Field; + + @BooleanConfigurationValidatorAnnotation(ConfigurationKey = BOOLEAN_KEY, + DefaultValue = false) + private boolean boolField; + + public TestConfigurationServiceFieldsValidation() throws Exception { + super(); + Base64 base64 = new Base64(); + this.encodedString = new String(base64.encode("base64Value".getBytes(Charsets.UTF_8)), Charsets.UTF_8); + this.encodedAccountKey = new String(base64.encode("someAccountKey".getBytes(Charsets.UTF_8)), Charsets.UTF_8); + Configuration configuration = new Configuration(); + configuration.addResource("azure-bfs-test.xml"); + configuration.set(INT_KEY, "1234565"); + configuration.set(LONG_KEY, "4194304"); + configuration.set(STRING_KEY, "stringValue"); + configuration.set(BASE64_KEY, encodedString); + configuration.set(BOOLEAN_KEY, "true"); + configuration.set(ConfigurationKeys.FS_AZURE_ACCOUNT_KEY_PROPERTY_NAME + "testaccount1.blob.core.windows.net", this.encodedAccountKey); + configService = new ConfigurationServiceImpl(configuration); + } + + @Test + public void testValidateFunctionsInConfigServiceImpl() throws Exception { + Field[] fields = this.getClass().getDeclaredFields(); + for (Field field : fields) { + field.setAccessible(true); + if (field.isAnnotationPresent(IntegerConfigurationValidatorAnnotation.class)) { + assertEquals(TEST_INT, configService.validateInt(field)); + } else if (field.isAnnotationPresent(LongConfigurationValidatorAnnotation.class)) { + assertEquals(DEFAULT_LONG, configService.validateLong(field)); + } else if (field.isAnnotationPresent(StringConfigurationValidatorAnnotation.class)) { + assertEquals("stringValue", configService.validateString(field)); + } else if (field.isAnnotationPresent(Base64StringConfigurationValidatorAnnotation.class)) { + assertEquals(this.encodedString, configService.validateBase64String(field)); + } else if (field.isAnnotationPresent(BooleanConfigurationValidatorAnnotation.class)) { + assertEquals(true, configService.validateBoolean(field)); + } + } + } + + @Test + public void testConfigServiceImplAnnotatedFieldsInitialized() throws Exception { + // test that all the ConfigurationServiceImpl annotated fields have been initialized in the constructor + assertEquals(DEFAULT_WRITE_BUFFER_SIZE, configService.getWriteBufferSize()); + assertEquals(DEFAULT_READ_BUFFER_SIZE, configService.getReadBufferSize()); + assertEquals(DEFAULT_MIN_BACKOFF_INTERVAL, configService.getMinBackoffIntervalMilliseconds()); + assertEquals(DEFAULT_MAX_BACKOFF_INTERVAL, configService.getMaxBackoffIntervalMilliseconds()); + assertEquals(DEFAULT_BACKOFF_INTERVAL, configService.getBackoffIntervalMilliseconds()); + assertEquals(DEFAULT_MAX_RETRY_ATTEMPTS, configService.getMaxIoRetries()); + assertEquals(MAX_AZURE_BLOCK_SIZE, configService.getAzureBlockSize()); + assertEquals(AZURE_BLOCK_LOCATION_HOST_DEFAULT, configService.getAzureBlockLocationHost()); + } + + @Test + public void testGetAccountKey() throws Exception { + String accountKey = configService.getStorageAccountKey("testaccount1.blob.core.windows.net"); + assertEquals(this.encodedAccountKey, accountKey); + } + + @Test (expected = ConfigurationPropertyNotFoundException.class) + public void testGetAccountKeyWithNonExistingAccountName() throws Exception { + configService.getStorageAccountKey("bogusAccountName"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/package-info.java new file mode 100644 index 0000000..97c1d71 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/services/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +package org.apache.hadoop.fs.azurebfs.services; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java new file mode 100644 index 0000000..e71e31d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/TestUriUtils.java @@ -0,0 +1,48 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs.utils; + +import org.junit.Assert; +import org.junit.Test; + +/** + * Test ABFS UriUtils. + */ +public final class TestUriUtils { + @Test + public void testIfUriContainsAbfs() throws Exception { + Assert.assertTrue(UriUtils.containsAbfsUrl("abfs.dfs.core.windows.net")); + Assert.assertTrue(UriUtils.containsAbfsUrl("abfs.dfs.preprod.core.windows.net")); + Assert.assertFalse(UriUtils.containsAbfsUrl("abfs.dfs.cores.windows.net")); + Assert.assertFalse(UriUtils.containsAbfsUrl("")); + Assert.assertFalse(UriUtils.containsAbfsUrl(null)); + Assert.assertFalse(UriUtils.containsAbfsUrl("abfs.dfs.cores.windows.net")); + Assert.assertFalse(UriUtils.containsAbfsUrl("xhdfs.blob.core.windows.net")); + } + + @Test + public void testExtractRawAccountName() throws Exception { + Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.core.windows.net")); + Assert.assertEquals("abfs", UriUtils.extractRawAccountFromAccountName("abfs.dfs.preprod.core.windows.net")); + Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net")); + Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("")); + Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName(null)); + Assert.assertEquals(null, UriUtils.extractRawAccountFromAccountName("abfs.dfs.cores.windows.net")); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/package-info.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/package-info.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/package-info.java new file mode 100644 index 0000000..d8cc940 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/utils/package-info.java @@ -0,0 +1,22 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +@InterfaceAudience.Private +@InterfaceStability.Evolving +package org.apache.hadoop.fs.azurebfs.utils; +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml b/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml new file mode 100644 index 0000000..caf8677 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/resources/abfs.xml @@ -0,0 +1,64 @@ +<!-- + ~ Licensed to the Apache Software Foundation (ASF) under one + ~ or more contributor license agreements. See the NOTICE file + ~ distributed with this work for additional information + ~ regarding copyright ownership. The ASF licenses this file + ~ to you under the Apache License, Version 2.0 (the + ~ "License"); you may not use this file except in compliance + ~ with the License. You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, software + ~ distributed under the License is distributed on an "AS IS" BASIS, + ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + ~ See the License for the specific language governing permissions and + ~ limitations under the License. + --> +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + <property> + <name>fs.contract.test.root-tests-enabled</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.supports-append</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.supports-seek</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.rename-overwrites-dest</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.rename-returns-false-if-source-missing</name> + <value>true</value> + </property> + + <property> + <name>fs.contract.rename-creates-dest-dirs</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.supports-settimes</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.supports-concat</name> + <value>false</value> + </property> + + <property> + <name>fs.contract.supports-getfilestatus</name> + <value>true</value> + </property> +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml b/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml new file mode 100644 index 0000000..508e5f6 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/resources/azure-bfs-test.xml @@ -0,0 +1,76 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> + +<?xml-stylesheet type="text/xsl" href="configuration.xsl"?> +<configuration xmlns:xi="http://www.w3.org/2001/XInclude";> + + <!-- + <property> + <name>fs.azure.test.account.name</name> + <value>{YOURACCOUNT}</value> + </property> + + <property> + <name>fs.azure.account.key.{YOURACCOUNT}.dfs.core.windows.net</name> + <value>{ACCOUNTKEY}</value> + </property> + + <property> + <name>fs.azure.test.account.key.{YOURACCOUNT}.dfs.core.windows.net</name> + <value>{ACCOUNTKEY}</value> + </property> + + <property> + <name>fs.contract.test.fs.abfs</name> + <value>abfs://{CONTAINERNAME}@{ACCOUNTNAME}.dfs.core.windows.net/value> + <description>The name of the azure file system for testing.</description> + </property> + + --> + <!-- Save above configuration properties in a separate file named --> + <!-- azure-bfs-auth-keys.xml in the same directory as this file. --> + <!-- DO NOT ADD azure-bfs-auth-keys.xml TO REVISION CONTROL. The keys to your --> + <!-- Azure Storage account are a secret and must not be shared. --> + + <include xmlns="http://www.w3.org/2001/XInclude"; href="azure-bfs-auth-keys.xml"> + <fallback /> + </include> + + <!--<property>--> + <!--<name>fs.azure.test.host.name</name>--> + <!--<value>{HOSTNAME}</value>--> + <!--</property>--> + + <!--<property>--> + <!--<name>fs.azure.test.host.port</name>--> + <!--<value>{PORT}</value>--> + <!--</property>--> + + <!--<property>--> + <!--<name>fs.azure.abfs.emulator.enabled</name>--> + <!--<value>false</value>--> + <!--</property>--> + + <property> + <name>fs.AbstractFileSystem.abfs.impl</name> + <value>org.apache.hadoop.fs.azurebfs.Abfs</value> + </property> + + <property> + <name>fs.AbstractFileSystem.abfss.impl</name> + <value>org.apache.hadoop.fs.azurebfs.Abfss</value> + </property> + +</configuration> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/f044deed/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties index a5e0c4f..fd0cd9d 100644 --- a/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties +++ b/hadoop-tools/hadoop-azure/src/test/resources/log4j.properties @@ -24,3 +24,5 @@ log4j.appender.stdout.layout.ConversionPattern=%d{ISO8601} %-5p [%t]: %c{2} (%F: log4j.logger.org.apache.hadoop.fs.azure.AzureFileSystemThreadPoolExecutor=DEBUG log4j.logger.org.apache.hadoop.fs.azure.BlockBlobAppendStream=DEBUG +log4j.logger.org.apache.hadoop.fs.azurebfs.contracts.services.TracingService=TRACE +log4j.logger.org.apache.hadoop.fs.azurebfs.services.AbfsClient=DEBUG --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
