http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java deleted file mode 100644 index 672ed9c..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestWasbUriAndConfiguration.java +++ /dev/null @@ -1,617 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure; - -import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.fail; -import static org.junit.Assume.assumeNotNull; - -import java.io.ByteArrayInputStream; -import java.io.DataInputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.net.URI; -import java.util.Date; -import java.util.EnumSet; -import java.io.File; - -import org.apache.hadoop.security.ProviderUtils; -import org.apache.hadoop.security.alias.CredentialProvider; -import org.apache.hadoop.security.alias.CredentialProviderFactory; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.AbstractFileSystem; -import org.apache.hadoop.fs.FileContext; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.UnsupportedFileSystemException; -import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount.CreateOptions; -import org.junit.After; -import org.junit.Assert; -import org.junit.Assume; -import org.junit.Before; -import org.junit.Rule; -import org.junit.Test; -import org.junit.rules.TemporaryFolder; - -import com.microsoft.azure.storage.blob.CloudBlobContainer; -import com.microsoft.azure.storage.blob.CloudBlockBlob; - -public class TestWasbUriAndConfiguration { - - private static final int FILE_SIZE = 4096; - private static final String PATH_DELIMITER = "/"; - - protected String accountName; - protected String accountKey; - protected static Configuration conf = null; - private boolean runningInSASMode = false; - @Rule - public final TemporaryFolder tempDir = new TemporaryFolder(); - - private AzureBlobStorageTestAccount testAccount; - - @After - public void tearDown() throws Exception { - if (testAccount != null) { - testAccount.cleanup(); - testAccount = null; - } - } - - @Before - public void setMode() { - runningInSASMode = AzureBlobStorageTestAccount.createTestConfiguration(). - getBoolean(AzureNativeFileSystemStore.KEY_USE_SECURE_MODE, false); - } - - private boolean validateIOStreams(Path filePath) throws IOException { - // Capture the file system from the test account. - FileSystem fs = testAccount.getFileSystem(); - return validateIOStreams(fs, filePath); - } - - private boolean validateIOStreams(FileSystem fs, Path filePath) - throws IOException { - - // Create and write a file - OutputStream outputStream = fs.create(filePath); - outputStream.write(new byte[FILE_SIZE]); - outputStream.close(); - - // Return true if the the count is equivalent to the file size. - return (FILE_SIZE == readInputStream(fs, filePath)); - } - - private int readInputStream(Path filePath) throws IOException { - // Capture the file system from the test account. - FileSystem fs = testAccount.getFileSystem(); - return readInputStream(fs, filePath); - } - - private int readInputStream(FileSystem fs, Path filePath) throws IOException { - // Read the file - InputStream inputStream = fs.open(filePath); - int count = 0; - while (inputStream.read() >= 0) { - count++; - } - inputStream.close(); - - // Return true if the the count is equivalent to the file size. - return count; - } - - // Positive tests to exercise making a connection with to Azure account using - // account key. - @Test - public void testConnectUsingKey() throws Exception { - - testAccount = AzureBlobStorageTestAccount.create(); - assumeNotNull(testAccount); - - // Validate input and output on the connection. - assertTrue(validateIOStreams(new Path("/wasb_scheme"))); - } - - @Test - public void testConnectUsingSAS() throws Exception { - - Assume.assumeFalse(runningInSASMode); - // Create the test account with SAS credentials. - testAccount = AzureBlobStorageTestAccount.create("", - EnumSet.of(CreateOptions.UseSas, CreateOptions.CreateContainer)); - assumeNotNull(testAccount); - // Validate input and output on the connection. - // NOTE: As of 4/15/2013, Azure Storage has a deficiency that prevents the - // full scenario from working (CopyFromBlob doesn't work with SAS), so - // just do a minor check until that is corrected. - assertFalse(testAccount.getFileSystem().exists(new Path("/IDontExist"))); - //assertTrue(validateIOStreams(new Path("/sastest.txt"))); - } - - @Test - public void testConnectUsingSASReadonly() throws Exception { - - Assume.assumeFalse(runningInSASMode); - // Create the test account with SAS credentials. - testAccount = AzureBlobStorageTestAccount.create("", EnumSet.of( - CreateOptions.UseSas, CreateOptions.CreateContainer, - CreateOptions.Readonly)); - assumeNotNull(testAccount); - - // Create a blob in there - final String blobKey = "blobForReadonly"; - CloudBlobContainer container = testAccount.getRealContainer(); - CloudBlockBlob blob = container.getBlockBlobReference(blobKey); - ByteArrayInputStream inputStream = new ByteArrayInputStream(new byte[] { 1, - 2, 3 }); - blob.upload(inputStream, 3); - inputStream.close(); - - // Make sure we can read it from the file system - Path filePath = new Path("/" + blobKey); - FileSystem fs = testAccount.getFileSystem(); - assertTrue(fs.exists(filePath)); - byte[] obtained = new byte[3]; - DataInputStream obtainedInputStream = fs.open(filePath); - obtainedInputStream.readFully(obtained); - obtainedInputStream.close(); - assertEquals(3, obtained[2]); - } - - @Test - public void testConnectUsingAnonymous() throws Exception { - - // Create test account with anonymous credentials - testAccount = AzureBlobStorageTestAccount.createAnonymous("testWasb.txt", - FILE_SIZE); - assumeNotNull(testAccount); - - // Read the file from the public folder using anonymous credentials. - assertEquals(FILE_SIZE, readInputStream(new Path("/testWasb.txt"))); - } - - @Test - public void testConnectToEmulator() throws Exception { - testAccount = AzureBlobStorageTestAccount.createForEmulator(); - assumeNotNull(testAccount); - assertTrue(validateIOStreams(new Path("/testFile"))); - } - - /** - * Tests that we can connect to fully qualified accounts outside of - * blob.core.windows.net - */ - @Test - public void testConnectToFullyQualifiedAccountMock() throws Exception { - Configuration conf = new Configuration(); - AzureBlobStorageTestAccount.setMockAccountKey(conf, - "mockAccount.mock.authority.net"); - AzureNativeFileSystemStore store = new AzureNativeFileSystemStore(); - MockStorageInterface mockStorage = new MockStorageInterface(); - store.setAzureStorageInteractionLayer(mockStorage); - NativeAzureFileSystem fs = new NativeAzureFileSystem(store); - fs.initialize( - new URI("wasb://[email protected]"), conf); - fs.createNewFile(new Path("/x")); - assertTrue(mockStorage.getBackingStore().exists( - "http://mockAccount.mock.authority.net/mockContainer/x";)); - fs.close(); - } - - public void testConnectToRoot() throws Exception { - - // Set up blob names. - final String blobPrefix = String.format("wasbtests-%s-%tQ-blob", - System.getProperty("user.name"), new Date()); - final String inblobName = blobPrefix + "_In" + ".txt"; - final String outblobName = blobPrefix + "_Out" + ".txt"; - - // Create test account with default root access. - testAccount = AzureBlobStorageTestAccount.createRoot(inblobName, FILE_SIZE); - assumeNotNull(testAccount); - - // Read the file from the default container. - assertEquals(FILE_SIZE, readInputStream(new Path(PATH_DELIMITER - + inblobName))); - - try { - // Capture file system. - FileSystem fs = testAccount.getFileSystem(); - - // Create output path and open an output stream to the root folder. - Path outputPath = new Path(PATH_DELIMITER + outblobName); - OutputStream outputStream = fs.create(outputPath); - fail("Expected an AzureException when writing to root folder."); - outputStream.write(new byte[FILE_SIZE]); - outputStream.close(); - } catch (AzureException e) { - assertTrue(true); - } catch (Exception e) { - String errMsg = String.format( - "Expected AzureException but got %s instead.", e); - assertTrue(errMsg, false); - } - } - - // Positive tests to exercise throttling I/O path. Connections are made to an - // Azure account using account key. - // - public void testConnectWithThrottling() throws Exception { - - testAccount = AzureBlobStorageTestAccount.createThrottled(); - - // Validate input and output on the connection. - assertTrue(validateIOStreams(new Path("/wasb_scheme"))); - } - - /** - * Creates a file and writes a single byte with the given value in it. - */ - private static void writeSingleByte(FileSystem fs, Path testFile, int toWrite) - throws Exception { - OutputStream outputStream = fs.create(testFile); - outputStream.write(toWrite); - outputStream.close(); - } - - /** - * Reads the file given and makes sure that it's a single-byte file with the - * given value in it. - */ - private static void assertSingleByteValue(FileSystem fs, Path testFile, - int expectedValue) throws Exception { - InputStream inputStream = fs.open(testFile); - int byteRead = inputStream.read(); - assertTrue("File unexpectedly empty: " + testFile, byteRead >= 0); - assertTrue("File has more than a single byte: " + testFile, - inputStream.read() < 0); - inputStream.close(); - assertEquals("Unxpected content in: " + testFile, expectedValue, byteRead); - } - - @Test - public void testMultipleContainers() throws Exception { - AzureBlobStorageTestAccount firstAccount = AzureBlobStorageTestAccount - .create("first"), secondAccount = AzureBlobStorageTestAccount - .create("second"); - assumeNotNull(firstAccount); - assumeNotNull(secondAccount); - try { - FileSystem firstFs = firstAccount.getFileSystem(), - secondFs = secondAccount.getFileSystem(); - Path testFile = new Path("/testWasb"); - assertTrue(validateIOStreams(firstFs, testFile)); - assertTrue(validateIOStreams(secondFs, testFile)); - // Make sure that we're really dealing with two file systems here. - writeSingleByte(firstFs, testFile, 5); - writeSingleByte(secondFs, testFile, 7); - assertSingleByteValue(firstFs, testFile, 5); - assertSingleByteValue(secondFs, testFile, 7); - } finally { - firstAccount.cleanup(); - secondAccount.cleanup(); - } - } - - @Test - public void testDefaultKeyProvider() throws Exception { - Configuration conf = new Configuration(); - String account = "testacct"; - String key = "testkey"; - - conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, key); - - String result = AzureNativeFileSystemStore.getAccountKeyFromConfiguration( - account, conf); - assertEquals(key, result); - } - - @Test - public void testCredsFromCredentialProvider() throws Exception { - - Assume.assumeFalse(runningInSASMode); - String account = "testacct"; - String key = "testkey"; - // set up conf to have a cred provider - final Configuration conf = new Configuration(); - final File file = tempDir.newFile("test.jks"); - final URI jks = ProviderUtils.nestURIForLocalJavaKeyStoreProvider( - file.toURI()); - conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, - jks.toString()); - - provisionAccountKey(conf, account, key); - - // also add to configuration as clear text that should be overridden - conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, - key + "cleartext"); - - String result = AzureNativeFileSystemStore.getAccountKeyFromConfiguration( - account, conf); - // result should contain the credential provider key not the config key - assertEquals("AccountKey incorrect.", key, result); - } - - void provisionAccountKey( - final Configuration conf, String account, String key) throws Exception { - // add our creds to the provider - final CredentialProvider provider = - CredentialProviderFactory.getProviders(conf).get(0); - provider.createCredentialEntry( - SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, key.toCharArray()); - provider.flush(); - } - - @Test - public void testValidKeyProvider() throws Exception { - Configuration conf = new Configuration(); - String account = "testacct"; - String key = "testkey"; - - conf.set(SimpleKeyProvider.KEY_ACCOUNT_KEY_PREFIX + account, key); - conf.setClass("fs.azure.account.keyprovider." + account, - SimpleKeyProvider.class, KeyProvider.class); - String result = AzureNativeFileSystemStore.getAccountKeyFromConfiguration( - account, conf); - assertEquals(key, result); - } - - @Test - public void testInvalidKeyProviderNonexistantClass() throws Exception { - Configuration conf = new Configuration(); - String account = "testacct"; - - conf.set("fs.azure.account.keyprovider." + account, - "org.apache.Nonexistant.Class"); - try { - AzureNativeFileSystemStore.getAccountKeyFromConfiguration(account, conf); - Assert.fail("Nonexistant key provider class should have thrown a " - + "KeyProviderException"); - } catch (KeyProviderException e) { - } - } - - @Test - public void testInvalidKeyProviderWrongClass() throws Exception { - Configuration conf = new Configuration(); - String account = "testacct"; - - conf.set("fs.azure.account.keyprovider." + account, "java.lang.String"); - try { - AzureNativeFileSystemStore.getAccountKeyFromConfiguration(account, conf); - Assert.fail("Key provider class that doesn't implement KeyProvider " - + "should have thrown a KeyProviderException"); - } catch (KeyProviderException e) { - } - } - - /** - * Tests the cases when the URI is specified with no authority, i.e. - * wasb:///path/to/file. - */ - @Test - public void testNoUriAuthority() throws Exception { - // For any combination of default FS being asv(s)/wasb(s)://c@a/ and - // the actual URI being asv(s)/wasb(s):///, it should work. - - String[] wasbAliases = new String[] { "wasb", "wasbs" }; - for (String defaultScheme : wasbAliases) { - for (String wantedScheme : wasbAliases) { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI(defaultScheme, authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - // Add references to file system implementations for wasb and wasbs. - conf.addResource("azure-test.xml"); - URI wantedUri = new URI(wantedScheme + ":///random/path"); - NativeAzureFileSystem obtained = (NativeAzureFileSystem) FileSystem - .get(wantedUri, conf); - assertNotNull(obtained); - assertEquals(new URI(wantedScheme, authority, null, null, null), - obtained.getUri()); - // Make sure makeQualified works as expected - Path qualified = obtained.makeQualified(new Path(wantedUri)); - assertEquals(new URI(wantedScheme, authority, wantedUri.getPath(), - null, null), qualified.toUri()); - // Cleanup for the next iteration to not cache anything in FS - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - // If the default FS is not a WASB FS, then specifying a URI without - // authority for the Azure file system should throw. - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - conf.set(FS_DEFAULT_NAME_KEY, "file:///"); - try { - FileSystem.get(new URI("wasb:///random/path"), conf); - fail("Should've thrown."); - } catch (IllegalArgumentException e) { - } - } - - @Test - public void testWasbAsDefaultFileSystemHasNoPort() throws Exception { - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasb", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - conf.addResource("azure-test.xml"); - - FileSystem fs = FileSystem.get(conf); - assertTrue(fs instanceof NativeAzureFileSystem); - assertEquals(-1, fs.getUri().getPort()); - - AbstractFileSystem afs = FileContext.getFileContext(conf) - .getDefaultFileSystem(); - assertTrue(afs instanceof Wasb); - assertEquals(-1, afs.getUri().getPort()); - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - - /** - * Tests the cases when the scheme specified is 'wasbs'. - */ - @Test - public void testAbstractFileSystemImplementationForWasbsScheme() throws Exception { - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs"); - conf.addResource("azure-test.xml"); - - FileSystem fs = FileSystem.get(conf); - assertTrue(fs instanceof NativeAzureFileSystem); - assertEquals("wasbs", fs.getScheme()); - - AbstractFileSystem afs = FileContext.getFileContext(conf) - .getDefaultFileSystem(); - assertTrue(afs instanceof Wasbs); - assertEquals(-1, afs.getUri().getPort()); - assertEquals("wasbs", afs.getUri().getScheme()); - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - - @Test - public void testNoAbstractFileSystemImplementationSpecifiedForWasbsScheme() throws Exception { - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - - FileSystem fs = FileSystem.get(conf); - assertTrue(fs instanceof NativeAzureFileSystem); - assertEquals("wasbs", fs.getScheme()); - - // should throw if 'fs.AbstractFileSystem.wasbs.impl'' is not specified - try{ - FileContext.getFileContext(conf).getDefaultFileSystem(); - fail("Should've thrown."); - }catch(UnsupportedFileSystemException e){ - } - - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } - - @Test - public void testCredentialProviderPathExclusions() throws Exception { - String providerPath = - "user:///,jceks://wasb/user/hrt_qa/sqoopdbpasswd.jceks," + - "jceks://[email protected]/my/path/test.jceks"; - Configuration config = new Configuration(); - config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, - providerPath); - String newPath = "user:///,jceks://[email protected]/my/path/test.jceks"; - - excludeAndTestExpectations(config, newPath); - } - - @Test - public void testExcludeAllProviderTypesFromConfig() throws Exception { - String providerPath = - "jceks://wasb/tmp/test.jceks," + - "jceks://wasb@/my/path/test.jceks"; - Configuration config = new Configuration(); - config.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, - providerPath); - String newPath = null; - - excludeAndTestExpectations(config, newPath); - } - - void excludeAndTestExpectations(Configuration config, String newPath) - throws Exception { - Configuration conf = ProviderUtils.excludeIncompatibleCredentialProviders( - config, NativeAzureFileSystem.class); - String effectivePath = conf.get( - CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, null); - assertEquals(newPath, effectivePath); - } - - @Test - public void testUserAgentConfig() throws Exception { - // Set the user agent - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs"); - - conf.set(AzureNativeFileSystemStore.USER_AGENT_ID_KEY, "TestClient"); - - FileSystem fs = FileSystem.get(conf); - AbstractFileSystem afs = FileContext.getFileContext(conf).getDefaultFileSystem(); - - assertTrue(afs instanceof Wasbs); - assertEquals(-1, afs.getUri().getPort()); - assertEquals("wasbs", afs.getUri().getScheme()); - - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - - // Unset the user agent - try { - testAccount = AzureBlobStorageTestAccount.createMock(); - Configuration conf = testAccount.getFileSystem().getConf(); - String authority = testAccount.getFileSystem().getUri().getAuthority(); - URI defaultUri = new URI("wasbs", authority, null, null, null); - conf.set(FS_DEFAULT_NAME_KEY, defaultUri.toString()); - conf.set("fs.AbstractFileSystem.wasbs.impl", "org.apache.hadoop.fs.azure.Wasbs"); - - conf.unset(AzureNativeFileSystemStore.USER_AGENT_ID_KEY); - - FileSystem fs = FileSystem.get(conf); - AbstractFileSystem afs = FileContext.getFileContext(conf).getDefaultFileSystem(); - assertTrue(afs instanceof Wasbs); - assertEquals(-1, afs.getUri().getPort()); - assertEquals("wasbs", afs.getUri().getScheme()); - - } finally { - testAccount.cleanup(); - FileSystem.closeAll(); - } - } -}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java new file mode 100644 index 0000000..fd21bd2 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractAppend.java @@ -0,0 +1,41 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractAppendTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; +import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; + +/** + * Append test, skipping one of them. + */ + +public class ITestAzureNativeContractAppend extends AbstractContractAppendTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } + + @Override + public void testRenameFileBeingAppended() throws Throwable { + skip("Skipping as renaming an opened file is not supported"); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java new file mode 100644 index 0000000..0ac046a --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractCreate.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractCreateTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractCreate extends AbstractContractCreateTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java new file mode 100644 index 0000000..4c6dd48 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDelete.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractDelete extends AbstractContractDeleteTest { + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java new file mode 100644 index 0000000..7769570 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractDistCp.java @@ -0,0 +1,47 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.azure.integration.AzureTestConstants; +import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; + +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.assumeScaleTestsEnabled; + +/** + * Contract test suite covering WASB integration with DistCp. + */ +public class ITestAzureNativeContractDistCp extends AbstractContractDistCpTest { + + @Override + protected int getTestTimeoutMillis() { + return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS; + } + + @Override + protected NativeAzureFileSystemContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } + + @Override + public void setup() throws Exception { + super.setup(); + assumeScaleTestsEnabled(getContract().getConf()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java new file mode 100644 index 0000000..9c09c0d --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractGetFileStatus.java @@ -0,0 +1,35 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractGetFileStatus + extends AbstractContractGetFileStatusTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java new file mode 100644 index 0000000..71654b8 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractMkdir.java @@ -0,0 +1,33 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractMkdir extends AbstractContractMkdirTest { + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java new file mode 100644 index 0000000..0b174e6 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractOpen.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractOpenTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractOpen extends AbstractContractOpenTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java new file mode 100644 index 0000000..474b874 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractRename.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractRenameTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractRename extends AbstractContractRenameTest { + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java new file mode 100644 index 0000000..673d5f8 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/ITestAzureNativeContractSeek.java @@ -0,0 +1,34 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.contract; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.contract.AbstractContractSeekTest; +import org.apache.hadoop.fs.contract.AbstractFSContract; + +/** + * Contract test. + */ +public class ITestAzureNativeContractSeek extends AbstractContractSeekTest{ + + @Override + protected AbstractFSContract createContract(Configuration conf) { + return new NativeAzureFileSystemContract(conf); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java index 28c13ea..a264aca 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/NativeAzureFileSystemContract.java @@ -18,15 +18,21 @@ package org.apache.hadoop.fs.azure.contract; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.integration.AzureTestUtils; import org.apache.hadoop.fs.contract.AbstractBondedFSContract; +/** + * Azure Contract. Test paths are created using any maven fork + * identifier, if defined. This guarantees paths unique to tests + * running in parallel. + */ public class NativeAzureFileSystemContract extends AbstractBondedFSContract { public static final String CONTRACT_XML = "wasb.xml"; - protected NativeAzureFileSystemContract(Configuration conf) { - super(conf); - //insert the base features + public NativeAzureFileSystemContract(Configuration conf) { + super(conf); //insert the base features addConfResource(CONTRACT_XML); } @@ -34,4 +40,9 @@ public class NativeAzureFileSystemContract extends AbstractBondedFSContract { public String getScheme() { return "wasb"; } -} \ No newline at end of file + + @Override + public Path getTestPath() { + return AzureTestUtils.createTestPath(super.getTestPath()); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractAppend.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractAppend.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractAppend.java deleted file mode 100644 index 8a2341e..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractAppend.java +++ /dev/null @@ -1,37 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractAppendTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; -import org.junit.Test; -import static org.apache.hadoop.fs.contract.ContractTestUtils.skip; - -public class TestAzureNativeContractAppend extends AbstractContractAppendTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } - - @Override - public void testRenameFileBeingAppended() throws Throwable { - skip("Skipping as renaming an opened file is not supported"); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractCreate.java deleted file mode 100644 index 531552d..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractCreate.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractCreateTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractCreate extends AbstractContractCreateTest{ - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDelete.java deleted file mode 100644 index 5e5c13b..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDelete.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractDeleteTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractDelete extends AbstractContractDeleteTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDistCp.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDistCp.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDistCp.java deleted file mode 100644 index a3750d4..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractDistCp.java +++ /dev/null @@ -1,33 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.tools.contract.AbstractContractDistCpTest; - -/** - * Contract test suite covering WASB integration with DistCp. - */ -public class TestAzureNativeContractDistCp extends AbstractContractDistCpTest { - - @Override - protected NativeAzureFileSystemContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractGetFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractGetFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractGetFileStatus.java deleted file mode 100644 index b0c59ee..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractGetFileStatus.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractGetFileStatusTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractGetFileStatus extends AbstractContractGetFileStatusTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractMkdir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractMkdir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractMkdir.java deleted file mode 100644 index 36df041..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractMkdir.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractMkdirTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractMkdir extends AbstractContractMkdirTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractOpen.java deleted file mode 100644 index d5147ac..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractOpen.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractOpenTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractOpen extends AbstractContractOpenTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractRename.java deleted file mode 100644 index 4d8b2b5..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractRename.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractRenameTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractRename extends AbstractContractRenameTest { - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractSeek.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractSeek.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractSeek.java deleted file mode 100644 index 30046dc..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/contract/TestAzureNativeContractSeek.java +++ /dev/null @@ -1,30 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azure.contract; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.contract.AbstractContractSeekTest; -import org.apache.hadoop.fs.contract.AbstractFSContract; - -public class TestAzureNativeContractSeek extends AbstractContractSeekTest{ - @Override - protected AbstractFSContract createContract(Configuration conf) { - return new NativeAzureFileSystemContract(conf); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java new file mode 100644 index 0000000..062d073 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AbstractAzureScaleTest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.integration; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.hadoop.fs.azure.AbstractWasbTestBase; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; + +import static org.apache.hadoop.fs.azure.integration.AzureTestUtils.*; + +/** + * Scale tests are only executed if the scale profile + * is set; the setup method will check this and skip + * tests if not. + * + */ +public abstract class AbstractAzureScaleTest + extends AbstractWasbTestBase implements Sizes { + + protected static final Logger LOG = + LoggerFactory.getLogger(AbstractAzureScaleTest.class); + + @Override + protected int getTestTimeoutMillis() { + return AzureTestConstants.SCALE_TEST_TIMEOUT_MILLIS; + } + + @Override + public void setUp() throws Exception { + super.setUp(); + LOG.debug("Scale test operation count = {}", getOperationCount()); + assumeScaleTestsEnabled(getConfiguration()); + } + + /** + * Create the test account. + * @return a test account + * @throws Exception on any failure to create the account. + */ + protected AzureBlobStorageTestAccount createTestAccount() throws Exception { + return AzureBlobStorageTestAccount.create(createConfiguration()); + } + + protected long getOperationCount() { + return getConfiguration().getLong(KEY_OPERATION_COUNT, + DEFAULT_OPERATION_COUNT); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java new file mode 100644 index 0000000..0b72f06 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestConstants.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.integration; + +import org.apache.hadoop.fs.Path; + +/** + * Constants for the Azure tests. + */ +public interface AzureTestConstants { + + /** + * Prefix for any cross-filesystem scale test options. + */ + String SCALE_TEST = "scale.test."; + + /** + * Prefix for wasb-specific scale tests. + */ + String AZURE_SCALE_TEST = "fs.azure.scale.test."; + + /** + * Prefix for FS wasb tests. + */ + String TEST_FS_WASB = "test.fs.azure."; + + /** + * Name of the test filesystem. + */ + String TEST_FS_WASB_NAME = TEST_FS_WASB + "name"; + + /** + * Tell tests that they are being executed in parallel: {@value}. + */ + String KEY_PARALLEL_TEST_EXECUTION = "test.parallel.execution"; + + /** + * A property set to true in maven if scale tests are enabled: {@value}. + */ + String KEY_SCALE_TESTS_ENABLED = AZURE_SCALE_TEST + "enabled"; + + /** + * The number of operations to perform: {@value}. + */ + String KEY_OPERATION_COUNT = SCALE_TEST + "operation.count"; + + /** + * The number of directory operations to perform: {@value}. + */ + String KEY_DIRECTORY_COUNT = SCALE_TEST + "directory.count"; + + /** + * The readahead buffer: {@value}. + */ + String KEY_READ_BUFFER_SIZE = AZURE_SCALE_TEST + "read.buffer.size"; + + int DEFAULT_READ_BUFFER_SIZE = 16384; + + /** + * Key for a multi MB test file: {@value}. + */ + String KEY_CSVTEST_FILE = AZURE_SCALE_TEST + "csvfile"; + + /** + * Default path for the multi MB test file: {@value}. + */ + String DEFAULT_CSVTEST_FILE = "wasb://[email protected]/network_intrusion_detection.csv"; + + /** + * Name of the property to define the timeout for scale tests: {@value}. + * Measured in seconds. + */ + String KEY_TEST_TIMEOUT = AZURE_SCALE_TEST + "timeout"; + + /** + * Name of the property to define the file size for the huge file + * tests: {@value}. + * Measured in KB; a suffix like "M", or "G" will change the unit. + */ + String KEY_HUGE_FILESIZE = AZURE_SCALE_TEST + "huge.filesize"; + + /** + * Name of the property to define the partition size for the huge file + * tests: {@value}. + * Measured in KB; a suffix like "M", or "G" will change the unit. + */ + String KEY_HUGE_PARTITION_SIZE = AZURE_SCALE_TEST + "huge.partitionsize"; + + /** + * The default huge size is small âfull 5GB+ scale tests are something + * to run in long test runs on EC2 VMs. {@value}. + */ + String DEFAULT_HUGE_FILESIZE = "10M"; + + /** + * The default number of operations to perform: {@value}. + */ + long DEFAULT_OPERATION_COUNT = 2005; + + /** + * Default number of directories to create when performing + * directory performance/scale tests. + */ + int DEFAULT_DIRECTORY_COUNT = 2; + + /** + * Default policy on scale tests: {@value}. + */ + boolean DEFAULT_SCALE_TESTS_ENABLED = false; + + /** + * Fork ID passed down from maven if the test is running in parallel. + */ + String TEST_UNIQUE_FORK_ID = "test.unique.fork.id"; + + /** + * Timeout in Milliseconds for standard tests: {@value}. + */ + int AZURE_TEST_TIMEOUT = 10 * 60 * 1000; + + /** + * Timeout in Seconds for Scale Tests: {@value}. + */ + int SCALE_TEST_TIMEOUT_SECONDS = 30 * 60; + + int SCALE_TEST_TIMEOUT_MILLIS = SCALE_TEST_TIMEOUT_SECONDS * 1000; + + + + String ACCOUNT_KEY_PROPERTY_NAME + = "fs.azure.account.key."; + String SAS_PROPERTY_NAME = "fs.azure.sas."; + String TEST_CONFIGURATION_FILE_NAME = "azure-test.xml"; + String TEST_ACCOUNT_NAME_PROPERTY_NAME + = "fs.azure.test.account.name"; + String MOCK_ACCOUNT_NAME + = "mockAccount.blob.core.windows.net"; + String MOCK_CONTAINER_NAME = "mockContainer"; + String WASB_AUTHORITY_DELIMITER = "@"; + String WASB_SCHEME = "wasb"; + String PATH_DELIMITER = "/"; + String AZURE_ROOT_CONTAINER = "$root"; + String MOCK_WASB_URI = "wasb://" + MOCK_CONTAINER_NAME + + WASB_AUTHORITY_DELIMITER + MOCK_ACCOUNT_NAME + "/"; + String USE_EMULATOR_PROPERTY_NAME + = "fs.azure.test.emulator"; + + String KEY_DISABLE_THROTTLING + = "fs.azure.disable.bandwidth.throttling"; + String KEY_READ_TOLERATE_CONCURRENT_APPEND + = "fs.azure.io.read.tolerate.concurrent.append"; + /** + * Path for page blobs: {@value}. + */ + String DEFAULT_PAGE_BLOB_DIRECTORY = "pageBlobs"; + + String DEFAULT_ATOMIC_RENAME_DIRECTORIES + = "/atomicRenameDir1,/atomicRenameDir2"; + + /** + * Base directory for page blobs. + */ + Path PAGE_BLOB_DIR = new Path("/" + DEFAULT_PAGE_BLOB_DIRECTORY); +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/3fbcf763/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java new file mode 100644 index 0000000..2fbbcd1 --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/integration/AzureTestUtils.java @@ -0,0 +1,479 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azure.integration; + +import java.io.IOException; +import java.net.URI; +import java.util.List; + +import org.junit.Assert; +import org.junit.Assume; +import org.junit.internal.AssumptionViolatedException; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import org.apache.commons.lang.StringUtils; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileContext; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.azure.AzureBlobStorageTestAccount; +import org.apache.hadoop.fs.azure.NativeAzureFileSystem; + +import static org.apache.hadoop.fs.azure.integration.AzureTestConstants.*; +import static org.apache.hadoop.test.MetricsAsserts.getLongCounter; +import static org.apache.hadoop.test.MetricsAsserts.getLongGauge; +import static org.apache.hadoop.test.MetricsAsserts.getMetrics; + +/** + * Utilities for the Azure tests. Based on {@code S3ATestUtils}, so + * (initially) has unused method. + */ +public final class AzureTestUtils extends Assert { + private static final Logger LOG = LoggerFactory.getLogger( + AzureTestUtils.class); + + /** + * Value to set a system property to (in maven) to declare that + * a property has been unset. + */ + public static final String UNSET_PROPERTY = "unset"; + + /** + * Create the test filesystem. + * + * If the test.fs.wasb.name property is not set, this will + * raise a JUnit assumption exception + * + * @param conf configuration + * @return the FS + * @throws IOException IO Problems + * @throws AssumptionViolatedException if the FS is not named + */ + public static NativeAzureFileSystem createTestFileSystem(Configuration conf) + throws IOException { + + String fsname = conf.getTrimmed(TEST_FS_WASB_NAME, ""); + + boolean liveTest = !StringUtils.isEmpty(fsname); + URI testURI = null; + if (liveTest) { + testURI = URI.create(fsname); + liveTest = testURI.getScheme().equals(WASB_SCHEME); + } + if (!liveTest) { + // Skip the test + throw new AssumptionViolatedException( + "No test filesystem in " + TEST_FS_WASB_NAME); + } + NativeAzureFileSystem fs1 = new NativeAzureFileSystem(); + fs1.initialize(testURI, conf); + return fs1; + } + + /** + * Create a file context for tests. + * + * If the test.fs.wasb.name property is not set, this will + * trigger a JUnit failure. + * + * Multipart purging is enabled. + * @param conf configuration + * @return the FS + * @throws IOException IO Problems + * @throws AssumptionViolatedException if the FS is not named + */ + public static FileContext createTestFileContext(Configuration conf) + throws IOException { + String fsname = conf.getTrimmed(TEST_FS_WASB_NAME, ""); + + boolean liveTest = !StringUtils.isEmpty(fsname); + URI testURI = null; + if (liveTest) { + testURI = URI.create(fsname); + liveTest = testURI.getScheme().equals(WASB_SCHEME); + } + if (!liveTest) { + // This doesn't work with our JUnit 3 style test cases, so instead we'll + // make this whole class not run by default + throw new AssumptionViolatedException("No test filesystem in " + + TEST_FS_WASB_NAME); + } + FileContext fc = FileContext.getFileContext(testURI, conf); + return fc; + } + + /** + * Get a long test property. + * <ol> + * <li>Look up configuration value (which can pick up core-default.xml), + * using {@code defVal} as the default value (if conf != null). + * </li> + * <li>Fetch the system property.</li> + * <li>If the system property is not empty or "(unset)": + * it overrides the conf value. + * </li> + * </ol> + * This puts the build properties in charge of everything. It's not a + * perfect design; having maven set properties based on a file, as ant let + * you do, is better for customization. + * + * As to why there's a special (unset) value, see + * {@link http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven} + * @param conf config: may be null + * @param key key to look up + * @param defVal default value + * @return the evaluated test property. + */ + public static long getTestPropertyLong(Configuration conf, + String key, long defVal) { + return Long.valueOf( + getTestProperty(conf, key, Long.toString(defVal))); + } + /** + * Get a test property value in bytes, using k, m, g, t, p, e suffixes. + * {@link org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix#string2long(String)} + * <ol> + * <li>Look up configuration value (which can pick up core-default.xml), + * using {@code defVal} as the default value (if conf != null). + * </li> + * <li>Fetch the system property.</li> + * <li>If the system property is not empty or "(unset)": + * it overrides the conf value. + * </li> + * </ol> + * This puts the build properties in charge of everything. It's not a + * perfect design; having maven set properties based on a file, as ant let + * you do, is better for customization. + * + * As to why there's a special (unset) value, see + * {@link http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven} + * @param conf config: may be null + * @param key key to look up + * @param defVal default value + * @return the evaluated test property. + */ + public static long getTestPropertyBytes(Configuration conf, + String key, String defVal) { + return org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix + .string2long(getTestProperty(conf, key, defVal)); + } + + /** + * Get an integer test property; algorithm described in + * {@link #getTestPropertyLong(Configuration, String, long)}. + * @param key key to look up + * @param defVal default value + * @return the evaluated test property. + */ + public static int getTestPropertyInt(Configuration conf, + String key, int defVal) { + return (int) getTestPropertyLong(conf, key, defVal); + } + + /** + * Get a boolean test property; algorithm described in + * {@link #getTestPropertyLong(Configuration, String, long)}. + * @param key key to look up + * @param defVal default value + * @return the evaluated test property. + */ + public static boolean getTestPropertyBool(Configuration conf, + String key, + boolean defVal) { + return Boolean.valueOf( + getTestProperty(conf, key, Boolean.toString(defVal))); + } + + /** + * Get a string test property. + * <ol> + * <li>Look up configuration value (which can pick up core-default.xml), + * using {@code defVal} as the default value (if conf != null). + * </li> + * <li>Fetch the system property.</li> + * <li>If the system property is not empty or "(unset)": + * it overrides the conf value. + * </li> + * </ol> + * This puts the build properties in charge of everything. It's not a + * perfect design; having maven set properties based on a file, as ant let + * you do, is better for customization. + * + * As to why there's a special (unset) value, see + * @see <a href="http://stackoverflow.com/questions/7773134/null-versus-empty-arguments-in-maven";> + * Stack Overflow</a> + * @param conf config: may be null + * @param key key to look up + * @param defVal default value + * @return the evaluated test property. + */ + + public static String getTestProperty(Configuration conf, + String key, + String defVal) { + String confVal = conf != null + ? conf.getTrimmed(key, defVal) + : defVal; + String propval = System.getProperty(key); + return StringUtils.isNotEmpty(propval) && !UNSET_PROPERTY.equals(propval) + ? propval : confVal; + } + + /** + * Verify the class of an exception. If it is not as expected, rethrow it. + * Comparison is on the exact class, not subclass-of inference as + * offered by {@code instanceof}. + * @param clazz the expected exception class + * @param ex the exception caught + * @return the exception, if it is of the expected class + * @throws Exception the exception passed in. + */ + public static Exception verifyExceptionClass(Class clazz, + Exception ex) + throws Exception { + if (!(ex.getClass().equals(clazz))) { + throw ex; + } + return ex; + } + + /** + * Turn off FS Caching: use if a filesystem with different options from + * the default is required. + * @param conf configuration to patch + */ + public static void disableFilesystemCaching(Configuration conf) { + conf.setBoolean("fs.wasb.impl.disable.cache", true); + } + + /** + * Create a test path, using the value of + * {@link AzureTestUtils#TEST_UNIQUE_FORK_ID} if it is set. + * @param defVal default value + * @return a path + */ + public static Path createTestPath(Path defVal) { + String testUniqueForkId = System.getProperty( + AzureTestConstants.TEST_UNIQUE_FORK_ID); + return testUniqueForkId == null + ? defVal + : new Path("/" + testUniqueForkId, "test"); + } + + /** + * Create a test page blob path using the value of + * {@link AzureTestConstants#TEST_UNIQUE_FORK_ID} if it is set. + * @param filename filename at the end of the path + * @return an absolute path + */ + public static Path blobPathForTests(FileSystem fs, String filename) { + String testUniqueForkId = System.getProperty( + AzureTestConstants.TEST_UNIQUE_FORK_ID); + return fs.makeQualified(new Path(PAGE_BLOB_DIR, + testUniqueForkId == null + ? filename + : (testUniqueForkId + "/" + filename))); + } + + /** + * Create a test path using the value of + * {@link AzureTestConstants#TEST_UNIQUE_FORK_ID} if it is set. + * @param filename filename at the end of the path + * @return an absolute path + */ + public static Path pathForTests(FileSystem fs, String filename) { + String testUniqueForkId = System.getProperty( + AzureTestConstants.TEST_UNIQUE_FORK_ID); + return fs.makeQualified(new Path( + testUniqueForkId == null + ? ("/test/" + filename) + : (testUniqueForkId + "/" + filename))); + } + + /** + * Get a unique fork ID. + * Returns a default value for non-parallel tests. + * @return a string unique for all test VMs running in this maven build. + */ + public static String getForkID() { + return System.getProperty( + AzureTestConstants.TEST_UNIQUE_FORK_ID, "fork-1"); + } + + /** + * Flag to indicate that this test is being executed in parallel. + * This is used by some of the scale tests to validate test time expectations. + * @return true if the build indicates this test is being run in parallel. + */ + public static boolean isParallelExecution() { + return Boolean.getBoolean(KEY_PARALLEL_TEST_EXECUTION); + } + + /** + * Asserts that {@code obj} is an instance of {@code expectedClass} using a + * descriptive assertion message. + * @param expectedClass class + * @param obj object to check + */ + public static void assertInstanceOf(Class<?> expectedClass, Object obj) { + Assert.assertTrue(String.format("Expected instance of class %s, but is %s.", + expectedClass, obj.getClass()), + expectedClass.isAssignableFrom(obj.getClass())); + } + + /** + * Builds a comma-separated list of class names. + * @param classes list of classes + * @return comma-separated list of class names + */ + public static <T extends Class<?>> String buildClassListString( + List<T> classes) { + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < classes.size(); ++i) { + if (i > 0) { + sb.append(','); + } + sb.append(classes.get(i).getName()); + } + return sb.toString(); + } + + /** + * This class should not be instantiated. + */ + private AzureTestUtils() { + } + + /** + * Assert that a configuration option matches the expected value. + * @param conf configuration + * @param key option key + * @param expected expected value + */ + public static void assertOptionEquals(Configuration conf, + String key, + String expected) { + assertEquals("Value of " + key, expected, conf.get(key)); + } + + /** + * Assume that a condition is met. If not: log at WARN and + * then throw an {@link AssumptionViolatedException}. + * @param message message in an assumption + * @param condition condition to probe + */ + public static void assume(String message, boolean condition) { + if (!condition) { + LOG.warn(message); + } + Assume.assumeTrue(message, condition); + } + + /** + * Gets the current value of the given gauge. + * @param fs filesystem + * @param gaugeName gauge name + * @return the gauge value + */ + public static long getLongGaugeValue(NativeAzureFileSystem fs, + String gaugeName) { + return getLongGauge(gaugeName, getMetrics(fs.getInstrumentation())); + } + + /** + * Gets the current value of the given counter. + * @param fs filesystem + * @param counterName counter name + * @return the counter value + */ + public static long getLongCounterValue(NativeAzureFileSystem fs, + String counterName) { + return getLongCounter(counterName, getMetrics(fs.getInstrumentation())); + } + + + /** + * Delete a path, catching any exception and downgrading to a log message. + * @param fs filesystem + * @param path path to delete + * @param recursive recursive delete? + * @throws IOException IO failure. + */ + public static void deleteQuietly(FileSystem fs, + Path path, + boolean recursive) throws IOException { + if (fs != null && path != null) { + try { + fs.delete(path, recursive); + } catch (IOException e) { + LOG.warn("When deleting {}", path, e); + } + } + } + + + /** + * Clean up the test account if non-null; return null to put in the + * field. + * @param testAccount test account to clean up + * @return null + * @throws Execption cleanup problems + */ + public static AzureBlobStorageTestAccount cleanup( + AzureBlobStorageTestAccount testAccount) throws Exception { + if (testAccount != null) { + testAccount.cleanup(); + testAccount = null; + } + return null; + } + + + /** + * Clean up the test account; any thrown exceptions are caught and + * logged. + * @param testAccount test account + * @return null, so that any fields can be reset. + */ + public static AzureBlobStorageTestAccount cleanupTestAccount( + AzureBlobStorageTestAccount testAccount) { + if (testAccount != null) { + try { + testAccount.cleanup(); + } catch (Exception e) { + LOG.error("While cleaning up test account: ", e); + } + } + return null; + } + + /** + * Assume that the scale tests are enabled by the relevant system property. + */ + public static void assumeScaleTestsEnabled(Configuration conf) { + boolean enabled = getTestPropertyBool( + conf, + KEY_SCALE_TESTS_ENABLED, + DEFAULT_SCALE_TESTS_ENABLED); + assume("Scale test disabled: to enable set property " + + KEY_SCALE_TESTS_ENABLED, + enabled); + } +} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
