http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java index d107c9d..d696481 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemBackCompat.java @@ -27,13 +27,11 @@ import org.junit.Test; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - /** * Test AzureBlobFileSystem back compatibility with WASB. */ -public class ITestAzureBlobFileSystemBackCompat extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemBackCompat extends + AbstractAbfsIntegrationTest { public ITestAzureBlobFileSystemBackCompat() { super(); } @@ -54,13 +52,13 @@ public class ITestAzureBlobFileSystemBackCompat extends DependencyInjectedTest { blockBlob.uploadText(""); FileStatus[] fileStatuses = fs.listStatus(new Path("/test/10/")); - assertEquals(fileStatuses.length, 2); - assertEquals(fileStatuses[0].getPath().getName(), "10"); + assertEquals(2, fileStatuses.length); + assertEquals("10", fileStatuses[0].getPath().getName()); assertTrue(fileStatuses[0].isDirectory()); - assertEquals(fileStatuses[0].getLen(), 0); - assertEquals(fileStatuses[1].getPath().getName(), "123"); + assertEquals(0, fileStatuses[0].getLen()); + assertEquals("123", fileStatuses[1].getPath().getName()); assertTrue(fileStatuses[1].isDirectory()); - assertEquals(fileStatuses[1].getLen(), 0); + assertEquals(0, fileStatuses[1].getLen()); } private String getBlobConnectionString() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java index c158e03..90eff97 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCopy.java @@ -33,30 +33,29 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileUtil; import org.apache.hadoop.fs.Path; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile; /** * Test copy operation. */ -public class ITestAzureBlobFileSystemCopy extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemCopy extends AbstractAbfsIntegrationTest { public ITestAzureBlobFileSystemCopy() { super(); } @Test public void testCopyFromLocalFileSystem() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); Path localFilePath = new Path(System.getProperty("test.build.data", "azure_test")); - FileSystem localFs = FileSystem.get(new Configuration()); + FileSystem localFs = FileSystem.getLocal(new Configuration()); localFs.delete(localFilePath, true); try { writeString(localFs, localFilePath, "Testing"); Path dstPath = new Path("copiedFromLocal"); assertTrue(FileUtil.copy(localFs, localFilePath, fs, dstPath, false, fs.getConf())); - assertTrue(fs.exists(dstPath)); + assertIsFile(fs, dstPath); assertEquals("Testing", readString(fs, dstPath)); fs.delete(dstPath, true); } finally { @@ -65,36 +64,32 @@ public class ITestAzureBlobFileSystemCopy extends DependencyInjectedTest { } private String readString(FileSystem fs, Path testFile) throws IOException { - FSDataInputStream inputStream = fs.open(testFile); - String ret = readString(inputStream); - inputStream.close(); - return ret; + return readString(fs.open(testFile)); } private String readString(FSDataInputStream inputStream) throws IOException { - BufferedReader reader = new BufferedReader(new InputStreamReader( - inputStream)); - final int bufferSize = 1024; - char[] buffer = new char[bufferSize]; - int count = reader.read(buffer, 0, bufferSize); - if (count > bufferSize) { - throw new IOException("Exceeded buffer size"); + try (BufferedReader reader = new BufferedReader(new InputStreamReader( + inputStream))) { + final int bufferSize = 1024; + char[] buffer = new char[bufferSize]; + int count = reader.read(buffer, 0, bufferSize); + if (count > bufferSize) { + throw new IOException("Exceeded buffer size"); + } + return new String(buffer, 0, count); } - inputStream.close(); - return new String(buffer, 0, count); } private void writeString(FileSystem fs, Path path, String value) throws IOException { - FSDataOutputStream outputStream = fs.create(path, true); - writeString(outputStream, value); + writeString(fs.create(path, true), value); } private void writeString(FSDataOutputStream outputStream, String value) throws IOException { - BufferedWriter writer = new BufferedWriter(new OutputStreamWriter( - outputStream)); - writer.write(value); - writer.close(); + try(BufferedWriter writer = new BufferedWriter( + new OutputStreamWriter(outputStream))) { + writer.write(value); + } } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java index c9b99e6..1e43f9a 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java @@ -24,18 +24,17 @@ import java.util.EnumSet; import org.junit.Test; import org.apache.hadoop.fs.CreateFlag; -import org.apache.hadoop.fs.FileAlreadyExistsException; -import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.assertTrue; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile; /** * Test create operation. */ -public class ITestAzureBlobFileSystemCreate extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemCreate extends + AbstractAbfsIntegrationTest { private static final Path TEST_FILE_PATH = new Path("testfile"); private static final Path TEST_FOLDER_PATH = new Path("testFolder"); private static final String TEST_CHILD_FILE = "childFile"; @@ -43,68 +42,65 @@ public class ITestAzureBlobFileSystemCreate extends DependencyInjectedTest { super(); } - @Test(expected = FileAlreadyExistsException.class) - public void testCreateFileWithExistingDir() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.mkdirs(TEST_FOLDER_PATH); - fs.create(TEST_FOLDER_PATH); - } - @Test - public void testEnsureFileCreated() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(TEST_FILE_PATH); - - FileStatus fileStatus = fs.getFileStatus(TEST_FILE_PATH); - assertNotNull(fileStatus); + public void testEnsureFileCreatedImmediately() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + FSDataOutputStream out = fs.create(TEST_FILE_PATH); + try { + assertIsFile(fs, TEST_FILE_PATH); + } finally { + out.close(); + } + assertIsFile(fs, TEST_FILE_PATH); } @Test @SuppressWarnings("deprecation") public void testCreateNonRecursive() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); Path testFile = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE); try { fs.createNonRecursive(testFile, true, 1024, (short) 1, 1024, null); - assertTrue("Should've thrown", false); - } catch (FileNotFoundException e) { + fail("Should've thrown"); + } catch (FileNotFoundException expected) { } fs.mkdirs(TEST_FOLDER_PATH); fs.createNonRecursive(testFile, true, 1024, (short) 1, 1024, null) .close(); - assertTrue(fs.exists(testFile)); + assertIsFile(fs, testFile); } @Test @SuppressWarnings("deprecation") public void testCreateNonRecursive1() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); Path testFile = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE); try { fs.createNonRecursive(testFile, FsPermission.getDefault(), EnumSet.of(CreateFlag.CREATE, CreateFlag.OVERWRITE), 1024, (short) 1, 1024, null); - assertTrue("Should've thrown", false); - } catch (FileNotFoundException e) { + fail("Should've thrown"); + } catch (FileNotFoundException expected) { } fs.mkdirs(TEST_FOLDER_PATH); fs.createNonRecursive(testFile, true, 1024, (short) 1, 1024, null) .close(); - assertTrue(fs.exists(testFile)); + assertIsFile(fs, testFile); + } @Test @SuppressWarnings("deprecation") public void testCreateNonRecursive2() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); Path testFile = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE); try { fs.createNonRecursive(testFile, FsPermission.getDefault(), false, 1024, (short) 1, 1024, null); - assertTrue("Should've thrown", false); + fail("Should've thrown"); } catch (FileNotFoundException e) { } fs.mkdirs(TEST_FOLDER_PATH); fs.createNonRecursive(testFile, true, 1024, (short) 1, 1024, null) .close(); - assertTrue(fs.exists(testFile)); + assertIsFile(fs, testFile); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java index 372a087..91d1723 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemDelete.java @@ -28,71 +28,79 @@ import java.util.concurrent.Future; import org.junit.Test; +import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import static org.junit.Assert.assertEquals; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertDeleted; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** * Test delete operation. */ -public class ITestAzureBlobFileSystemDelete extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemDelete extends + AbstractAbfsIntegrationTest { public ITestAzureBlobFileSystemDelete() { super(); } @Test public void testDeleteRoot() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); fs.mkdirs(new Path("/testFolder0")); fs.mkdirs(new Path("/testFolder1")); fs.mkdirs(new Path("/testFolder2")); - fs.create(new Path("/testFolder1/testfile")); - fs.create(new Path("/testFolder1/testfile2")); - fs.create(new Path("/testFolder1/testfile3")); + touch(new Path("/testFolder1/testfile")); + touch(new Path("/testFolder1/testfile2")); + touch(new Path("/testFolder1/testfile3")); - FileStatus[] ls = fs.listStatus(new Path("/")); - assertEquals(4, ls.length); // and user dir + Path root = new Path("/"); + FileStatus[] ls = fs.listStatus(root); + assertEquals(3, ls.length); - fs.delete(new Path("/"), true); - ls = fs.listStatus(new Path("/")); - assertEquals(0, ls.length); + fs.delete(root, true); + ls = fs.listStatus(root); + assertEquals("listing size", 0, ls.length); } - @Test(expected = FileNotFoundException.class) + @Test() public void testOpenFileAfterDelete() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(new Path("/testFile")); - fs.delete(new Path("/testFile"), false); + final AzureBlobFileSystem fs = getFileSystem(); + Path testfile = new Path("/testFile"); + touch(testfile); + assertDeleted(fs, testfile, false); - fs.open(new Path("/testFile")); + intercept(FileNotFoundException.class, + () -> fs.open(testfile)); } - @Test(expected = FileNotFoundException.class) + @Test public void testEnsureFileIsDeleted() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(new Path("testfile")); - fs.delete(new Path("testfile"), false); - - fs.getFileStatus(new Path("testfile")); + final AzureBlobFileSystem fs = getFileSystem(); + Path testfile = new Path("testfile"); + touch(testfile); + assertDeleted(fs, testfile, false); + assertPathDoesNotExist(fs, "deleted", testfile); } - @Test(expected = FileNotFoundException.class) + @Test public void testDeleteDirectory() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.mkdirs(new Path("testfile")); + final AzureBlobFileSystem fs = getFileSystem(); + Path dir = new Path("testfile"); + fs.mkdirs(dir); fs.mkdirs(new Path("testfile/test1")); fs.mkdirs(new Path("testfile/test1/test2")); - fs.delete(new Path("testfile"), true); - fs.getFileStatus(new Path("testfile")); + assertDeleted(fs, dir, true); + assertPathDoesNotExist(fs, "deleted", dir); } - @Test(expected = FileNotFoundException.class) + @Test public void testDeleteFirstLevelDirectory() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final List<Future> tasks = new ArrayList<>(); + final AzureBlobFileSystem fs = getFileSystem(); + final List<Future<Void>> tasks = new ArrayList<>(); ExecutorService es = Executors.newFixedThreadPool(10); for (int i = 0; i < 1000; i++) { @@ -100,7 +108,7 @@ public class ITestAzureBlobFileSystemDelete extends DependencyInjectedTest { Callable<Void> callable = new Callable<Void>() { @Override public Void call() throws Exception { - fs.create(fileName); + touch(fileName); return null; } }; @@ -113,7 +121,12 @@ public class ITestAzureBlobFileSystemDelete extends DependencyInjectedTest { } es.shutdownNow(); - fs.delete(new Path("/test"), true); - fs.getFileStatus(new Path("/test")); + Path dir = new Path("/test"); + // first try a non-recursive delete, expect failure + intercept(FileAlreadyExistsException.class, + () -> fs.delete(dir, false)); + assertDeleted(fs, dir, true); + assertPathDoesNotExist(fs, "deleted", dir); + } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java index ad22f99..057dfa0 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2E.java @@ -38,7 +38,7 @@ import static org.junit.Assert.assertArrayEquals; /** * Test end to end between ABFS client and ABFS server. */ -public class ITestAzureBlobFileSystemE2E extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemE2E extends AbstractAbfsIntegrationTest { private static final Path TEST_FILE = new Path("testfile"); private static final int TEST_BYTE = 100; private static final int TEST_OFFSET = 100; @@ -53,11 +53,11 @@ public class ITestAzureBlobFileSystemE2E extends DependencyInjectedTest { @Test public void testWriteOneByteToFile() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - FSDataOutputStream stream = fs.create(TEST_FILE); + final AzureBlobFileSystem fs = getFileSystem(); - stream.write(TEST_BYTE); - stream.close(); + try(FSDataOutputStream stream = fs.create(TEST_FILE)) { + stream.write(TEST_BYTE); + } FileStatus fileStatus = fs.getFileStatus(TEST_FILE); assertEquals(1, fileStatus.getLen()); @@ -65,52 +65,52 @@ public class ITestAzureBlobFileSystemE2E extends DependencyInjectedTest { @Test public void testReadWriteBytesToFile() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); testWriteOneByteToFile(); - FSDataInputStream inputStream = fs.open(TEST_FILE, TEST_DEFAULT_BUFFER_SIZE); - int i = inputStream.read(); - inputStream.close(); - - assertEquals(TEST_BYTE, i); + try(FSDataInputStream inputStream = fs.open(TEST_FILE, + TEST_DEFAULT_BUFFER_SIZE)) { + assertEquals(TEST_BYTE, inputStream.read()); + } } @Test (expected = IOException.class) public void testOOBWrites() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); int readBufferSize = fs.getAbfsStore().getAbfsConfiguration().getReadBufferSize(); - fs.create(TEST_FILE); - FSDataOutputStream writeStream = fs.create(TEST_FILE); - byte[] bytesToRead = new byte[readBufferSize]; final byte[] b = new byte[2 * readBufferSize]; new Random().nextBytes(b); - writeStream.write(b); - writeStream.flush(); - writeStream.close(); - FSDataInputStream readStream = fs.open(TEST_FILE); - readStream.read(bytesToRead, 0, readBufferSize); + try(FSDataOutputStream writeStream = fs.create(TEST_FILE)) { + writeStream.write(b); + writeStream.flush(); + } + + try (FSDataInputStream readStream = fs.open(TEST_FILE)) { + assertEquals(readBufferSize, + readStream.read(bytesToRead, 0, readBufferSize)); - writeStream = fs.create(TEST_FILE); - writeStream.write(b); - writeStream.flush(); - writeStream.close(); + try (FSDataOutputStream writeStream = fs.create(TEST_FILE)) { + writeStream.write(b); + writeStream.flush(); + } - readStream.read(bytesToRead, 0, readBufferSize); - readStream.close(); + assertEquals(readBufferSize, + readStream.read(bytesToRead, 0, readBufferSize)); + } } @Test public void testWriteWithBufferOffset() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final FSDataOutputStream stream = fs.create(TEST_FILE); + final AzureBlobFileSystem fs = getFileSystem(); final byte[] b = new byte[1024 * 1000]; new Random().nextBytes(b); - stream.write(b, TEST_OFFSET, b.length - TEST_OFFSET); - stream.close(); + try(final FSDataOutputStream stream = fs.create(TEST_FILE)) { + stream.write(b, TEST_OFFSET, b.length - TEST_OFFSET); + } final byte[] r = new byte[TEST_DEFAULT_READ_BUFFER_SIZE]; FSDataInputStream inputStream = fs.open(TEST_FILE, TEST_DEFAULT_BUFFER_SIZE); @@ -124,13 +124,11 @@ public class ITestAzureBlobFileSystemE2E extends DependencyInjectedTest { @Test public void testReadWriteHeavyBytesToFileWithSmallerChunks() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final FSDataOutputStream stream = fs.create(TEST_FILE); + final AzureBlobFileSystem fs = getFileSystem(); final byte[] writeBuffer = new byte[5 * 1000 * 1024]; new Random().nextBytes(writeBuffer); - stream.write(writeBuffer); - stream.close(); + write(TEST_FILE, writeBuffer); final byte[] readBuffer = new byte[5 * 1000 * 1024]; FSDataInputStream inputStream = fs.open(TEST_FILE, TEST_DEFAULT_BUFFER_SIZE); http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java index 616253b..04690de 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemE2EScale.java @@ -26,7 +26,6 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import org.junit.Assert; import org.junit.Test; import org.apache.hadoop.fs.FSDataInputStream; @@ -35,28 +34,24 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - /** * Test end to end between ABFS client and ABFS server with heavy traffic. */ -public class ITestAzureBlobFileSystemE2EScale extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemE2EScale extends + AbstractAbfsScaleTest { private static final int TEN = 10; private static final int ONE_THOUSAND = 1000; private static final int BASE_SIZE = 1024; private static final int ONE_MB = 1024 * 1024; private static final int DEFAULT_WRITE_TIMES = 100; - private static final Path TEST_FILE = new Path("testfile"); + private static final Path TEST_FILE = new Path("ITestAzureBlobFileSystemE2EScale"); public ITestAzureBlobFileSystemE2EScale() { - super(); } @Test - public void testWriteHeavyBytesToFile() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + public void testWriteHeavyBytesToFileAcrossThreads() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); final FSDataOutputStream stream = fs.create(TEST_FILE); ExecutorService es = Executors.newFixedThreadPool(TEN); @@ -65,7 +60,8 @@ public class ITestAzureBlobFileSystemE2EScale extends DependencyInjectedTest { new Random().nextBytes(b); List<Future<Void>> tasks = new ArrayList<>(); - for (int i = 0; i < DEFAULT_WRITE_TIMES; i++) { + int operationCount = DEFAULT_WRITE_TIMES; + for (int i = 0; i < operationCount; i++) { Callable<Void> callable = new Callable<Void>() { @Override public Void call() throws Exception { @@ -86,48 +82,38 @@ public class ITestAzureBlobFileSystemE2EScale extends DependencyInjectedTest { es.shutdownNow(); FileStatus fileStatus = fs.getFileStatus(TEST_FILE); - assertEquals(testWriteBufferSize * DEFAULT_WRITE_TIMES, fileStatus.getLen()); - } - - @Test - public void testReadWriteHeavyBytesToFile() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final FSDataOutputStream stream = fs.create(TEST_FILE); - - int testBufferSize = 5 * TEN * ONE_THOUSAND * BASE_SIZE; - final byte[] b = new byte[testBufferSize]; - new Random().nextBytes(b); - stream.write(b); - stream.close(); - - final byte[] r = new byte[testBufferSize]; - FSDataInputStream inputStream = fs.open(TEST_FILE, 4 * ONE_MB); - int result = inputStream.read(r); - inputStream.close(); - - assertNotEquals(-1, result); - assertArrayEquals(r, b); + assertEquals(testWriteBufferSize * operationCount, fileStatus.getLen()); } @Test public void testReadWriteHeavyBytesToFileWithStatistics() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final FSDataOutputStream stream = fs.create(TEST_FILE); - final FileSystem.Statistics abfsStatistics = fs.getFsStatistics(); - abfsStatistics.reset(); + final AzureBlobFileSystem fs = getFileSystem(); + final FileSystem.Statistics abfsStatistics; + int testBufferSize; + final byte[] sourceData; + try(final FSDataOutputStream stream = fs.create(TEST_FILE)) { + abfsStatistics = fs.getFsStatistics(); + abfsStatistics.reset(); + + testBufferSize = 5 * TEN * ONE_THOUSAND * BASE_SIZE; + sourceData = new byte[testBufferSize]; + new Random().nextBytes(sourceData); + stream.write(sourceData); + } - int testBufferSize = 5 * TEN * ONE_THOUSAND * BASE_SIZE; - final byte[] b = new byte[testBufferSize]; - new Random().nextBytes(b); - stream.write(b); - stream.close(); + final byte[] remoteData = new byte[testBufferSize]; + int bytesRead; + try (FSDataInputStream inputStream = fs.open(TEST_FILE, 4 * ONE_MB)) { + bytesRead = inputStream.read(remoteData); + } - final byte[] r = new byte[testBufferSize]; - FSDataInputStream inputStream = fs.open(TEST_FILE, 4 * ONE_MB); - inputStream.read(r); - inputStream.close(); + String stats = abfsStatistics.toString(); + assertEquals("Bytes read in " + stats, + remoteData.length, abfsStatistics.getBytesRead()); + assertEquals("bytes written in " + stats, + sourceData.length, abfsStatistics.getBytesWritten()); + assertEquals("bytesRead from read() call", testBufferSize, bytesRead ); + assertArrayEquals("round tripped data", sourceData, remoteData); - Assert.assertEquals(r.length, abfsStatistics.getBytesRead()); - Assert.assertEquals(b.length, abfsStatistics.getBytesWritten()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java index bfa662d..791694b 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFileStatus.java @@ -18,6 +18,8 @@ package org.apache.hadoop.fs.azurebfs; +import java.io.IOException; + import org.junit.Test; import org.apache.hadoop.fs.FileStatus; @@ -25,12 +27,11 @@ import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsAction; import org.apache.hadoop.fs.permission.FsPermission; -import static org.junit.Assert.assertEquals; - /** * Test FileStatus. */ -public class ITestAzureBlobFileSystemFileStatus extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemFileStatus extends + AbstractAbfsIntegrationTest { private static final Path TEST_FILE = new Path("testFile"); private static final Path TEST_FOLDER = new Path("testDir"); public ITestAzureBlobFileSystemFileStatus() { @@ -41,24 +42,38 @@ public class ITestAzureBlobFileSystemFileStatus extends DependencyInjectedTest { public void testEnsureStatusWorksForRoot() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); - fs.getFileStatus(new Path("/")); - fs.listStatus(new Path("/")); + Path root = new Path("/"); + FileStatus[] rootls = fs.listStatus(root); + assertEquals("root listing", 0, rootls.length); } @Test public void testFileStatusPermissionsAndOwnerAndGroup() throws Exception { final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(TEST_FILE); - fs.mkdirs(TEST_FOLDER); + touch(TEST_FILE); + validateStatus(fs, TEST_FILE); + } + + private FileStatus validateStatus(final AzureBlobFileSystem fs, final Path name) + throws IOException { + FileStatus fileStatus = fs.getFileStatus(name); + String errorInStatus = "error in " + fileStatus + " from " + fs; + assertEquals(errorInStatus + ": permission", + new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL), + fileStatus.getPermission()); + assertEquals(errorInStatus + ": owner", + fs.getOwnerUser(), fileStatus.getOwner()); + assertEquals(errorInStatus + ": group", + fs.getOwnerUserPrimaryGroup(), fileStatus.getGroup()); + return fileStatus; + } - FileStatus fileStatus = fs.getFileStatus(TEST_FILE); - assertEquals(new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL), fileStatus.getPermission()); - assertEquals(fs.getOwnerUser(), fileStatus.getGroup()); - assertEquals(fs.getOwnerUserPrimaryGroup(), fileStatus.getOwner()); + @Test + public void testFolderStatusPermissionsAndOwnerAndGroup() throws Exception { + final AzureBlobFileSystem fs = this.getFileSystem(); + fs.mkdirs(TEST_FOLDER); - fileStatus = fs.getFileStatus(TEST_FOLDER); - assertEquals(new FsPermission(FsAction.ALL, FsAction.ALL, FsAction.ALL), fileStatus.getPermission()); - assertEquals(fs.getOwnerUser(), fileStatus.getGroup()); - assertEquals(fs.getOwnerUserPrimaryGroup(), fileStatus.getOwner()); + validateStatus(fs, TEST_FOLDER); } + } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java index 8c2e8ce..d90f018 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemFlush.java @@ -34,14 +34,10 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import static org.junit.Assert.assertArrayEquals; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotEquals; - /** * Test flush operation. */ -public class ITestAzureBlobFileSystemFlush extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemFlush extends AbstractAbfsScaleTest { private static final int BASE_SIZE = 1024; private static final int ONE_THOUSAND = 1000; private static final int TEST_BUFFER_SIZE = 5 * ONE_THOUSAND * BASE_SIZE; @@ -56,146 +52,145 @@ public class ITestAzureBlobFileSystemFlush extends DependencyInjectedTest { } @Test - public void testAbfsOutputStreamAsyncFlushWithRetainUncommitedData() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final FSDataOutputStream stream = fs.create(TEST_FILE_PATH); - - final byte[] b = new byte[TEST_BUFFER_SIZE]; - new Random().nextBytes(b); + public void testAbfsOutputStreamAsyncFlushWithRetainUncommittedData() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + final byte[] b; + try(final FSDataOutputStream stream = fs.create(TEST_FILE_PATH)) { + b = new byte[TEST_BUFFER_SIZE]; + new Random().nextBytes(b); - for (int i = 0; i < 2; i++) { - stream.write(b); + for (int i = 0; i < 2; i++) { + stream.write(b); - for (int j = 0; j < FLUSH_TIMES; j++) { - stream.flush(); - Thread.sleep(10); + for (int j = 0; j < FLUSH_TIMES; j++) { + stream.flush(); + Thread.sleep(10); + } } } - stream.close(); - final byte[] r = new byte[TEST_BUFFER_SIZE]; - FSDataInputStream inputStream = fs.open(TEST_FILE_PATH, 4 * ONE_MB); + try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH, 4 * ONE_MB)) { + while (inputStream.available() != 0) { + int result = inputStream.read(r); - while (inputStream.available() != 0) { - int result = inputStream.read(r); - - assertNotEquals(-1, result); - assertArrayEquals(r, b); + assertNotEquals("read returned -1", -1, result); + assertArrayEquals("buffer read from stream", r, b); + } } - - inputStream.close(); } @Test public void testAbfsOutputStreamSyncFlush() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final FSDataOutputStream stream = fs.create(TEST_FILE_PATH); - - final byte[] b = new byte[TEST_BUFFER_SIZE]; - new Random().nextBytes(b); - stream.write(b); + final AzureBlobFileSystem fs = getFileSystem(); + final byte[] b; + try(final FSDataOutputStream stream = fs.create(TEST_FILE_PATH)) { + b = new byte[TEST_BUFFER_SIZE]; + new Random().nextBytes(b); + stream.write(b); - for (int i = 0; i < FLUSH_TIMES; i++) { - stream.hsync(); - stream.hflush(); - Thread.sleep(10); + for (int i = 0; i < FLUSH_TIMES; i++) { + stream.hsync(); + stream.hflush(); + Thread.sleep(10); + } } - stream.close(); final byte[] r = new byte[TEST_BUFFER_SIZE]; - FSDataInputStream inputStream = fs.open(TEST_FILE_PATH, 4 * ONE_MB); - int result = inputStream.read(r); - - assertNotEquals(-1, result); - assertArrayEquals(r, b); + try(FSDataInputStream inputStream = fs.open(TEST_FILE_PATH, 4 * ONE_MB)) { + int result = inputStream.read(r); - inputStream.close(); + assertNotEquals(-1, result); + assertArrayEquals(r, b); + } } @Test public void testWriteHeavyBytesToFileSyncFlush() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final FSDataOutputStream stream = fs.create(TEST_FILE_PATH); - final FileSystem.Statistics abfsStatistics = fs.getFsStatistics(); - abfsStatistics.reset(); - - ExecutorService es = Executors.newFixedThreadPool(10); - - final byte[] b = new byte[TEST_BUFFER_SIZE]; - new Random().nextBytes(b); - - List<Future<Void>> tasks = new ArrayList<>(); - for (int i = 0; i < FLUSH_TIMES; i++) { - Callable<Void> callable = new Callable<Void>() { - @Override - public Void call() throws Exception { - stream.write(b); - return null; - } - }; - - tasks.add(es.submit(callable)); - } + final AzureBlobFileSystem fs = getFileSystem(); + final FileSystem.Statistics abfsStatistics; + ExecutorService es; + try(final FSDataOutputStream stream = fs.create(TEST_FILE_PATH)) { + abfsStatistics = fs.getFsStatistics(); + abfsStatistics.reset(); + + es = Executors.newFixedThreadPool(10); + + final byte[] b = new byte[TEST_BUFFER_SIZE]; + new Random().nextBytes(b); + + List<Future<Void>> tasks = new ArrayList<>(); + for (int i = 0; i < FLUSH_TIMES; i++) { + Callable<Void> callable = new Callable<Void>() { + @Override + public Void call() throws Exception { + stream.write(b); + return null; + } + }; + + tasks.add(es.submit(callable)); + } - boolean shouldStop = false; - while (!shouldStop) { - shouldStop = true; - for (Future<Void> task : tasks) { - if (!task.isDone()) { - stream.hsync(); - shouldStop = false; - Thread.sleep(THREAD_SLEEP_TIME); + boolean shouldStop = false; + while (!shouldStop) { + shouldStop = true; + for (Future<Void> task : tasks) { + if (!task.isDone()) { + stream.hsync(); + shouldStop = false; + Thread.sleep(THREAD_SLEEP_TIME); + } } } - } - tasks.clear(); - stream.close(); + tasks.clear(); + } es.shutdownNow(); FileStatus fileStatus = fs.getFileStatus(TEST_FILE_PATH); - assertEquals((long) TEST_BUFFER_SIZE * FLUSH_TIMES, fileStatus.getLen()); - assertEquals((long) TEST_BUFFER_SIZE * FLUSH_TIMES, abfsStatistics.getBytesWritten()); + long expectedWrites = (long) TEST_BUFFER_SIZE * FLUSH_TIMES; + assertEquals("Wrong file length in " + fileStatus, expectedWrites, fileStatus.getLen()); + assertEquals("wrong bytes Written count in " + abfsStatistics, + expectedWrites, abfsStatistics.getBytesWritten()); } @Test public void testWriteHeavyBytesToFileAsyncFlush() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(TEST_FILE_PATH); - final FSDataOutputStream stream = fs.create(TEST_FILE_PATH); + final AzureBlobFileSystem fs = getFileSystem(); ExecutorService es = Executors.newFixedThreadPool(10); + try(final FSDataOutputStream stream = fs.create(TEST_FILE_PATH)) { + + final byte[] b = new byte[TEST_BUFFER_SIZE]; + new Random().nextBytes(b); + + List<Future<Void>> tasks = new ArrayList<>(); + for (int i = 0; i < FLUSH_TIMES; i++) { + Callable<Void> callable = new Callable<Void>() { + @Override + public Void call() throws Exception { + stream.write(b); + return null; + } + }; + + tasks.add(es.submit(callable)); + } - final byte[] b = new byte[TEST_BUFFER_SIZE]; - new Random().nextBytes(b); - - List<Future<Void>> tasks = new ArrayList<>(); - for (int i = 0; i < FLUSH_TIMES; i++) { - Callable<Void> callable = new Callable<Void>() { - @Override - public Void call() throws Exception { - stream.write(b); - return null; - } - }; - - tasks.add(es.submit(callable)); - } - - boolean shouldStop = false; - while (!shouldStop) { - shouldStop = true; - for (Future<Void> task : tasks) { - if (!task.isDone()) { - stream.flush(); - shouldStop = false; + boolean shouldStop = false; + while (!shouldStop) { + shouldStop = true; + for (Future<Void> task : tasks) { + if (!task.isDone()) { + stream.flush(); + shouldStop = false; + } } } + Thread.sleep(THREAD_SLEEP_TIME); + tasks.clear(); } - Thread.sleep(THREAD_SLEEP_TIME); - tasks.clear(); - stream.close(); es.shutdownNow(); FileStatus fileStatus = fs.getFileStatus(TEST_FILE_PATH); http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java index d2ed400..5a6e46d 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemInitAndCreate.java @@ -22,29 +22,32 @@ import java.io.FileNotFoundException; import org.junit.Test; +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azurebfs.constants.ConfigurationKeys; /** * Test filesystem initialization and creation. */ -public class ITestAzureBlobFileSystemInitAndCreate extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemInitAndCreate extends + AbstractAbfsIntegrationTest { public ITestAzureBlobFileSystemInitAndCreate() { - super(); this.getConfiguration().unset(ConfigurationKeys.AZURE_CREATE_REMOTE_FILESYSTEM_DURING_INITIALIZATION); } @Override - public void initialize() { + public void setup() { } @Override - public void testCleanup() { + public void teardown() { } @Test (expected = FileNotFoundException.class) public void ensureFilesystemWillNotBeCreatedIfCreationConfigIsNotSet() throws Exception { - super.initialize(); - this.getFileSystem(); + super.setup(); + final AzureBlobFileSystem fs = this.getFileSystem(); + FileStatus[] fileStatuses = fs.listStatus(new Path("/")); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java index 6059766..b87abe6 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemListStatus.java @@ -26,20 +26,21 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import org.junit.Assert; import org.junit.Test; +import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; +import static org.apache.hadoop.test.LambdaTestUtils.intercept; /** * Test listStatus operation. */ -public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemListStatus extends + AbstractAbfsIntegrationTest { private static final int TEST_FILES_NUMBER = 6000; public ITestAzureBlobFileSystemListStatus() { super(); @@ -47,8 +48,8 @@ public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest { @Test public void testListPath() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final List<Future> tasks = new ArrayList<>(); + final AzureBlobFileSystem fs = getFileSystem(); + final List<Future<Void>> tasks = new ArrayList<>(); ExecutorService es = Executors.newFixedThreadPool(10); for (int i = 0; i < TEST_FILES_NUMBER; i++) { @@ -56,7 +57,7 @@ public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest { Callable<Void> callable = new Callable<Void>() { @Override public Void call() throws Exception { - fs.create(fileName); + touch(fileName); return null; } }; @@ -70,63 +71,101 @@ public class ITestAzureBlobFileSystemListStatus extends DependencyInjectedTest { es.shutdownNow(); FileStatus[] files = fs.listStatus(new Path("/")); - Assert.assertEquals(files.length, TEST_FILES_NUMBER + 1 /* user directory */); + assertEquals(TEST_FILES_NUMBER, files.length /* user directory */); } + /** + * Creates a file, verifies that listStatus returns it, + * even while the file is still open for writing. + */ @Test public void testListFileVsListDir() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(new Path("/testFile")); - - FileStatus[] testFiles = fs.listStatus(new Path("/testFile")); - Assert.assertEquals(testFiles.length, 1); - Assert.assertFalse(testFiles[0].isDirectory()); + final AzureBlobFileSystem fs = getFileSystem(); + Path path = new Path("/testFile"); + try(FSDataOutputStream ignored = fs.create(path)) { + FileStatus[] testFiles = fs.listStatus(path); + assertEquals("length of test files", 1, testFiles.length); + FileStatus status = testFiles[0]; + assertIsFileReference(status); + } } @Test public void testListFileVsListDir2() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); fs.mkdirs(new Path("/testFolder")); fs.mkdirs(new Path("/testFolder/testFolder2")); fs.mkdirs(new Path("/testFolder/testFolder2/testFolder3")); - fs.create(new Path("/testFolder/testFolder2/testFolder3/testFile")); - - FileStatus[] testFiles = fs.listStatus(new Path("/testFolder/testFolder2/testFolder3/testFile")); - Assert.assertEquals(testFiles.length, 1); - Assert.assertEquals(testFiles[0].getPath(), new Path(this.getTestUrl(), - "/testFolder/testFolder2/testFolder3/testFile")); - Assert.assertFalse(testFiles[0].isDirectory()); + Path testFile0Path = new Path("/testFolder/testFolder2/testFolder3/testFile"); + ContractTestUtils.touch(fs, testFile0Path); + + FileStatus[] testFiles = fs.listStatus(testFile0Path); + assertEquals("Wrong listing size of file " + testFile0Path, + 1, testFiles.length); + FileStatus file0 = testFiles[0]; + assertEquals("Wrong path for " + file0, + new Path(getTestUrl(), "/testFolder/testFolder2/testFolder3/testFile"), + file0.getPath()); + assertIsFileReference(file0); } @Test(expected = FileNotFoundException.class) public void testListNonExistentDir() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); fs.listStatus(new Path("/testFile/")); } @Test public void testListFiles() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.mkdirs(new Path("/test")); + final AzureBlobFileSystem fs = getFileSystem(); + Path testDir = new Path("/test"); + fs.mkdirs(testDir); FileStatus[] fileStatuses = fs.listStatus(new Path("/")); - assertEquals(fileStatuses.length, 2); + assertEquals(1, fileStatuses.length); fs.mkdirs(new Path("/test/sub")); - fileStatuses = fs.listStatus(new Path("/test")); - assertEquals(fileStatuses.length, 1); - assertEquals(fileStatuses[0].getPath().getName(), "sub"); - assertTrue(fileStatuses[0].isDirectory()); - assertEquals(fileStatuses[0].getLen(), 0); - - fs.create(new Path("/test/f")); - fileStatuses = fs.listStatus(new Path("/test")); - assertEquals(fileStatuses.length, 2); - assertEquals(fileStatuses[0].getPath().getName(), "f"); - assertFalse(fileStatuses[0].isDirectory()); - assertEquals(fileStatuses[0].getLen(), 0); - assertEquals(fileStatuses[1].getPath().getName(), "sub"); - assertTrue(fileStatuses[1].isDirectory()); - assertEquals(fileStatuses[1].getLen(), 0); + fileStatuses = fs.listStatus(testDir); + assertEquals(1, fileStatuses.length); + assertEquals("sub", fileStatuses[0].getPath().getName()); + assertIsDirectoryReference(fileStatuses[0]); + Path childF = fs.makeQualified(new Path("/test/f")); + touch(childF); + fileStatuses = fs.listStatus(testDir); + assertEquals(2, fileStatuses.length); + final FileStatus childStatus = fileStatuses[0]; + assertEquals(childF, childStatus.getPath()); + assertEquals("f", childStatus.getPath().getName()); + assertIsFileReference(childStatus); + assertEquals(0, childStatus.getLen()); + final FileStatus status1 = fileStatuses[1]; + assertEquals("sub", status1.getPath().getName()); + assertIsDirectoryReference(status1); + // look at the child through getFileStatus + LocatedFileStatus locatedChildStatus = fs.listFiles(childF, false).next(); + assertIsFileReference(locatedChildStatus); + + fs.delete(testDir, true); + intercept(FileNotFoundException.class, + () -> fs.listFiles(childF, false).next()); + + // do some final checks on the status (failing due to version checks) + assertEquals("Path mismatch of " + locatedChildStatus, + childF, locatedChildStatus.getPath()); + assertEquals("locatedstatus.equals(status)", + locatedChildStatus, childStatus); + assertEquals("status.equals(locatedstatus)", + childStatus, locatedChildStatus); + } + + private void assertIsDirectoryReference(FileStatus status) { + assertTrue("Not a directory: " + status, status.isDirectory()); + assertFalse("Not a directory: " + status, status.isFile()); + assertEquals(0, status.getLen()); + } + + private void assertIsFileReference(FileStatus status) { + assertFalse("Not a file: " + status, status.isDirectory()); + assertTrue("Not a file: " + status, status.isFile()); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java index b61908c..1bb2c54 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemMkDir.java @@ -18,71 +18,30 @@ package org.apache.hadoop.fs.azurebfs; -import java.util.concurrent.Callable; - import org.junit.Test; -import org.apache.hadoop.fs.FileAlreadyExistsException; import org.apache.hadoop.fs.Path; -import static org.apache.hadoop.test.LambdaTestUtils.intercept; -import static org.junit.Assert.assertTrue; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs; /** * Test mkdir operation. */ -public class ITestAzureBlobFileSystemMkDir extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemMkDir extends AbstractAbfsIntegrationTest { public ITestAzureBlobFileSystemMkDir() { super(); } @Test public void testCreateDirWithExistingDir() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - assertTrue(fs.mkdirs(new Path("testFolder"))); - assertTrue(fs.mkdirs(new Path("testFolder"))); - } - - @Test(expected = FileAlreadyExistsException.class) - public void createDirectoryUnderFile() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(new Path("testFile")); - fs.mkdirs(new Path("testFile/TestDirectory")); - } - - @Test - public void testCreateDirectoryOverExistingFiles() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(new Path("/testPath")); - FileAlreadyExistsException ex = intercept( - FileAlreadyExistsException.class, - new Callable<Void>() { - @Override - public Void call() throws Exception { - fs.mkdirs(new Path("/testPath")); - return null; - } - }); - - assertTrue(ex instanceof FileAlreadyExistsException); - - fs.create(new Path("/testPath1/file1")); - ex = intercept( - FileAlreadyExistsException.class, - new Callable<Void>() { - @Override - public Void call() throws Exception { - fs.mkdirs(new Path("/testPath1/file1")); - return null; - } - }); - - assertTrue(ex instanceof FileAlreadyExistsException); + final AzureBlobFileSystem fs = getFileSystem(); + Path path = new Path("testFolder"); + assertMkdirs(fs, path); + assertMkdirs(fs, path); } @Test public void testCreateRoot() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - assertTrue(fs.mkdirs(new Path("/"))); + assertMkdirs(getFileSystem(), new Path("/")); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java deleted file mode 100644 index fef7f47..0000000 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemOpen.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.fs.azurebfs; - -import java.io.FileNotFoundException; - -import org.junit.Test; - -import org.apache.hadoop.fs.Path; - -/** - * Test open operation. - */ -public class ITestAzureBlobFileSystemOpen extends DependencyInjectedTest { - public ITestAzureBlobFileSystemOpen() throws Exception { - super(); - } - - @Test(expected = FileNotFoundException.class) - public void testOpenDirectory() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.mkdirs(new Path("testFolder")); - fs.open(new Path("testFolder")); - } -} http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java index 8b96c69..c61de67 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRandomRead.java @@ -18,32 +18,31 @@ package org.apache.hadoop.fs.azurebfs; -import org.apache.hadoop.fs.Path; +import java.io.EOFException; +import java.io.IOException; +import java.util.Random; +import java.util.concurrent.Callable; + +import org.junit.Test; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FSExceptionMessages; -import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.azure.NativeAzureFileSystem; import org.apache.hadoop.fs.contract.ContractTestUtils; -import org.junit.Test; - -import java.io.EOFException; -import java.io.IOException; -import java.util.Random; -import java.util.concurrent.Callable; import static org.apache.hadoop.test.LambdaTestUtils.intercept; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertArrayEquals; - /** * Test random read operation. */ -public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemRandomRead extends + AbstractAbfsScaleTest { private static final int KILOBYTE = 1024; private static final int MEGABYTE = KILOBYTE * KILOBYTE; private static final long TEST_FILE_SIZE = 8 * MEGABYTE; @@ -62,6 +61,9 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { private static final String ABFS = "ABFS"; private static long testFileLength = 0; + private static final Logger LOG = + LoggerFactory.getLogger(ITestAzureBlobFileSystemRandomRead.class); + public ITestAzureBlobFileSystemRandomRead() throws Exception { super(); } @@ -76,7 +78,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { // forward seek and read a kilobyte into first kilobyte of bufferV2 inputStream.seek(5 * MEGABYTE); int numBytesRead = inputStream.read(buffer, 0, KILOBYTE); - assertEquals(KILOBYTE, numBytesRead); + assertEquals("Wrong number of bytes read", KILOBYTE, numBytesRead); int len = MEGABYTE; int offset = buffer.length - len; @@ -84,7 +86,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { // reverse seek and read a megabyte into last megabyte of bufferV1 inputStream.seek(3 * MEGABYTE); numBytesRead = inputStream.read(buffer, offset, len); - assertEquals(len, numBytesRead); + assertEquals("Wrong number of bytes read after seek", len, numBytesRead); } } @@ -391,7 +393,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { afterSeekElapsedMs = sequentialRead(ABFS, this.getFileSystem(), true); ratio = afterSeekElapsedMs / beforeSeekElapsedMs; - System.out.println((String.format( + LOG.info((String.format( "beforeSeekElapsedMs=%1$d, afterSeekElapsedMs=%2$d, ratio=%3$.2f", (long) beforeSeekElapsedMs, (long) afterSeekElapsedMs, @@ -425,7 +427,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { ratio = v2ElapsedMs / v1ElapsedMs; - System.out.println(String.format( + LOG.info(String.format( "v1ElapsedMs=%1$d, v2ElapsedMs=%2$d, ratio=%3$.2f", (long) v1ElapsedMs, (long) v2ElapsedMs, @@ -464,7 +466,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { } long elapsedTimeMs = timer.elapsedTimeMs(); - System.out.println(String.format( + LOG.info(String.format( "v%1$s: bytesRead=%2$d, elapsedMs=%3$d, Mbps=%4$.2f," + " afterReverseSeek=%5$s", version, @@ -496,7 +498,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { } while (bytesRead > 0 && totalBytesRead < minBytesToRead); long elapsedTimeMs = timer.elapsedTimeMs(); inputStream.close(); - System.out.println(String.format( + LOG.info(String.format( "v%1$d: totalBytesRead=%2$d, elapsedTimeMs=%3$d, Mbps=%4$.2f", version, totalBytesRead, @@ -535,7 +537,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { character = (character == 'z') ? 'a' : (char) ((int) character + 1); } - System.out.println(String.format("Creating test file %s of size: %d ", TEST_FILE_PATH, TEST_FILE_SIZE)); + LOG.info(String.format("Creating test file %s of size: %d ", TEST_FILE_PATH, TEST_FILE_SIZE)); ContractTestUtils.NanoTimer timer = new ContractTestUtils.NanoTimer(); try (FSDataOutputStream outputStream = fs.create(TEST_FILE_PATH)) { @@ -544,7 +546,7 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { outputStream.write(buffer); bytesWritten += buffer.length; } - System.out.println(String.format("Closing stream %s", outputStream)); + LOG.info("Closing stream {}", outputStream); ContractTestUtils.NanoTimer closeTimer = new ContractTestUtils.NanoTimer(); outputStream.close(); @@ -578,4 +580,4 @@ public class ITestAzureBlobFileSystemRandomRead extends DependencyInjectedTest { assertArrayEquals("Mismatch in read data", bufferV1, bufferV2); } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java index a0e648c..1a0edaf 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRename.java @@ -18,7 +18,6 @@ package org.apache.hadoop.fs.azurebfs; -import java.io.FileNotFoundException; import java.util.ArrayList; import java.util.List; import java.util.concurrent.Callable; @@ -26,93 +25,74 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; -import org.junit.Assert; import org.junit.Test; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.Path; -import static org.junit.Assert.assertFalse; -import static org.junit.Assert.assertTrue; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertRenameOutcome; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile; /** * Test rename operation. */ -public class ITestAzureBlobFileSystemRename extends DependencyInjectedTest { +public class ITestAzureBlobFileSystemRename extends + AbstractAbfsIntegrationTest { public ITestAzureBlobFileSystemRename() { - super(); } - @Test(expected = FileNotFoundException.class) + @Test public void testEnsureFileIsRenamed() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.create(new Path("testfile")); - fs.rename(new Path("testfile"), new Path("testfile2")); - - FileStatus fileStatus = fs.getFileStatus(new Path("testfile2")); - assertNotNull(fileStatus); - - fs.getFileStatus(new Path("testfile")); + final AzureBlobFileSystem fs = getFileSystem(); + Path src = path("testEnsureFileIsRenamed-src"); + touch(src); + Path dest = path("testEnsureFileIsRenamed-dest"); + fs.delete(dest, true); + assertRenameOutcome(fs, src, dest, true); + + assertIsFile(fs, dest); + assertPathDoesNotExist(fs, "expected renamed", src); } @Test - public void testRenameFile() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - fs.mkdirs(new Path("/testSrc")); - fs.create(new Path("/testSrc/file1")); - - fs.rename(new Path("/testSrc"), new Path("/testDst")); - FileStatus[] fileStatus = fs.listStatus(new Path("/testDst")); - assertNotNull(fileStatus); + public void testRenameFileUnderDir() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + Path sourceDir = new Path("/testSrc"); + assertMkdirs(fs, sourceDir); + String filename = "file1"; + Path file1 = new Path(sourceDir, filename); + touch(file1); + + Path destDir = new Path("/testDst"); + assertRenameOutcome(fs, sourceDir, destDir, true); + FileStatus[] fileStatus = fs.listStatus(destDir); + assertNotNull("Null file status", fileStatus); + FileStatus status = fileStatus[0]; + assertEquals("Wrong filename in " + status, + filename, status.getPath().getName()); } @Test - public void testRenameFileUsingUnicode() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - //known issue: ListStatus operation to folders/files whose name contains '?' will fail - //This is because Auto rest client didn't encode '?' in the uri query parameters - String[] folders1 = new String[]{"/%2c%26", "/ÃáΠâ", "/A +B", "/A~`!@#$%^&*()-_+={};:'>,<B"}; - String[] folders2 = new String[]{"/abcÃâ123", "/abcÃáΠâ123", "/B+ C", "/B~`!@#$%^&*()-_+={};:'>,<C"}; - String[] files = new String[]{"/%2c%27", "/ä¸æ", "/C +D", "/C~`!@#$%^&*()-_+={};:'>,<D"}; - - for (int i = 0; i < 4; i++) { - Path folderPath1 = new Path(folders1[i]); - assertTrue(fs.mkdirs(folderPath1)); - assertTrue(fs.exists(folderPath1)); - - Path filePath = new Path(folders1[i] + files[i]); - fs.create(filePath); - assertTrue(fs.exists(filePath)); - - Path folderPath2 = new Path(folders2[i]); - fs.rename(folderPath1, folderPath2); - assertFalse(fs.exists(folderPath1)); - assertTrue(fs.exists(folderPath2)); - - FileStatus[] fileStatus = fs.listStatus(folderPath2); - assertEquals("/" + fileStatus[0].getPath().getName(), files[i]); - assertNotNull(fileStatus); - } - } - - @Test(expected = FileNotFoundException.class) public void testRenameDirectory() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); fs.mkdirs(new Path("testDir")); - fs.mkdirs(new Path("testDir/test1")); + Path test1 = new Path("testDir/test1"); + fs.mkdirs(test1); fs.mkdirs(new Path("testDir/test1/test2")); fs.mkdirs(new Path("testDir/test1/test2/test3")); - Assert.assertTrue(fs.rename(new Path("testDir/test1"), new Path("testDir/test10"))); - fs.getFileStatus(new Path("testDir/test1")); + assertRenameOutcome(fs, test1, + new Path("testDir/test10"), true); + assertPathDoesNotExist(fs, "rename source dir", test1 ); } - @Test(expected = FileNotFoundException.class) + @Test public void testRenameFirstLevelDirectory() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - final List<Future> tasks = new ArrayList<>(); + final AzureBlobFileSystem fs = getFileSystem(); + final List<Future<Void>> tasks = new ArrayList<>(); ExecutorService es = Executors.newFixedThreadPool(10); for (int i = 0; i < 1000; i++) { @@ -120,7 +100,7 @@ public class ITestAzureBlobFileSystemRename extends DependencyInjectedTest { Callable<Void> callable = new Callable<Void>() { @Override public Void call() throws Exception { - fs.create(fileName); + touch(fileName); return null; } }; @@ -133,20 +113,25 @@ public class ITestAzureBlobFileSystemRename extends DependencyInjectedTest { } es.shutdownNow(); - fs.rename(new Path("/test"), new Path("/renamedDir")); + Path source = new Path("/test"); + Path dest = new Path("/renamedDir"); + assertRenameOutcome(fs, source, dest, true); - FileStatus[] files = fs.listStatus(new Path("/renamedDir")); - Assert.assertEquals(files.length, 1000); - fs.getFileStatus(new Path("/test")); + FileStatus[] files = fs.listStatus(dest); + assertEquals("Wrong number of files in listing", 1000, files.length); + assertPathDoesNotExist(fs, "rename source dir", source); } @Test public void testRenameRoot() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - boolean renamed = fs.rename(new Path("/"), new Path("/ddd")); - assertFalse(renamed); - - renamed = fs.rename(new Path(fs.getUri().toString() + "/"), new Path(fs.getUri().toString() + "/s")); - assertFalse(renamed); + final AzureBlobFileSystem fs = getFileSystem(); + assertRenameOutcome(fs, + new Path("/"), + new Path("/testRenameRoot"), + false); + assertRenameOutcome(fs, + new Path(fs.getUri().toString() + "/"), + new Path(fs.getUri().toString() + "/s"), + false); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java new file mode 100644 index 0000000..0ac7fcf --- /dev/null +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemRenameUnicode.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.fs.azurebfs; + +import java.util.Arrays; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.Path; + +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsDirectory; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertIsFile; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertMkdirs; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathDoesNotExist; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertPathExists; +import static org.apache.hadoop.fs.contract.ContractTestUtils.assertRenameOutcome; + +/** + * Parameterized test of rename operations of unicode paths. + */ +@RunWith(Parameterized.class) +public class ITestAzureBlobFileSystemRenameUnicode extends + AbstractAbfsIntegrationTest { + + @Parameterized.Parameter + public String srcDir; + + @Parameterized.Parameter(1) + public String destDir; + + @Parameterized.Parameter(2) + public String filename; + + @Parameterized.Parameters + public static Iterable<Object[]> params() { + return Arrays.asList( + new Object[][]{ + {"/src", "/dest", "filename"}, + {"/%2c%26", "/abcÃâ123", "%2c%27"}, + {"/ÃáΠâ", "/abcÃáΠâ123", "ä¸æ"}, + {"/A +B", "/B+ C", "C +D"}, + { + "/A~`!@#$%^&*()-_+={};:'>,<B", + "/B~`!@#$%^&*()-_+={};:'>,<C", + "C~`!@#$%^&*()-_+={};:'>,<D" + } + }); + } + + public ITestAzureBlobFileSystemRenameUnicode() { + } + + /** + * Known issue: ListStatus operation to folders/files whose name contains '?' will fail. + * This is because Auto rest client didn't encode '?' in the uri query parameters + */ + @Test + public void testRenameFileUsingUnicode() throws Exception { + final AzureBlobFileSystem fs = getFileSystem(); + Path folderPath1 = new Path(srcDir); + assertMkdirs(fs, folderPath1); + assertIsDirectory(fs, folderPath1); + Path filePath = new Path(folderPath1 + "/" + filename); + touch(filePath); + assertIsFile(fs, filePath); + + Path folderPath2 = new Path(destDir); + assertRenameOutcome(fs, folderPath1, folderPath2, true); + assertPathDoesNotExist(fs, "renamed", folderPath1); + assertIsDirectory(fs, folderPath2); + assertPathExists(fs, "renamed file", new Path(folderPath2 + "/" + filename)); + + FileStatus[] fileStatus = fs.listStatus(folderPath2); + assertNotNull(fileStatus); + assertTrue("Empty listing returned from listStatus(\"" + folderPath2 + "\")", + fileStatus.length > 0); + assertEquals(fileStatus[0].getPath().getName(), filename); + } +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java index 29af1b8..3a44909 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemInitialization.java @@ -20,9 +20,9 @@ package org.apache.hadoop.fs.azurebfs; import java.net.URI; -import org.junit.Assert; import org.junit.Test; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; @@ -30,30 +30,45 @@ import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; /** * Test AzureBlobFileSystem initialization. */ -public class ITestFileSystemInitialization extends DependencyInjectedTest { +public class ITestFileSystemInitialization extends AbstractAbfsIntegrationTest { public ITestFileSystemInitialization() { super(); } @Test public void ensureAzureBlobFileSystemIsInitialized() throws Exception { - final FileSystem fs = this.getFileSystem(); - final String accountName = this.getAccountName(); - final String filesystem = this.getFileSystemName(); + final AzureBlobFileSystem fs = getFileSystem(); + final String accountName = getAccountName(); + final String filesystem = getFileSystemName(); - Assert.assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SCHEME, filesystem + "@" + accountName, null, null, null)); - Assert.assertNotNull(fs.getWorkingDirectory()); + assertEquals(fs.getUri(), + new URI(FileSystemUriSchemes.ABFS_SCHEME, + filesystem + "@" + accountName, + null, + null, + null)); + assertNotNull("working directory", fs.getWorkingDirectory()); } @Test public void ensureSecureAzureBlobFileSystemIsInitialized() throws Exception { - final String accountName = this.getAccountName(); - final String filesystem = this.getFileSystemName(); - final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null); - this.getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); - - final FileSystem fs = this.getFileSystem(); - Assert.assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, filesystem + "@" + accountName, null, null, null)); - Assert.assertNotNull(fs.getWorkingDirectory()); + final String accountName = getAccountName(); + final String filesystem = getFileSystemName(); + final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, + filesystem + "@" + accountName, + null, + null, + null); + Configuration conf = getConfiguration(); + conf.set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); + + try(SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.newInstance(conf)) { + assertEquals(fs.getUri(), new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, + filesystem + "@" + accountName, + null, + null, + null)); + assertNotNull("working directory", fs.getWorkingDirectory()); + } } -} \ No newline at end of file +} http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java index 62d967e..1c71125 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemProperties.java @@ -20,7 +20,6 @@ package org.apache.hadoop.fs.azurebfs; import java.util.Hashtable; -import org.junit.Assert; import org.junit.Ignore; import org.junit.Test; @@ -34,31 +33,29 @@ import static org.junit.Assert.assertEquals; /** * Test FileSystemProperties. */ -public class ITestFileSystemProperties extends DependencyInjectedTest { +public class ITestFileSystemProperties extends AbstractAbfsIntegrationTest { private static final int TEST_DATA = 100; private static final Path TEST_PATH = new Path("/testfile"); public ITestFileSystemProperties() { - super(); } @Test public void testReadWriteBytesToFileAndEnsureThreadPoolCleanup() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); testWriteOneByteToFileAndEnsureThreadPoolCleanup(); - FSDataInputStream inputStream = fs.open(TEST_PATH, 4 * 1024 * 1024); - int i = inputStream.read(); - - assertEquals(TEST_DATA, i); + try(FSDataInputStream inputStream = fs.open(TEST_PATH, 4 * 1024 * 1024)) { + int i = inputStream.read(); + assertEquals(TEST_DATA, i); + } } @Test public void testWriteOneByteToFileAndEnsureThreadPoolCleanup() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); - FSDataOutputStream stream = fs.create(TEST_PATH); - - stream.write(TEST_DATA); - stream.close(); + final AzureBlobFileSystem fs = getFileSystem(); + try(FSDataOutputStream stream = fs.create(TEST_PATH)) { + stream.write(TEST_DATA); + } FileStatus fileStatus = fs.getFileStatus(TEST_PATH); assertEquals(1, fileStatus.getLen()); @@ -67,60 +64,60 @@ public class ITestFileSystemProperties extends DependencyInjectedTest { @Test @Ignore("JDK7 doesn't support PATCH, so PUT is used. Fix is applied in latest test tenant") public void testBase64FileSystemProperties() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); final Hashtable<String, String> properties = new Hashtable<>(); properties.put("key", "{ value: value }"); fs.getAbfsStore().setFilesystemProperties(properties); Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getFilesystemProperties(); - Assert.assertEquals(properties, fetchedProperties); + assertEquals(properties, fetchedProperties); } @Test public void testBase64PathProperties() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); final Hashtable<String, String> properties = new Hashtable<>(); properties.put("key", "{ value: valueTest }"); - fs.create(TEST_PATH); + touch(TEST_PATH); fs.getAbfsStore().setPathProperties(TEST_PATH, properties); Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getPathProperties(TEST_PATH); - Assert.assertEquals(properties, fetchedProperties); + assertEquals(properties, fetchedProperties); } @Test (expected = Exception.class) public void testBase64InvalidFileSystemProperties() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); final Hashtable<String, String> properties = new Hashtable<>(); properties.put("key", "{ value: valueæ² }"); fs.getAbfsStore().setFilesystemProperties(properties); Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getFilesystemProperties(); - Assert.assertEquals(properties, fetchedProperties); + assertEquals(properties, fetchedProperties); } @Test (expected = Exception.class) public void testBase64InvalidPathProperties() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); final Hashtable<String, String> properties = new Hashtable<>(); properties.put("key", "{ value: valueTestå © }"); - fs.create(TEST_PATH); + touch(TEST_PATH); fs.getAbfsStore().setPathProperties(TEST_PATH, properties); Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getPathProperties(TEST_PATH); - Assert.assertEquals(properties, fetchedProperties); + assertEquals(properties, fetchedProperties); } @Test public void testSetFileSystemProperties() throws Exception { - final AzureBlobFileSystem fs = this.getFileSystem(); + final AzureBlobFileSystem fs = getFileSystem(); final Hashtable<String, String> properties = new Hashtable<>(); properties.put("containerForDevTest", "true"); fs.getAbfsStore().setFilesystemProperties(properties); Hashtable<String, String> fetchedProperties = fs.getAbfsStore().getFilesystemProperties(); - Assert.assertEquals(properties, fetchedProperties); + assertEquals(properties, fetchedProperties); } } http://git-wip-us.apache.org/repos/asf/hadoop/blob/75b184c6/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java ---------------------------------------------------------------------- diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java index ef61e52..5d1cf91 100644 --- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java +++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestFileSystemRegistration.java @@ -20,10 +20,9 @@ package org.apache.hadoop.fs.azurebfs; import java.net.URI; -import org.junit.Assert; import org.junit.Test; -import org.apache.hadoop.fs.AbstractFileSystem; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.CommonConfigurationKeysPublic; import org.apache.hadoop.fs.FileContext; import org.apache.hadoop.fs.FileSystem; @@ -31,33 +30,76 @@ import org.apache.hadoop.fs.azurebfs.constants.FileSystemUriSchemes; /** * Test AzureBlobFileSystem registration. + * Use casts to have interesting stack traces on failures. */ -public class ITestFileSystemRegistration extends DependencyInjectedTest { +public class ITestFileSystemRegistration extends AbstractAbfsIntegrationTest { + + protected static final String ABFS = "org.apache.hadoop.fs.azurebfs.Abfs"; + protected static final String ABFSS = "org.apache.hadoop.fs.azurebfs.Abfss"; + public ITestFileSystemRegistration() throws Exception { - super(); + } + + private void assertConfigMatches(Configuration conf, String key, String expected) { + String v = conf.get(key); + assertNotNull("No value for key " + key, v); + assertEquals("Wrong value for key " + key, expected, v); + } + + @Test + public void testAbfsFileSystemRegistered() throws Throwable { + assertConfigMatches(new Configuration(true), + "fs.abfs.impl", + "org.apache.hadoop.fs.azurebfs.AzureBlobFileSystem"); + } + + @Test + public void testSecureAbfsFileSystemRegistered() throws Throwable { + assertConfigMatches(new Configuration(true), + "fs.abfss.impl", + "org.apache.hadoop.fs.azurebfs.SecureAzureBlobFileSystem"); + } + + @Test + public void testAbfsFileContextRegistered() throws Throwable { + assertConfigMatches(new Configuration(true), + "fs.AbstractFileSystem.abfs.impl", + ABFS); + } + + @Test + public void testSecureAbfsFileContextRegistered() throws Throwable { + assertConfigMatches(new Configuration(true), + "fs.AbstractFileSystem.abfss.impl", + ABFSS); } @Test public void ensureAzureBlobFileSystemIsDefaultFileSystem() throws Exception { - FileSystem fs = FileSystem.get(this.getConfiguration()); - Assert.assertTrue(fs instanceof AzureBlobFileSystem); + AzureBlobFileSystem fs = (AzureBlobFileSystem) FileSystem.get(getConfiguration()); + assertNotNull("filesystem", fs); - AbstractFileSystem afs = FileContext.getFileContext(this.getConfiguration()).getDefaultFileSystem(); - Assert.assertTrue(afs instanceof Abfs); + Abfs afs = (Abfs) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem(); + assertNotNull("filecontext", afs); } @Test public void ensureSecureAzureBlobFileSystemIsDefaultFileSystem() throws Exception { - final String accountName = this.getAccountName(); - final String fileSystemName = this.getFileSystemName(); - - final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, fileSystemName + "@" + accountName, null, null, null); - this.getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, defaultUri.toString()); + final String accountName = getAccountName(); + final String fileSystemName = getFileSystemName(); - FileSystem fs = FileSystem.get(this.getConfiguration()); - Assert.assertTrue(fs instanceof SecureAzureBlobFileSystem); + final URI defaultUri = new URI(FileSystemUriSchemes.ABFS_SECURE_SCHEME, + fileSystemName + "@" + accountName, + null, + null, + null); + getConfiguration().set(CommonConfigurationKeysPublic.FS_DEFAULT_NAME_KEY, + defaultUri.toString()); - AbstractFileSystem afs = FileContext.getFileContext(this.getConfiguration()).getDefaultFileSystem(); - Assert.assertTrue(afs instanceof Abfss); + SecureAzureBlobFileSystem fs = (SecureAzureBlobFileSystem) FileSystem.get( + getConfiguration()); + assertNotNull("filesystem", fs); + Abfss afs = (Abfss) FileContext.getFileContext(getConfiguration()).getDefaultFileSystem(); + assertNotNull("filecontext", afs); } -} \ No newline at end of file +} --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org