anmolanmol1234 commented on code in PR #7272:
URL: https://github.com/apache/hadoop/pull/7272#discussion_r1917814590
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemCreate.java:
##########
@@ -504,4 +538,875 @@ private <E extends Throwable> void
validateCreateFileException(final Class<E> ex
private AbfsRestOperationException getMockAbfsRestOperationException(int
status) {
return new AbfsRestOperationException(status, "", "", new Exception());
}
-}
+
+
+ /**
+ * Attempts to test multiple flush calls.
+ */
+ @Test
+ public void testMultipleFlush() throws Throwable {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
+ try (FSDataOutputStream out = fs.create(testPath)) {
+ out.write('1');
+ out.hsync();
+ out.write('2');
+ out.hsync();
+ }
+ }
+
+ /**
+ * Delete the blob before flush and verify that an exception should be
thrown.
+ */
+ @Test
+ public void testDeleteBeforeFlush() throws Throwable {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Path testPath = new Path(TEST_FOLDER_PATH, TEST_CHILD_FILE);
+ try (FSDataOutputStream out = fs.create(testPath)) {
+ out.write('1');
+ fs.delete(testPath, false);
+ out.hsync();
+ // this will cause the next write to failAll
+ } catch (IOException fnfe) {
+ //appendblob outputStream does not generate suppressed exception on
close as it is
+ //single threaded code
+ if
(!fs.getAbfsStore().isAppendBlobKey(fs.makeQualified(testPath).toString())) {
+ // the exception raised in close() must be in the caught exception's
+ // suppressed list
+ Throwable[] suppressed = fnfe.getSuppressed();
+ assertEquals("suppressed count", 1, suppressed.length);
+ Throwable inner = suppressed[0];
+ if (!(inner instanceof IOException)) {
+ throw inner;
+ }
+ GenericTestUtils.assertExceptionContains(fnfe.getMessage(),
inner.getCause(), inner.getCause().getMessage());
+ }
+ }
+ }
+
+ /**
+ * Creating subdirectory on existing file path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsFailsForSubdirectoryOfExistingFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/d"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d/e")));
+
+ Assertions.assertThat(fs.exists(new Path("a/b/c"))).isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/d"))).isTrue();
+ // Asserting directory created still exists as explicit.
+ Assertions.assertThat(
+ DirectoryStateHelper.isExplicitDirectory(new Path("a/b/d"), fs,
+ getTestTracingContext(fs, true)))
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Try creating file same as an existing directory.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateDirectoryAndFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c"))).isTrue();
+ intercept(IOException.class, () -> fs.create(new Path("a/b/c")));
+ // Asserting that directory still exists as explicit
+ Assertions.assertThat(
+ DirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creating same file without specifying overwrite.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ fs.create(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path does not exist")
+ .isTrue();
+ }
+
+ /**
+ * Creating same file with overwrite flag set to false.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameFileWithOverwriteFalse() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path does not exist")
+ .isTrue();
+ intercept(IOException.class, () -> fs.create(new Path("a/b/c"), false));
+ }
+
+ /**
+ * Creation of already existing subpath should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSubPath() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path does not exist")
+ .isTrue();
+ intercept(IOException.class, () -> fs.create(new Path("a/b")));
+ }
+
+ /**
+ * Creating path with parent explicit.
+ */
+ @Test
+ public void testCreatePathParentExplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path does not exist")
+ .isTrue();
+ fs.create(new Path("a/b/c/d"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d")))
+ .describedAs("Path does not exist")
+ .isTrue();
+
+ // asserting that parent stays explicit
+ Assertions.assertThat(
+ DirectoryStateHelper.isExplicitDirectory(new Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Test create on implicit directory with explicit parent.
+ * @throws Exception
+ */
+ @Test
+ public void testParentExplicitPathImplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ Assume.assumeTrue(fs.getAbfsStore().getClientHandler().getIngressClient()
instanceof AbfsBlobClient);
+ fs.mkdirs(new Path("/explicitParent"));
+ String sourcePathName = "/explicitParent/implicitDir";
+ Path sourcePath = new Path(sourcePathName);
+ createAzCopyFolder(sourcePath);
+
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, true));
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, false));
+
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(sourcePath.getParent(),
fs, getTestTracingContext(fs, true)))
+ .describedAs("Parent directory should be explicit.")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(sourcePath,
fs, getTestTracingContext(fs, true)))
+ .describedAs("Path should be implicit.")
+ .isTrue();
+ }
+
+ /**
+ * Test create on implicit directory with implicit parent
+ * @throws Exception
+ */
+ @Test
+ public void testParentImplicitPathImplicit() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+ Assume.assumeTrue(fs.getAbfsStore().getClientHandler().getIngressClient()
instanceof AbfsBlobClient);
+ String parentPathName = "/implicitParent";
+ Path parentPath = new Path(parentPathName);
+ String sourcePathName = "/implicitParent/implicitDir";
+ Path sourcePath = new Path(sourcePathName);
+
+ createAzCopyFolder(parentPath);
+ createAzCopyFolder(sourcePath);
+
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, true));
+ intercept(IOException.class, () ->
+ fs.create(sourcePath, false));
+
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(parentPath,
fs, getTestTracingContext(fs, true)))
+ .describedAs("Parent directory is implicit.")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(sourcePath,
fs, getTestTracingContext(fs, true)))
+ .describedAs("Path should also be implicit.")
+ .isTrue();
+ }
+
+ /**
+ * Tests create file when file exists already
+ * Verifies using eTag for overwrite = true/false
+ */
+ @Test
+ public void testCreateFileExistsImplicitParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+ String parentPathName = "/implicitParent";
+ Path parentPath = new Path(parentPathName);
+ createAzCopyFolder(parentPath);
+
+ String fileName = "/implicitParent/testFile";
+ Path filePath = new Path(fileName);
+ fs.create(filePath);
+ String eTag = extractFileEtag(fileName);
+
+ // testing createFile on already existing file path
+ fs.create(filePath, true);
+
+ String eTagAfterCreateOverwrite = extractFileEtag(fileName);
+
+ Assertions.assertThat(eTag.equals(eTagAfterCreateOverwrite))
+ .describedAs("New file eTag after create overwrite should be different
from old")
+ .isFalse();
+
+ intercept(IOException.class, () ->
+ fs.create(filePath, false));
+
+ String eTagAfterCreate = extractFileEtag(fileName);
+
+ Assertions.assertThat(eTagAfterCreateOverwrite.equals(eTagAfterCreate))
+ .describedAs("File eTag should not change as creation fails")
+ .isTrue();
+
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(parentPath,
fs, getTestTracingContext(fs, true)))
+ .describedAs("Parent path should also change to explicit.")
+ .isTrue();
+ }
+
+ /**
+ * Tests create file when the parent is an existing file
+ * should fail.
+ * @throws Exception FileAlreadyExists for blob and IOException for dfs.
+ */
+ @Test
+ public void testCreateFileParentFile() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AzureBlobFileSystemStore store = fs.getAbfsStore();
+
+ String parentName = "/testParentFile";
+ Path parent = new Path(parentName);
+ fs.create(parent);
+
+ String childName = "/testParentFile/testChildFile";
+ Path child = new Path(childName);
+ IOException e = intercept(IOException.class, () ->
+ fs.create(child, false));
+
+ // asserting that parent stays explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path(parentName)),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not a file")
+ .isFalse();
+ }
+
+ /**
+ * Creating directory on existing file path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateMkdirs() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a/b/c"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d")));
+ }
+
+ /**
+ * Test mkdirs.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirs() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b"));
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.mkdirs(new Path("a/b/c/e"));
+
+ Assertions.assertThat(fs.exists(new Path("a/b")))
+ .describedAs("Path a/b does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d")))
+ .describedAs("Path a/b/c/d does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/e")))
+ .describedAs("Path a/b/c/e does not exist")
+ .isTrue();
+
+ // Asserting that directories created as explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path a/b is not an explicit directory")
+ .isTrue();
+ FileStatus status1 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c/d")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status1.isDirectory())
+ .describedAs("Path a/b/c/d is not an explicit directory")
+ .isTrue();
+ FileStatus status2 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c/e")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status2.isDirectory())
+ .describedAs("Path a/b/c/e is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creating subpath of directory path should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsCreateSubPath() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path a/b/c does not exist")
+ .isTrue();
+ intercept(IOException.class, () -> fs.create(new Path("a/b")));
+
+ // Asserting that directories created as explicit
+ FileStatus status2 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status2.isDirectory())
+ .describedAs("Path a/b/c is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Test creation of directory by level.
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirsByLevel() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a"));
+ fs.mkdirs(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/c/d/e"));
+
+ Assertions.assertThat(fs.exists(new Path("a")))
+ .describedAs("Path a does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path a/b/c does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d/e")))
+ .describedAs("Path a/b/c/d/e does not exist")
+ .isTrue();
+
+ // Asserting that directories created as explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path a is not an explicit directory")
+ .isTrue();
+ FileStatus status1 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status1.isDirectory())
+ .describedAs("Path a/b/c is not an explicit directory")
+ .isTrue();
+ FileStatus status2 = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c/d/e")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status2.isDirectory())
+ .describedAs("Path a/b/c/d/e is not an explicit directory")
+ .isTrue();
+ }
+
+ /*
+ Delete part of a path and validate sub path exists.
+ */
+ @Test
+ public void testMkdirsWithDelete() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b"));
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.delete(new Path("a/b/c/d"));
+ fs.getFileStatus(new Path("a/b/c"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path a/b/c does not exist")
+ .isTrue();
+ }
+
+ /**
+ * Verify mkdir and rename of parent.
+ */
+ @Test
+ public void testMkdirsWithRename() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c/d"));
+ fs.create(new Path("e/file"));
+ fs.delete(new Path("a/b/c/d"));
+ Assertions.assertThat(fs.rename(new Path("e"), new Path("a/b/c/d")))
+ .describedAs("Failed to rename path e to a/b/c/d")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d/file")))
+ .describedAs("Path a/b/c/d/file does not exist")
+ .isTrue();
+ }
+
+ /**
+ * Create a file with name /dir1 and then mkdirs for /dir1/dir2 should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testFileCreateMkdirsRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.setWorkingDirectory(new Path("/"));
+ final Path p1 = new Path("dir1");
+ fs.create(p1);
+ intercept(IOException.class, () -> fs.mkdirs(new Path("dir1/dir2")));
+ }
+
+ /**
+ * Create a file with name /dir1 and then mkdirs for /dir1/dir2 should fail.
+ * @throws Exception
+ */
+ @Test
+ public void testFileCreateMkdirsNonRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path p1 = new Path("dir1");
+ fs.create(p1);
+ intercept(IOException.class, () -> fs.mkdirs(new Path("dir1/dir2")));
+ }
+
+ /**
+ * Creation of same directory without overwrite flag should pass.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ fs.mkdirs(new Path("a/b/c"));
+
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Path a/b/c does not exist")
+ .isTrue();
+ // Asserting that directories created as explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a/b/c")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path a/b/c is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creation of same directory without overwrite flag should pass.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSamePathDirectory() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.create(new Path("a"));
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a")));
+ }
+
+ /**
+ * Creation of directory with root as parent
+ */
+ @Test
+ public void testMkdirOnRootAsParent() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("a");
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("a")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path a is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory on root
+ */
+ @Test
+ public void testMkdirOnRoot() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/");
+ fs.setWorkingDirectory(new Path("/"));
+ fs.mkdirs(path);
+
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(new
Path("/")),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory on path with unicode chars
+ */
+ @Test
+ public void testMkdirUnicode() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/dir\u0031");
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(path),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory on same path with parallel threads.
+ */
+ @Test
+ public void testMkdirParallelRequests() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path path = new Path("/dir1");
+
+ ExecutorService es = Executors.newFixedThreadPool(3);
+
+ List<CompletableFuture<Void>> tasks = new ArrayList<>();
+
+ for (int i = 0; i < 3; i++) {
+ CompletableFuture<Void> future = CompletableFuture.runAsync(() -> {
+ try {
+ fs.mkdirs(path);
+ } catch (IOException e) {
+ throw new CompletionException(e);
+ }
+ }, es);
+ tasks.add(future);
+ }
+
+ // Wait for all the tasks to complete
+ CompletableFuture.allOf(tasks.toArray(new CompletableFuture[0])).join();
+
+ // Assert that the directory created by mkdir exists as explicit
+ FileStatus status = fs.getAbfsStore().getFileStatus(fs.makeQualified(path),
+ new TracingContext(getTestTracingContext(fs, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+
+ /**
+ * Creation of directory with overwrite set to false should not fail
according to DFS code.
+ * @throws Exception
+ */
+ @Test
+ public void testCreateSameDirectoryOverwriteFalse() throws Exception {
+ Configuration configuration = getRawConfiguration();
+ configuration.setBoolean(FS_AZURE_ENABLE_MKDIR_OVERWRITE, false);
+ AzureBlobFileSystem fs1 = (AzureBlobFileSystem)
FileSystem.newInstance(configuration);
+ fs1.mkdirs(new Path("a/b/c"));
+ fs1.mkdirs(new Path("a/b/c"));
+
+ // Asserting that directories created as explicit
+ FileStatus status = fs1.getAbfsStore().getFileStatus(fs1.makeQualified(new
Path("a/b/c")),
+ new TracingContext(getTestTracingContext(fs1, true)));
+ Assertions.assertThat(status.isDirectory())
+ .describedAs("Path is not an explicit directory")
+ .isTrue();
+ }
+
+ /**
+ * Try creating directory same as an existing file.
+ */
+ @Test
+ public void testCreateDirectoryAndFileRecreation() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ fs.mkdirs(new Path("a/b/c"));
+ fs.create(new Path("a/b/c/d"));
+ Assertions.assertThat(fs.exists(new Path("a/b/c")))
+ .describedAs("Directory a/b/c does not exist")
+ .isTrue();
+ Assertions.assertThat(fs.exists(new Path("a/b/c/d")))
+ .describedAs("File a/b/c/d does not exist")
+ .isTrue();
+ intercept(IOException.class, () -> fs.mkdirs(new Path("a/b/c/d")));
+ }
+
+ @Test
+ public void testCreateNonRecursiveForAtomicDirectoryFile() throws Exception {
+ AzureBlobFileSystem fileSystem = getFileSystem();
+ fileSystem.setWorkingDirectory(new Path("/"));
+ fileSystem.mkdirs(new Path("/hbase/dir"));
+ fileSystem.createFile(new Path("/hbase/dir/file"))
+ .overwrite(false)
+ .replication((short) 1)
+ .bufferSize(1024)
+ .blockSize(1024)
+ .build();
+ Assertions.assertThat(fileSystem.exists(new Path("/hbase/dir/file")))
+ .describedAs("File /hbase/dir/file does not exist")
+ .isTrue();
+ }
+
+ @Test
+ public void testMkdirOnNonExistingPathWithImplicitParentDir() throws
Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+ createAzCopyFolder(implicitPath);
+
+ // Creating a directory on non-existing path inside an implicit directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path created by azcopy did not become explicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory created by mkdir does not exist as explicit")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as explicit directory
+ * @throws Exception
+ */
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as explicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingExplicitDirWithImplicitParentDir() throws
Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating implicit directory to be used as parent
+ createAzCopyFolder(implicitPath);
+
+ // Creating an explicit directory on the path first
+ fs.mkdirs(path);
+
+ // Creating a directory on existing explicit directory inside an implicit
directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path created by azcopy did not become explicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory created by mkdir does not exist as explicit")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory with parent directory existing as explicit.
+ * And the directory to be created existing as implicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingImplicitDirWithExplicitParentDir() throws
Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path explicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ // Creating an explicit directory to be used a parent
+ fs.mkdirs(explicitPath);
+
+ createAzCopyFolder(path);
+
+ // Creating a directory on existing implicit directory inside an explicit
directory
+ fs.mkdirs(path);
+
+ // Asserting that the directory created by mkdir exists as explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(explicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Explicit parent directory does not exist as explicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory created by mkdir does not exist as explicit")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as implicit directory
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingImplicitDirWithImplicitParentDir() throws
Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir3");
+ final Path path = new Path("dir3/dir4");
+
+ createAzCopyFolder(implicitPath);
+
+ // Creating an implicit directory on path
+ createAzCopyFolder(path);
+
+ // Creating a directory on existing implicit directory inside an implicit
directory
+ fs.mkdirs(path);
+
+ // Asserting that path created by azcopy becomes explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path created by azcopy did not become explicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory created by mkdir does not exist as explicit")
+ .isTrue();
+ }
+
+ /**
+ * Creation of directory with parent directory existing as implicit.
+ * And the directory to be created existing as file
+ * @throws Exception
+ */
+ @Test
+ public void testMkdirOnExistingFileWithImplicitParentDir() throws Exception {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final Path implicitPath = new Path("dir1");
+ final Path path = new Path("dir1/dir2");
+
+ createAzCopyFolder(implicitPath);
+
+ // Creating a file on path
+ fs.create(path);
+
+ // Creating a directory on existing file inside an implicit directory
+ // Asserting that the mkdir fails
+ LambdaTestUtils.intercept(FileAlreadyExistsException.class, () -> {
+ fs.mkdirs(path);
+ });
+
+ // Asserting that path created by azcopy becomes explicit.
+
Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(implicitPath,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Path created by azcopy did not become explicit")
+ .isTrue();
+
+ // Asserting that the file still exists at path.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(path,
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("File still exists at path")
+ .isFalse();
+ }
+
+ /**
+ * 1. a/b/c as implicit.
+ * 2. Create marker for b.
+ * 3. Do mkdir on a/b/c/d.
+ * 4. Verify all b,c,d have marker but a is implicit.
+ */
+ @Test
+ public void testImplicitExplicitFolder() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ final AzureBlobFileSystem fs = (AzureBlobFileSystem)
FileSystem.newInstance(configuration);
+ final Path implicitPath = new Path("a/b/c");
+
+ createAzCopyFolder(implicitPath);
+
+ Path path = makeQualified(new Path("a/b"));
+ AbfsBlobClient blobClient = (AbfsBlobClient)
fs.getAbfsStore().getClient(AbfsServiceType.BLOB);
+ blobClient.createPath(path.toUri().getPath(), false, true,
+ null, false, null, null, getTestTracingContext(fs, true),
+ true);
+
+ fs.mkdirs(new Path("a/b/c/d"));
+
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(new
Path("a"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a' should be implicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b' should be explicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b/c' should be explicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b/c/d"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b/c/d' should be explicit")
+ .isTrue();
+ }
+
+ /**
+ * 1. a/b/c implicit.
+ * 2. Marker for a and c.
+ * 3. mkdir on a/b/c/d.
+ * 4. Verify a,c,d are explicit but b is implicit.
+ */
+ @Test
+ public void testImplicitExplicitFolder1() throws Exception {
+ Configuration configuration = Mockito.spy(getRawConfiguration());
+ final AzureBlobFileSystem fs = (AzureBlobFileSystem)
FileSystem.newInstance(configuration);
+ final Path implicitPath = new Path("a/b/c");
+
+ createAzCopyFolder(implicitPath);
+
+ Path path = makeQualified(new Path("a"));
+ AbfsBlobClient blobClient = (AbfsBlobClient)
fs.getAbfsStore().getClient(AbfsServiceType.BLOB);
+ blobClient.createPath(path.toUri().getPath(), false, true,
+ null, false, null, null, getTestTracingContext(fs, true), true);
+
+ Path newPath = makeQualified(new Path("a/b/c"));
+ blobClient.createPath(newPath.toUri().getPath(), false, true,
+ null, false, null, null, getTestTracingContext(fs, true), true);
+
+ fs.mkdirs(new Path("a/b/c/d"));
+
+ Assertions.assertThat(DirectoryStateHelper.isImplicitDirectory(new
Path("a/b"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b' should be implicit")
+ .isTrue();
+
+ // Asserting that the directory created by mkdir exists as explicit.
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a' should be explicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b/c"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b/c' should be explicit")
+ .isTrue();
+ Assertions.assertThat(DirectoryStateHelper.isExplicitDirectory(new
Path("a/b/c/d"),
+ fs, getTestTracingContext(fs, true)))
+ .describedAs("Directory 'a/b/c/d' should be explicit")
+ .isTrue();
+ }
+
+ /**
+ * Extracts the eTag for an existing file
+ * @param fileName file Path in String from container root
+ * @return String etag for the file
+ * @throws IOException
+ */
+ private String extractFileEtag(String fileName) throws IOException {
+ final AzureBlobFileSystem fs = getFileSystem();
+ final AbfsClient client = fs.getAbfsClient();
+ final TracingContext testTracingContext = getTestTracingContext(fs, false);
+ AbfsRestOperation op;
+ op = client.getPathStatus(fileName, true, testTracingContext, null);
+ return AzureBlobFileSystemStore.extractEtagHeader(op.getResult());
+ }
+}
Review Comment:
taken
##########
hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azurebfs/ITestAzureBlobFileSystemLease.java:
##########
@@ -370,4 +397,4 @@ public void testAcquireRetry() throws Exception {
tracingContext);
});
}
-}
+}
Review Comment:
taken
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
