This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit d91978555d0b593efa385a8943f0983d7978592c Author: Rakesh Radhakrishnan <[email protected]> AuthorDate: Thu Nov 19 21:23:45 2020 +0530 HDDS-4357: Rename : make rename an atomic ops by updating key path entry in dir/file table (#1557) HDDS-4357: Rename : make rename an atomic ops by updating key path entry in dir/file table --- .../org/apache/hadoop/ozone/om/OMConfigKeys.java | 2 + .../apache/hadoop/ozone/om/helpers/OmKeyInfo.java | 4 + .../hadoop/ozone/om/helpers/OzoneFSUtils.java | 43 +++ .../hadoop/fs/ozone/TestOzoneFileSystem.java | 298 ++++++++++++++++++++- .../hadoop/fs/ozone/TestOzoneFileSystemV1.java | 193 ++++++++++++- .../org/apache/hadoop/ozone/om/KeyManagerImpl.java | 27 +- .../org/apache/hadoop/ozone/om/OzoneManager.java | 2 + .../om/ratis/utils/OzoneManagerRatisUtils.java | 4 + .../om/request/bucket/OMBucketCreateRequest.java | 24 ++ .../ozone/om/request/file/OMFileRequest.java | 94 +++++++ .../ozone/om/request/key/OMKeyRenameRequestV1.java | 292 ++++++++++++++++++++ .../ozone/om/response/key/OMKeyRenameResponse.java | 11 + ...ameResponse.java => OMKeyRenameResponseV1.java} | 68 +++-- .../fs/ozone/BasicOzoneClientAdapterImpl.java | 5 + .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 13 + .../ozone/BasicRootedOzoneClientAdapterImpl.java | 7 + .../apache/hadoop/fs/ozone/OzoneClientAdapter.java | 1 + 17 files changed, 1038 insertions(+), 50 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index c57a202..758f7d3 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -246,4 +246,6 @@ public final class OMConfigKeys { public static final String OZONE_OM_LAYOUT_VERSION = "ozone.om.layout.version"; public static final String OZONE_OM_LAYOUT_VERSION_DEFAULT = "V0"; + + public static final String OZONE_OM_LAYOUT_VERSION_V1 = "V1"; } diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java index 494e1b9..596f43f 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OmKeyInfo.java @@ -281,6 +281,10 @@ public final class OmKeyInfo extends WithObjectID { return OzoneAclUtil.setAcl(acls, newAcls); } + public void setParentObjectID(long parentObjectID) { + this.parentObjectID = parentObjectID; + } + /** * Builder of OmKeyInfo. */ diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java index 63bfd8f..e9d4cf9 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java @@ -162,4 +162,47 @@ public final class OzoneFSUtils { return parentPath.equals(childParent); } + + /** + * The function returns parent directory from the given absolute path. For + * example, the given key path '/a/b/c/d/e/file1' then it returns parent + * directory name as 'e'. + * + * @param keyName key name + * @return parent directory. If not found then return keyName itself. + */ + public static String getParentDir(@Nonnull String keyName) { + java.nio.file.Path fileName = Paths.get(keyName).getParent(); + if (fileName != null) { + return fileName.toString(); + } + // failed to find a parent directory. + return keyName; + } + + /** + * This function appends the given file name to the given key name path. + * + * @param keyName key name + * @param fileName file name + * @return full path + */ + public static String appendFileNameToKeyPath(String keyName, + String fileName) { + StringBuilder newToKeyName = new StringBuilder(keyName); + newToKeyName.append(OZONE_URI_DELIMITER); + newToKeyName.append(fileName); + return newToKeyName.toString(); + } + + /** + * Returns the number of path components in the given keyName. + * + * @param keyName keyname + * @return path components count + */ + public static int getFileCount(String keyName) { + java.nio.file.Path keyPath = Paths.get(keyName); + return keyPath.getNameCount(); + } } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index c9e56a7..b816519 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -267,6 +267,16 @@ public class TestOzoneFileSystem { testNonExplicitlyCreatedPathExistsAfterItsLeafsWereRemoved(); testRenameDir(); + testRenameFile(); + testRenameWithNonExistentSource(); + testRenameDirToItsOwnSubDir(); + testRenameSourceAndDestinAreSame(); + testRenameToExistingDir(); + testRenameToNewSubDirShouldNotExist(); + testRenameDirToFile(); + testRenameFileToDir(); + testRenameDestinationParentDoesntExist(); + testRenameToParentDir(); testSeekOnFileLength(); testDeleteRoot(); @@ -621,6 +631,16 @@ public class TestOzoneFileSystem { stream.seek(fileLength); assertEquals(-1, stream.read()); } + + // non-existent file + Path fileNotExists = new Path("/file_notexist"); + try { + fs.open(fileNotExists); + Assert.fail("Should throw FILE_NOT_FOUND error as file doesn't exist!"); + } catch (FileNotFoundException fnfe) { + Assert.assertTrue("Expected FILE_NOT_FOUND error", + fnfe.getMessage().contains("FILE_NOT_FOUND")); + } } public void testDeleteRoot() throws IOException { @@ -653,8 +673,277 @@ public class TestOzoneFileSystem { interimPath.getName(), fileStatus.getPath().getName()); } - private void testRenameDir() throws Exception { + /** + * Case-1) fromKeyName should exist, otw throws exception. + */ + protected void testRenameWithNonExistentSource() throws Exception { + final String root = "/root"; + final String dir1 = root + "/dir1"; + final String dir2 = root + "/dir2"; + final Path source = new Path(fs.getUri().toString() + dir1); + final Path destin = new Path(fs.getUri().toString() + dir2); + + // creates destin + fs.mkdirs(destin); + LOG.info("Created destin dir: {}", destin); + + LOG.info("Rename op-> source:{} to destin:{}}", source, destin); + assertFalse("Expected to fail rename as src doesn't exist", + fs.rename(source, destin)); + } + + /** + * Case-2) Cannot rename a directory to its own subdirectory. + */ + protected void testRenameDirToItsOwnSubDir() throws Exception { + final String root = "/root"; + final String dir1 = root + "/dir1"; + final Path dir1Path = new Path(fs.getUri().toString() + dir1); + // Add a sub-dir1 to the directory to be moved. + final Path subDir1 = new Path(dir1Path, "sub_dir1"); + fs.mkdirs(subDir1); + LOG.info("Created dir1 {}", subDir1); + + final Path sourceRoot = new Path(fs.getUri().toString() + root); + LOG.info("Rename op-> source:{} to destin:{}", sourceRoot, subDir1); + try { + fs.rename(sourceRoot, subDir1); + Assert.fail("Should throw exception : Cannot rename a directory to" + + " its own subdirectory"); + } catch (IllegalArgumentException iae) { + // expected + } + } + + /** + * Case-3) If src == destin then check source and destin of same type. + */ + protected void testRenameSourceAndDestinAreSame() throws Exception { + final String root = "/root"; + final String dir1 = root + "/dir1"; + final String dir2 = dir1 + "/dir2"; + final Path dir2Path = new Path(fs.getUri().toString() + dir2); + fs.mkdirs(dir2Path); + + // File rename + Path file1 = new Path(fs.getUri().toString() + dir2 + "/file1"); + ContractTestUtils.touch(fs, file1); + + assertTrue(fs.rename(file1, file1)); + assertTrue(fs.rename(dir2Path, dir2Path)); + } + + /** + * Case-4) Rename from /a, to /b. + * <p> + * Expected Result: After rename the directory structure will be /b/a. + */ + protected void testRenameToExistingDir() throws Exception { + // created /a + final Path aSourcePath = new Path(fs.getUri().toString() + "/a"); + fs.mkdirs(aSourcePath); + + // created /b + final Path bDestinPath = new Path(fs.getUri().toString() + "/b"); + fs.mkdirs(bDestinPath); + + // Add a sub-directory '/a/c' to '/a'. This is to verify that after + // rename sub-directory also be moved. + final Path acPath = new Path(fs.getUri().toString() + "/a/c"); + fs.mkdirs(acPath); + + // Rename from /a to /b. + assertTrue("Rename failed", fs.rename(aSourcePath, bDestinPath)); + + final Path baPath = new Path(fs.getUri().toString() + "/b/a"); + final Path bacPath = new Path(fs.getUri().toString() + "/b/a/c"); + assertTrue("Rename failed", fs.exists(baPath)); + assertTrue("Rename failed", fs.exists(bacPath)); + } + + /** + * Case-5) If new destin '/dst/source' exists then throws exception. + * If destination is a directory then rename source as sub-path of it. + * <p> + * For example: rename /a to /b will lead to /b/a. This new path should + * not exist. + */ + protected void testRenameToNewSubDirShouldNotExist() throws Exception { + // Case-5.a) Rename directory from /a to /b. + // created /a + final Path aSourcePath = new Path(fs.getUri().toString() + "/a"); + fs.mkdirs(aSourcePath); + + // created /b + final Path bDestinPath = new Path(fs.getUri().toString() + "/b"); + fs.mkdirs(bDestinPath); + + // Add a sub-directory '/b/a' to '/b'. This is to verify that rename + // throws exception as new destin /b/a already exists. + final Path baPath = new Path(fs.getUri().toString() + "/b/a"); + fs.mkdirs(baPath); + + try { + fs.rename(aSourcePath, bDestinPath); + Assert.fail("Should fail as new destination dir exists!"); + } catch (FileAlreadyExistsException faee) { + // expected as new sub-path /b/a already exists. + } + + // Case-5.b) Rename file from /a/b/c/file1 to /a. + // Should be failed since /a/file1 exists. + final Path abcPath = new Path(fs.getUri().toString() + "/a/b/c"); + fs.mkdirs(abcPath); + Path abcFile1 = new Path(abcPath, "/file1"); + ContractTestUtils.touch(fs, abcFile1); + + final Path aFile1 = new Path(fs.getUri().toString() + "/a/file1"); + ContractTestUtils.touch(fs, aFile1); + + final Path aDestinPath = new Path(fs.getUri().toString() + "/a"); + + try { + fs.rename(abcFile1, aDestinPath); + Assert.fail("Should fail as new destination file exists!"); + } catch (FileAlreadyExistsException faee) { + // expected as new sub-path /a/file1 already exists. + } + } + + /** + * Case-6) Rename directory to an existed file, should be failed. + */ + protected void testRenameDirToFile() throws Exception { + final String root = "/root"; + Path rootPath = new Path(fs.getUri().toString() + root); + fs.mkdirs(rootPath); + + Path file1Destin = new Path(fs.getUri().toString() + root + "/file1"); + ContractTestUtils.touch(fs, file1Destin); + Path abcRootPath = new Path(fs.getUri().toString() + "/a/b/c"); + fs.mkdirs(abcRootPath); + try { + fs.rename(abcRootPath, file1Destin); + Assert.fail("key already exists /root_dir/file1"); + } catch (FileAlreadyExistsException faee) { + // expected + } + } + + /** + * Rename file to a non-existent destin file. + */ + protected void testRenameFile() throws Exception { + final String root = "/root"; + Path rootPath = new Path(fs.getUri().toString() + root); + fs.mkdirs(rootPath); + + Path file1Source = new Path(fs.getUri().toString() + root + + "/file1_Copy"); + ContractTestUtils.touch(fs, file1Source); + Path file1Destin = new Path(fs.getUri().toString() + root + "/file1"); + assertTrue("Renamed failed", fs.rename(file1Source, file1Destin)); + assertTrue("Renamed failed: /root/file1", fs.exists(file1Destin)); + + /** + * Reading several times, this is to verify that OmKeyInfo#keyName cached + * entry is not modified. While reading back, OmKeyInfo#keyName will be + * prepared and assigned to fullkeyPath name. + */ + for (int i = 0; i < 10; i++) { + FileStatus[] fStatus = fs.listStatus(rootPath); + assertEquals("Renamed failed", 1, fStatus.length); + assertEquals("Wrong path name!", file1Destin, fStatus[0].getPath()); + } + } + + /** + * Rename file to an existed directory. + */ + protected void testRenameFileToDir() throws Exception { + final String root = "/root"; + Path rootPath = new Path(fs.getUri().toString() + root); + fs.mkdirs(rootPath); + + Path file1Destin = new Path(fs.getUri().toString() + root + "/file1"); + ContractTestUtils.touch(fs, file1Destin); + Path abcRootPath = new Path(fs.getUri().toString() + "/a/b/c"); + fs.mkdirs(abcRootPath); + assertTrue("Renamed failed", fs.rename(file1Destin, abcRootPath)); + assertTrue("Renamed filed: /a/b/c/file1", fs.exists(new Path(abcRootPath, + "file1"))); + } + + + /** + * Fails if the (a) parent of dst does not exist or (b) parent is a file. + */ + protected void testRenameDestinationParentDoesntExist() throws Exception { + final String root = "/root_dir"; + final String dir1 = root + "/dir1"; + final String dir2 = dir1 + "/dir2"; + final Path dir2SourcePath = new Path(fs.getUri().toString() + dir2); + fs.mkdirs(dir2SourcePath); + + // (a) parent of dst does not exist. /root_dir/b/c + final Path destinPath = new Path(fs.getUri().toString() + root + "/b/c"); + try { + fs.rename(dir2SourcePath, destinPath); + Assert.fail("Should fail as parent of dst does not exist!"); + } catch (FileNotFoundException fnfe) { + // expected + } + + // (b) parent of dst is a file. /root_dir/file1/c + Path filePath = new Path(fs.getUri().toString() + root + "/file1"); + ContractTestUtils.touch(fs, filePath); + + Path newDestinPath = new Path(filePath, "c"); + try { + fs.rename(dir2SourcePath, newDestinPath); + Assert.fail("Should fail as parent of dst is a file!"); + } catch (IOException ioe) { + // expected + } + } + + /** + * Rename to the source's parent directory, it will succeed. + * 1. Rename from /root_dir/dir1/dir2 to /root_dir. + * Expected result : /root_dir/dir2 + * <p> + * 2. Rename from /root_dir/dir1/file1 to /root_dir. + * Expected result : /root_dir/file1. + */ + protected void testRenameToParentDir() throws Exception { + final String root = "/root_dir"; + final String dir1 = root + "/dir1"; + final String dir2 = dir1 + "/dir2"; + final Path dir2SourcePath = new Path(fs.getUri().toString() + dir2); + fs.mkdirs(dir2SourcePath); + final Path destRootPath = new Path(fs.getUri().toString() + root); + + Path file1Source = new Path(fs.getUri().toString() + dir1 + "/file2"); + ContractTestUtils.touch(fs, file1Source); + + // rename source directory to its parent directory(destination). + assertTrue("Rename failed", fs.rename(dir2SourcePath, destRootPath)); + final Path expectedPathAfterRename = + new Path(fs.getUri().toString() + root + "/dir2"); + assertTrue("Rename failed", + fs.exists(expectedPathAfterRename)); + + // rename source file to its parent directory(destination). + assertTrue("Rename failed", fs.rename(file1Source, destRootPath)); + final Path expectedFilePathAfterRename = + new Path(fs.getUri().toString() + root + "/file2"); + assertTrue("Rename failed", + fs.exists(expectedFilePathAfterRename)); + } + + protected void testRenameDir() throws Exception { final String dir = "/root_dir/dir1"; + Path rootDir = new Path(fs.getUri().toString() + "/root_dir"); final Path source = new Path(fs.getUri().toString() + dir); final Path dest = new Path(source.toString() + ".renamed"); // Add a sub-dir to the directory to be moved. @@ -676,6 +965,13 @@ public class TestOzoneFileSystem { // Renaming to same path when src is specified with scheme. assertTrue("Renaming to same path should be success.", fs.rename(source, new Path(dir))); + + // rename root directory + Path rootDestinDir = new Path(fs.getUri().toString() + "/root_dir" + + ".renamed"); + fs.rename(rootDir, rootDestinDir); + assertTrue("Directory rename failed", fs.exists(rootDestinDir)); + assertFalse("Directory rename failed", fs.exists(rootDir)); } private OzoneKeyDetails getKey(Path keyPath, boolean isDirectory) throws IOException { diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java index 415aec8..21e2157 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java @@ -24,6 +24,7 @@ import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.RemoteIterator; import org.apache.hadoop.fs.Path; +import org.apache.hadoop.fs.contract.ContractTestUtils; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.db.Table; import org.apache.hadoop.hdds.utils.db.TableIterator; @@ -31,6 +32,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.jetbrains.annotations.NotNull; @@ -50,9 +52,7 @@ import java.util.Iterator; import java.util.Map; import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.FS_TRASH_INTERVAL_KEY; -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; -import static org.junit.Assert.fail; +import static org.junit.Assert.*; /** * Ozone file system tests that are not covered by contract tests, @@ -260,6 +260,160 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { expectedFilesCount, actualCount); } + /** + * Case-1) fromKeyName should exist, otw throws exception. + */ + protected void testRenameWithNonExistentSource() throws Exception { + final String root = "/root"; + final String dir1 = root + "/dir1"; + final String dir2 = root + "/dir2"; + final Path source = new Path(fs.getUri().toString() + dir1); + final Path destin = new Path(fs.getUri().toString() + dir2); + + // creates destin + fs.mkdirs(destin); + LOG.info("Created destin dir: {}", destin); + + LOG.info("Rename op-> source:{} to destin:{}}", source, destin); + try { + fs.rename(source, destin); + Assert.fail("Should throw exception : Source doesn't exist!"); + } catch (OMException ome) { + // expected + assertEquals(ome.getResult(), OMException.ResultCodes.KEY_NOT_FOUND); + } + } + + /** + * Case-2) Cannot rename a directory to its own subdirectory. + */ + protected void testRenameDirToItsOwnSubDir() throws Exception { + final String root = "/root"; + final String dir1 = root + "/dir1"; + final Path dir1Path = new Path(fs.getUri().toString() + dir1); + // Add a sub-dir1 to the directory to be moved. + final Path subDir1 = new Path(dir1Path, "sub_dir1"); + fs.mkdirs(subDir1); + LOG.info("Created dir1 {}", subDir1); + + final Path sourceRoot = new Path(fs.getUri().toString() + root); + LOG.info("Rename op-> source:{} to destin:{}", sourceRoot, subDir1); + try { + fs.rename(sourceRoot, subDir1); + Assert.fail("Should throw exception : Cannot rename a directory to" + + " its own subdirectory"); + } catch (OMException ome) { + // expected + assertEquals(ome.getResult(), OMException.ResultCodes.KEY_RENAME_ERROR); + } + } + + /** + * Case-5) If new destin '/dst/source' exists then throws exception. + * If destination is a directory then rename source as sub-path of it. + * <p> + * For example: rename /a to /b will lead to /b/a. This new path should + * not exist. + */ + protected void testRenameToNewSubDirShouldNotExist() throws Exception { + // Case-5.a) Rename directory from /a to /b. + // created /a + final Path aSourcePath = new Path(fs.getUri().toString() + "/a"); + fs.mkdirs(aSourcePath); + + // created /b + final Path bDestinPath = new Path(fs.getUri().toString() + "/b"); + fs.mkdirs(bDestinPath); + + // Add a sub-directory '/b/a' to '/b'. This is to verify that rename + // throws exception as new destin /b/a already exists. + final Path baPath = new Path(fs.getUri().toString() + "/b/a"); + fs.mkdirs(baPath); + + try { + fs.rename(aSourcePath, bDestinPath); + Assert.fail("Should fail as new destination dir exists!"); + } catch (OMException ome) { + // expected as new sub-path /b/a already exists. + assertEquals(ome.getResult(), OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + + // Case-5.b) Rename file from /a/b/c/file1 to /a. + // Should be failed since /a/file1 exists. + final Path abcPath = new Path(fs.getUri().toString() + "/a/b/c"); + fs.mkdirs(abcPath); + Path abcFile1 = new Path(abcPath, "/file1"); + ContractTestUtils.touch(fs, abcFile1); + + final Path aFile1 = new Path(fs.getUri().toString() + "/a/file1"); + ContractTestUtils.touch(fs, aFile1); + + final Path aDestinPath = new Path(fs.getUri().toString() + "/a"); + + try { + fs.rename(abcFile1, aDestinPath); + Assert.fail("Should fail as new destination file exists!"); + } catch (OMException ome) { + // expected as new sub-path /b/a already exists. + assertEquals(ome.getResult(), OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } + + /** + * Case-6) Rename directory to an existed file, should be failed. + */ + protected void testRenameDirToFile() throws Exception { + final String root = "/root"; + Path rootPath = new Path(fs.getUri().toString() + root); + fs.mkdirs(rootPath); + + Path file1Destin = new Path(fs.getUri().toString() + root + "/file1"); + ContractTestUtils.touch(fs, file1Destin); + Path abcRootPath = new Path(fs.getUri().toString() + "/a/b/c"); + fs.mkdirs(abcRootPath); + try { + fs.rename(abcRootPath, file1Destin); + Assert.fail("key already exists /root_dir/file1"); + } catch (OMException ome) { + // expected + assertEquals(ome.getResult(), OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } + + /** + * Fails if the (a) parent of dst does not exist or (b) parent is a file. + */ + protected void testRenameDestinationParentDoesntExist() throws Exception { + final String root = "/root_dir"; + final String dir1 = root + "/dir1"; + final String dir2 = dir1 + "/dir2"; + final Path dir2SourcePath = new Path(fs.getUri().toString() + dir2); + fs.mkdirs(dir2SourcePath); + + // (a) parent of dst does not exist. /root_dir/b/c + final Path destinPath = new Path(fs.getUri().toString() + root + "/b/c"); + try { + fs.rename(dir2SourcePath, destinPath); + Assert.fail("Should fail as parent of dst does not exist!"); + } catch (OMException ome) { + // expected + assertEquals(ome.getResult(), OMException.ResultCodes.KEY_RENAME_ERROR); + } + + // (b) parent of dst is a file. /root_dir/file1/c + Path filePath = new Path(fs.getUri().toString() + root + "/file1"); + ContractTestUtils.touch(fs, filePath); + + Path newDestinPath = new Path(filePath, "c"); + try { + fs.rename(dir2SourcePath, newDestinPath); + Assert.fail("Should fail as parent of dst is a file!"); + } catch (OMException ome) { + // expected + assertEquals(ome.getResult(), OMException.ResultCodes.KEY_RENAME_ERROR); + } + } + @Test(timeout = 300_000) @Override public void testFileSystem() throws Exception { @@ -292,6 +446,35 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { tableCleanup(); testListStatusOnLargeDirectory(); tableCleanup(); + + testNonExplicitlyCreatedPathExistsAfterItsLeafsWereRemoved(); + tableCleanup(); + + testRenameDir(); + tableCleanup(); + testRenameFile(); + tableCleanup(); + testRenameWithNonExistentSource(); + tableCleanup(); + testRenameDirToItsOwnSubDir(); + tableCleanup(); + testRenameSourceAndDestinAreSame(); + tableCleanup(); + testRenameToExistingDir(); + tableCleanup(); + testRenameToNewSubDirShouldNotExist(); + tableCleanup(); + testRenameDirToFile(); + tableCleanup(); + testRenameFileToDir(); + tableCleanup(); + testRenameDestinationParentDoesntExist(); + tableCleanup(); + testRenameToParentDir(); + tableCleanup(); + + testSeekOnFileLength(); + tableCleanup(); } /** @@ -348,8 +531,8 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { keyTableIterator.next(); } - Iterator<Map.Entry<CacheKey<String>, CacheValue<OmDirectoryInfo>>> - keyCacheIterator = metadataMgr.getDirectoryTable().cacheIterator(); + Iterator<Map.Entry<CacheKey<String>, CacheValue<OmKeyInfo>>> + keyCacheIterator = metadataMgr.getKeyTable().cacheIterator(); while(keyCacheIterator.hasNext()){ keyCacheIterator.next(); keyCacheIterator.remove(); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java index bdc7ae1..8c326e6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/KeyManagerImpl.java @@ -2056,10 +2056,16 @@ public class KeyManagerImpl implements KeyManager { String volumeName = args.getVolumeName(); String bucketName = args.getBucketName(); String keyName = args.getKeyName(); - OzoneFileStatus fileStatus = getOzoneFileStatus(volumeName, bucketName, - keyName, args.getRefreshPipeline(), args.getSortDatanodes(), - clientAddress); - //if key is not of type file or if key is not found we throw an exception + OzoneFileStatus fileStatus; + if (OzoneManagerRatisUtils.isOmLayoutVersionV1()) { + fileStatus = getOzoneFileStatusV1(volumeName, bucketName, keyName, + args.getSortDatanodes(), clientAddress, false); + } else { + fileStatus = getOzoneFileStatus(volumeName, bucketName, + keyName, args.getRefreshPipeline(), args.getSortDatanodes(), + clientAddress); + } + //if key is not of type file or if key is not found we throw an exception if (fileStatus.isFile()) { return fileStatus.getKeyInfo(); } @@ -2537,13 +2543,18 @@ public class KeyManagerImpl implements KeyManager { continue; } - cacheOmKeyInfo.setFileName(cacheOmKeyInfo.getKeyName()); + // make OmKeyInfo local copy to reset keyname to "fullKeyPath". + // In DB keyName stores only the leaf node but the list + // returning to the user should have full path. + OmKeyInfo omKeyInfo = cacheOmKeyInfo.copyObject(); + + omKeyInfo.setFileName(omKeyInfo.getKeyName()); String fullKeyPath = OMFileRequest.getAbsolutePath(prefixKeyPath, - cacheOmKeyInfo.getKeyName()); - cacheOmKeyInfo.setKeyName(fullKeyPath); + omKeyInfo.getKeyName()); + omKeyInfo.setKeyName(fullKeyPath); countEntries = addKeyInfoToFileStatusList(fileStatusList, prefixKeyInDB, - seekKeyInDB, startKey, countEntries, cacheKey, cacheOmKeyInfo, + seekKeyInDB, startKey, countEntries, cacheKey, omKeyInfo, false); } return countEntries; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index b1f40d4..ccbfb89 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -3693,6 +3693,8 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl boolean omLayoutVersionV1 = StringUtils.equalsIgnoreCase(version, "V1"); OzoneManagerRatisUtils.setOmLayoutVersionV1(omLayoutVersionV1); + LOG.info("Configured {}={} and enabled:{} optimized OM FS operations", + OZONE_OM_LAYOUT_VERSION, version, omLayoutVersionV1); } /** diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java index d4c0f17..d2dd5c7 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/ratis/utils/OzoneManagerRatisUtils.java @@ -44,6 +44,7 @@ import org.apache.hadoop.ozone.om.request.key.OMKeyCreateRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyDeleteRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyPurgeRequest; import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequest; +import org.apache.hadoop.ozone.om.request.key.OMKeyRenameRequestV1; import org.apache.hadoop.ozone.om.request.key.OMKeysRenameRequest; import org.apache.hadoop.ozone.om.request.key.OMTrashRecoverRequest; import org.apache.hadoop.ozone.om.request.key.acl.OMKeyAddAclRequest; @@ -149,6 +150,9 @@ public final class OzoneManagerRatisUtils { case DeleteKeys: return new OMKeysDeleteRequest(omRequest); case RenameKey: + if (omLayoutVersionV1) { + return new OMKeyRenameRequestV1(omRequest); + } return new OMKeyRenameRequest(omRequest); case RenameKeys: return new OMKeysRenameRequest(omRequest); diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java index 678d04e..444b212 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/bucket/OMBucketCreateRequest.java @@ -20,14 +20,18 @@ package org.apache.hadoop.ozone.om.request.bucket; import java.io.IOException; import java.util.ArrayList; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import com.google.common.base.Optional; +import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.ozone.OmUtils; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.helpers.OmVolumeArgs; import org.apache.hadoop.ozone.om.helpers.OzoneAclUtil; import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; @@ -155,6 +159,9 @@ public class OMBucketCreateRequest extends OMClientRequest { getOmRequest()); OmBucketInfo omBucketInfo = OmBucketInfo.getFromProtobuf(bucketInfo); + // Add layout version V1 to bucket info + addLayoutVersionToBucket(ozoneManager, omBucketInfo); + AuditLogger auditLogger = ozoneManager.getAuditLogger(); OzoneManagerProtocolProtos.UserInfo userInfo = getOmRequest().getUserInfo(); @@ -333,4 +340,21 @@ public class OMBucketCreateRequest extends OMClientRequest { } + private void addLayoutVersionToBucket(OzoneManager ozoneManager, + OmBucketInfo omBucketInfo) { + Map<String, String> metadata = omBucketInfo.getMetadata(); + if (metadata == null) { + metadata = new HashMap<>(); + } + OzoneConfiguration configuration = ozoneManager.getConfiguration(); + // TODO: Many unit test cases has null config and done a simple null + // check now. It can be done later, to avoid massive test code changes. + if (configuration != null) { + String layOutVersion = configuration + .get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, + OMConfigKeys.OZONE_OM_LAYOUT_VERSION_DEFAULT); + metadata.put(OMConfigKeys.OZONE_OM_LAYOUT_VERSION, layOutVersion); + omBucketInfo.setMetadata(metadata); + } + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java index 5225d82..e7b43d6 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/file/OMFileRequest.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hdds.utils.db.cache.CacheValue; import org.apache.hadoop.ozone.OzoneAcl; import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; @@ -665,4 +666,97 @@ public final class OMFileRequest { } return prefixName.concat(OzoneConsts.OZONE_URI_DELIMITER).concat(fileName); } + + /** + * Build DirectoryInfo from OmKeyInfo. + * + * @param keyInfo omKeyInfo + * @return omDirectoryInfo object + */ + public static OmDirectoryInfo getDirectoryInfo(OmKeyInfo keyInfo){ + OmDirectoryInfo.Builder builder = new OmDirectoryInfo.Builder(); + builder.setParentObjectID(keyInfo.getParentObjectID()); + builder.setAcls(keyInfo.getAcls()); + builder.addAllMetadata(keyInfo.getMetadata()); + builder.setCreationTime(keyInfo.getCreationTime()); + builder.setModificationTime(keyInfo.getModificationTime()); + builder.setObjectID(keyInfo.getObjectID()); + builder.setUpdateID(keyInfo.getUpdateID()); + builder.setName(OzoneFSUtils.getFileName(keyInfo.getKeyName())); + return builder.build(); + } + + /** + * Verify that the given toKey directory is a sub directory of fromKey + * directory. + * <p> + * For example, special case of renaming a directory to its own + * sub-directory is not allowed. + * + * @param fromKeyName source path + * @param toKeyName destination path + * @param isDir true represents a directory type otw a file type + * @throws OMException if the dest dir is a sub-dir of source dir. + */ + public static void verifyToDirIsASubDirOfFromDirectory(String fromKeyName, + String toKeyName, boolean isDir) throws OMException { + if (!isDir) { + return; + } + Path dstParent = Paths.get(toKeyName).getParent(); + while (dstParent != null) { + if (Paths.get(fromKeyName).equals(dstParent)) { + throw new OMException("Cannot rename a directory to its own " + + "subdirectory", OMException.ResultCodes.KEY_RENAME_ERROR); + // TODO: Existing rename throws java.lang.IllegalArgumentException. + // Should we throw same exception ? + } + dstParent = dstParent.getParent(); + } + } + + /** + * Verify parent exists for the destination path and return destination + * path parent Id. + * <p> + * Check whether dst parent dir exists or not. If the parent exists, then the + * source can be renamed to dst path. + * + * @param volumeName volume name + * @param bucketName bucket name + * @param toKeyName destination path + * @param fromKeyName source path + * @param metaMgr metadata manager + * @throws IOException if the destination parent dir doesn't exists. + */ + public static long getToKeyNameParentId(String volumeName, + String bucketName, String toKeyName, String fromKeyName, + OMMetadataManager metaMgr) throws IOException { + + int totalDirsCount = OzoneFSUtils.getFileCount(toKeyName); + // skip parent is root '/' + if (totalDirsCount <= 1) { + String bucketKey = metaMgr.getBucketKey(volumeName, bucketName); + OmBucketInfo omBucketInfo = + metaMgr.getBucketTable().get(bucketKey); + return omBucketInfo.getObjectID(); + } + + String toKeyParentDir = OzoneFSUtils.getParentDir(toKeyName); + + OzoneFileStatus toKeyParentDirStatus = getOMKeyInfoIfExists(metaMgr, + volumeName, bucketName, toKeyParentDir, 0); + // check if the immediate parent exists + if (toKeyParentDirStatus == null) { + throw new OMException(String.format( + "Failed to rename %s to %s, %s doesn't exist", fromKeyName, + toKeyName, toKeyParentDir), + OMException.ResultCodes.KEY_RENAME_ERROR); + } else if (toKeyParentDirStatus.isFile()){ + throw new OMException(String.format( + "Failed to rename %s to %s, %s is a file", fromKeyName, toKeyName, + toKeyParentDir), OMException.ResultCodes.KEY_RENAME_ERROR); + } + return toKeyParentDirStatus.getKeyInfo().getObjectID(); + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestV1.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestV1.java new file mode 100644 index 0000000..74e53fe --- /dev/null +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyRenameRequestV1.java @@ -0,0 +1,292 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * <p> + * http://www.apache.org/licenses/LICENSE-2.0 + * <p> + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.ozone.om.request.key; + +import com.google.common.base.Optional; +import org.apache.hadoop.hdds.utils.db.Table; +import org.apache.hadoop.hdds.utils.db.cache.CacheKey; +import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; +import org.apache.hadoop.ozone.audit.AuditLogger; +import org.apache.hadoop.ozone.audit.OMAction; +import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.OMMetrics; +import org.apache.hadoop.ozone.om.OzoneManager; +import org.apache.hadoop.ozone.om.exceptions.OMException; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; +import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; +import org.apache.hadoop.ozone.om.helpers.OzoneFileStatus; +import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerDoubleBufferHelper; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; +import org.apache.hadoop.ozone.om.request.util.OmResponseUtil; +import org.apache.hadoop.ozone.om.response.OMClientResponse; +import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponse; +import org.apache.hadoop.ozone.om.response.key.OMKeyRenameResponseV1; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.*; +import org.apache.hadoop.ozone.security.acl.IAccessAuthorizer; +import org.apache.hadoop.ozone.security.acl.OzoneObj; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import java.io.IOException; +import java.util.Map; + +import static org.apache.hadoop.ozone.om.exceptions.OMException.ResultCodes.KEY_NOT_FOUND; +import static org.apache.hadoop.ozone.om.lock.OzoneManagerLock.Resource.BUCKET_LOCK; + +/** + * Handles rename key request layout version V1. + */ +public class OMKeyRenameRequestV1 extends OMKeyRenameRequest { + + private static final Logger LOG = + LoggerFactory.getLogger(OMKeyRenameRequestV1.class); + + public OMKeyRenameRequestV1(OMRequest omRequest) { + super(omRequest); + } + + @Override + @SuppressWarnings("methodlength") + public OMClientResponse validateAndUpdateCache(OzoneManager ozoneManager, + long trxnLogIndex, OzoneManagerDoubleBufferHelper omDoubleBufferHelper) { + + RenameKeyRequest renameKeyRequest = getOmRequest().getRenameKeyRequest(); + KeyArgs keyArgs = renameKeyRequest.getKeyArgs(); + Map<String, String> auditMap = buildAuditMap(keyArgs, renameKeyRequest); + + String volumeName = keyArgs.getVolumeName(); + String bucketName = keyArgs.getBucketName(); + String fromKeyName = keyArgs.getKeyName(); + String toKeyName = renameKeyRequest.getToKeyName(); + + OMMetrics omMetrics = ozoneManager.getMetrics(); + omMetrics.incNumKeyRenames(); + + AuditLogger auditLogger = ozoneManager.getAuditLogger(); + + OMResponse.Builder omResponse = OmResponseUtil.getOMResponseBuilder( + getOmRequest()); + + OMMetadataManager omMetadataManager = ozoneManager.getMetadataManager(); + boolean acquiredLock = false; + OMClientResponse omClientResponse = null; + IOException exception = null; + OmKeyInfo fromKeyValue; + String fromKey = null; + Result result; + try { + if (toKeyName.length() == 0 || fromKeyName.length() == 0) { + throw new OMException("Key name is empty", + OMException.ResultCodes.INVALID_KEY_NAME); + } + + keyArgs = resolveBucketLink(ozoneManager, keyArgs, auditMap); + volumeName = keyArgs.getVolumeName(); + bucketName = keyArgs.getBucketName(); + + // check Acls to see if user has access to perform delete operation on + // old key and create operation on new key + checkKeyAcls(ozoneManager, volumeName, bucketName, fromKeyName, + IAccessAuthorizer.ACLType.DELETE, OzoneObj.ResourceType.KEY); + checkKeyAcls(ozoneManager, volumeName, bucketName, toKeyName, + IAccessAuthorizer.ACLType.CREATE, OzoneObj.ResourceType.KEY); + + acquiredLock = omMetadataManager.getLock().acquireWriteLock(BUCKET_LOCK, + volumeName, bucketName); + + // Validate bucket and volume exists or not. + validateBucketAndVolume(omMetadataManager, volumeName, bucketName); + + // Check if fromKey exists + OzoneFileStatus fromKeyFileStatus = + OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, volumeName, + bucketName, fromKeyName, 0); + // case-1) fromKeyName should exist, otw throws exception + if (fromKeyFileStatus == null) { + // TODO: Add support for renaming open key + throw new OMException("Key not found " + fromKey, KEY_NOT_FOUND); + } + + // source existed + fromKeyValue = fromKeyFileStatus.getKeyInfo(); + boolean isRenameDirectory = fromKeyFileStatus.isDirectory(); + + // case-2) Cannot rename a directory to its own subdirectory + OMFileRequest.verifyToDirIsASubDirOfFromDirectory(fromKeyName, + toKeyName, fromKeyFileStatus.isDirectory()); + + OzoneFileStatus toKeyFileStatus = + OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, + volumeName, bucketName, toKeyName, 0); + + // Check if toKey exists. + if(toKeyFileStatus != null) { + // Destination exists and following are different cases: + OmKeyInfo toKeyValue = toKeyFileStatus.getKeyInfo(); + + if (fromKeyValue.getKeyName().equals(toKeyValue.getKeyName())) { + // case-3) If src == destin then check source and destin of same type + // (a) If dst is a file then return true. + // (b) Otherwise throws exception. + // TODO: Discuss do we need to throw exception for file as well. + if (toKeyFileStatus.isFile()) { + result = Result.SUCCESS; + } else { + throw new OMException("Key already exists " + toKeyName, + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else if (toKeyFileStatus.isDirectory()) { + // case-4) If dst is a directory then rename source as sub-path of it + // For example: rename /source to /dst will lead to /dst/source + String fromFileName = OzoneFSUtils.getFileName(fromKeyName); + String newToKeyName = OzoneFSUtils.appendFileNameToKeyPath(toKeyName, + fromFileName); + OzoneFileStatus newToOzoneFileStatus = + OMFileRequest.getOMKeyInfoIfExists(omMetadataManager, + volumeName, bucketName, newToKeyName, 0); + + if (newToOzoneFileStatus != null) { + // case-5) If new destin '/dst/source' exists then throws exception + throw new OMException(String.format( + "Failed to rename %s to %s, file already exists or not " + + "empty!", fromKeyName, newToKeyName), + OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + + omClientResponse = renameKey(toKeyValue.getObjectID(), trxnLogIndex, + fromKeyValue, isRenameDirectory, newToKeyName, + keyArgs.getModificationTime(), omResponse, ozoneManager); + result = Result.SUCCESS; + } else { + // case-6) If destination is a file type and if exists then throws + // key already exists exception. + throw new OMException("Failed to rename, key already exists " + + toKeyName, OMException.ResultCodes.KEY_ALREADY_EXISTS); + } + } else { + // Destination doesn't exist and the cases are: + // case-7) Check whether dst parent dir exists or not. If parent + // doesn't exist then throw exception, otw the source can be renamed to + // destination path. + long toKeyParentId = OMFileRequest.getToKeyNameParentId(volumeName, + bucketName, toKeyName, fromKeyName, omMetadataManager); + + omClientResponse = renameKey(toKeyParentId, trxnLogIndex, + fromKeyValue, isRenameDirectory, toKeyName, + keyArgs.getModificationTime(), omResponse, ozoneManager); + + result = Result.SUCCESS; + } + } catch (IOException ex) { + result = Result.FAILURE; + exception = ex; + omClientResponse = new OMKeyRenameResponse(createErrorOMResponse( + omResponse, exception)); + } finally { + addResponseToDoubleBuffer(trxnLogIndex, omClientResponse, + omDoubleBufferHelper); + if (acquiredLock) { + omMetadataManager.getLock().releaseWriteLock(BUCKET_LOCK, volumeName, + bucketName); + } + } + + auditLog(auditLogger, buildAuditMessage(OMAction.RENAME_KEY, auditMap, + exception, getOmRequest().getUserInfo())); + + switch (result) { + case SUCCESS: + LOG.debug("Rename Key is successfully completed for volume:{} bucket:{}" + + " fromKey:{} toKey:{}. ", volumeName, bucketName, + fromKeyName, toKeyName); + break; + case FAILURE: + ozoneManager.getMetrics().incNumKeyRenameFails(); + LOG.error("Rename key failed for volume:{} bucket:{} fromKey:{} " + + "toKey:{}. Key: {} not found.", volumeName, bucketName, + fromKeyName, toKeyName, fromKeyName); + break; + default: + LOG.error("Unrecognized Result for OMKeyRenameRequest: {}", + renameKeyRequest); + } + return omClientResponse; + } + + @SuppressWarnings("parameternumber") + private OMClientResponse renameKey(long toKeyParentId, + long trxnLogIndex, OmKeyInfo fromKeyValue, boolean isRenameDirectory, + String toKeyName, long modificationTime, OMResponse.Builder omResponse, + OzoneManager ozoneManager) { + + String dbFromKey = fromKeyValue.getPath(); + String toKeyFileName = OzoneFSUtils.getFileName(toKeyName); + + fromKeyValue.setUpdateID(trxnLogIndex, ozoneManager.isRatisEnabled()); + // Set toFileName + fromKeyValue.setKeyName(toKeyFileName); + fromKeyValue.setFileName(toKeyFileName); + // Set toKeyObjectId + fromKeyValue.setParentObjectID(toKeyParentId); + //Set modification time + fromKeyValue.setModificationTime(modificationTime); + + // destination dbKeyName + String dbToKey = fromKeyValue.getPath(); + + // Add to cache. + // dbFromKey should be deleted, dbToKey should be added with newly updated + // omKeyInfo. + // Add from_key and to_key details into cache. + OMMetadataManager metadataMgr = ozoneManager.getMetadataManager(); + if (isRenameDirectory) { + Table<String, OmDirectoryInfo> dirTable = metadataMgr.getDirectoryTable(); + dirTable.addCacheEntry(new CacheKey<>(dbFromKey), + new CacheValue<>(Optional.absent(), trxnLogIndex)); + + dirTable.addCacheEntry(new CacheKey<>(dbToKey), + new CacheValue<>(Optional.of(OMFileRequest. + getDirectoryInfo(fromKeyValue)), trxnLogIndex)); + } else { + Table<String, OmKeyInfo> keyTable = metadataMgr.getKeyTable(); + + keyTable.addCacheEntry(new CacheKey<>(dbFromKey), + new CacheValue<>(Optional.absent(), trxnLogIndex)); + + keyTable.addCacheEntry(new CacheKey<>(dbToKey), + new CacheValue<>(Optional.of(fromKeyValue), trxnLogIndex)); + } + + OMClientResponse omClientResponse = new OMKeyRenameResponseV1(omResponse + .setRenameKeyResponse(RenameKeyResponse.newBuilder()).build(), + dbFromKey, dbToKey, fromKeyValue, isRenameDirectory); + return omClientResponse; + } + + private Map<String, String> buildAuditMap( + KeyArgs keyArgs, RenameKeyRequest renameKeyRequest) { + Map<String, String> auditMap = buildKeyArgsAuditMap(keyArgs); + auditMap.remove(OzoneConsts.KEY); + auditMap.put(OzoneConsts.SRC_KEY, keyArgs.getKeyName()); + auditMap.put(OzoneConsts.DST_KEY, renameKeyRequest.getToKeyName()); + return auditMap; + } +} diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java index 7470b37..3b7edf1 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java @@ -70,4 +70,15 @@ public class OMKeyRenameResponse extends OMClientResponse { renameKeyInfo); } + public OmKeyInfo getRenameKeyInfo() { + return renameKeyInfo; + } + + public String getFromKeyName() { + return fromKeyName; + } + + public String getToKeyName() { + return toKeyName; + } } diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseV1.java similarity index 50% copy from hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java copy to hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseV1.java index 7470b37..7a9b159 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponse.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/response/key/OMKeyRenameResponseV1.java @@ -18,56 +18,52 @@ package org.apache.hadoop.ozone.om.response.key; +import org.apache.hadoop.hdds.utils.db.BatchOperation; import org.apache.hadoop.ozone.om.OMMetadataManager; +import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; +import org.apache.hadoop.ozone.om.request.file.OMFileRequest; import org.apache.hadoop.ozone.om.response.CleanupTableInfo; -import org.apache.hadoop.ozone.om.response.OMClientResponse; -import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos - .OMResponse; -import org.apache.hadoop.hdds.utils.db.BatchOperation; +import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse; -import java.io.IOException; import javax.annotation.Nonnull; +import java.io.IOException; -import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.KEY_TABLE; +import static org.apache.hadoop.ozone.om.OmMetadataManagerImpl.*; /** - * Response for RenameKey request. + * Response for RenameKey request layout version V1. */ -@CleanupTableInfo(cleanupTables = {KEY_TABLE}) -public class OMKeyRenameResponse extends OMClientResponse { - - private String fromKeyName; - private String toKeyName; - private OmKeyInfo renameKeyInfo; +@CleanupTableInfo(cleanupTables = {FILE_TABLE, DIRECTORY_TABLE}) +public class OMKeyRenameResponseV1 extends OMKeyRenameResponse { - public OMKeyRenameResponse(@Nonnull OMResponse omResponse, - String fromKeyName, String toKeyName, @Nonnull OmKeyInfo renameKeyInfo) { - super(omResponse); - this.fromKeyName = fromKeyName; - this.toKeyName = toKeyName; - this.renameKeyInfo = renameKeyInfo; - } + private boolean isRenameDirectory; - /** - * For when the request is not successful. - * For a successful request, the other constructor should be used. - */ - public OMKeyRenameResponse(@Nonnull OMResponse omResponse) { - super(omResponse); - checkStatusNotOK(); + public OMKeyRenameResponseV1(@Nonnull OMResponse omResponse, + String fromKeyName, String toKeyName, @Nonnull OmKeyInfo renameKeyInfo, + boolean isRenameDirectory) { + super(omResponse, fromKeyName, toKeyName, renameKeyInfo); + this.isRenameDirectory = isRenameDirectory; } @Override public void addToDBBatch(OMMetadataManager omMetadataManager, - BatchOperation batchOperation) throws IOException { - String volumeName = renameKeyInfo.getVolumeName(); - String bucketName = renameKeyInfo.getBucketName(); - omMetadataManager.getKeyTable().deleteWithBatch(batchOperation, - omMetadataManager.getOzoneKey(volumeName, bucketName, fromKeyName)); - omMetadataManager.getKeyTable().putWithBatch(batchOperation, - omMetadataManager.getOzoneKey(volumeName, bucketName, toKeyName), - renameKeyInfo); - } + BatchOperation batchOperation) throws IOException { + + if (isRenameDirectory) { + omMetadataManager.getDirectoryTable().deleteWithBatch(batchOperation, + getFromKeyName()); + OmDirectoryInfo renameDirInfo = + OMFileRequest.getDirectoryInfo(getRenameKeyInfo()); + omMetadataManager.getDirectoryTable().putWithBatch(batchOperation, + getToKeyName(), renameDirInfo); + + } else { + omMetadataManager.getKeyTable().deleteWithBatch(batchOperation, + getFromKeyName()); + omMetadataManager.getKeyTable().putWithBatch(batchOperation, + getToKeyName(), getRenameKeyInfo()); + } + } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java index 0b50988..5ba7211 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneClientAdapterImpl.java @@ -47,6 +47,7 @@ import org.apache.hadoop.ozone.client.OzoneClientFactory; import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -521,4 +522,8 @@ public class BasicOzoneClientAdapterImpl implements OzoneClientAdapter { return blockLocations; } + @Override + public String getBucketLayoutVersion() { + return bucket.getMetadata().get(OMConfigKeys.OZONE_OM_LAYOUT_VERSION); + } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index f6d2ef5..b33144a 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.annotation.InterfaceStability; import org.apache.hadoop.hdds.conf.ConfigurationSource; import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.utils.LegacyHadoopConfigurationSource; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils; import org.apache.hadoop.security.UserGroupInformation; @@ -315,6 +316,7 @@ public class BasicOzoneFileSystem extends FileSystem { String srcPath = src.toUri().getPath(); String dstPath = dst.toUri().getPath(); + // TODO: Discuss do we need to throw exception. if (srcPath.equals(dstPath)) { return true; } @@ -326,6 +328,12 @@ public class BasicOzoneFileSystem extends FileSystem { return false; } + String layOutVersion = adapter.getBucketLayoutVersion(); + if (layOutVersion != null && + OMConfigKeys.OZONE_OM_LAYOUT_VERSION_V1.equals(layOutVersion)) { + return renameV1(srcPath, dstPath); + } + // Check if the source exists FileStatus srcStatus; try { @@ -406,6 +414,11 @@ public class BasicOzoneFileSystem extends FileSystem { return result; } + private boolean renameV1(String srcPath, String dstPath) throws IOException { + adapter.renameKey(srcPath, dstPath); + return true; + } + /** * Intercept rename to trash calls from TrashPolicyDefault. */ diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java index 848119d..1d749d8 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneClientAdapterImpl.java @@ -57,6 +57,7 @@ import org.apache.hadoop.ozone.client.OzoneKey; import org.apache.hadoop.ozone.client.OzoneVolume; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; +import org.apache.hadoop.ozone.om.OMConfigKeys; import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyLocationInfo; @@ -1028,4 +1029,10 @@ public class BasicRootedOzoneClientAdapterImpl null, null, null, new BlockLocation[0] ); } + + @Override + public String getBucketLayoutVersion() { + // TODO: Need to refine this part. + return OMConfigKeys.OZONE_OM_LAYOUT_VERSION_DEFAULT; + } } diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java index 2b76c22..be93bd6 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/OzoneClientAdapter.java @@ -75,4 +75,5 @@ public interface OzoneClientAdapter { FileStatusAdapter getFileStatus(String key, URI uri, Path qualifiedPath, String userName) throws IOException; + String getBucketLayoutVersion(); } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
