This is an automated email from the ASF dual-hosted git repository. rakeshr pushed a commit to branch HDDS-2939 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit c2d28796b64c0f2a999431cb96f4fbbf5bad2a58 Author: Rakesh Radhakrishnan <[email protected]> AuthorDate: Mon Feb 8 19:29:09 2021 +0530 HDDS-4805. [FSO]Fix findbugs issues after HDDS-2195 (#1906) --- .../apache/hadoop/fs/ozone/TestOzoneFileOps.java | 15 +++- .../hadoop/fs/ozone/TestOzoneFileSystem.java | 60 ++++++++------ .../hadoop/fs/ozone/TestOzoneFileSystemV1.java | 94 +++++++++++----------- .../hadoop/ozone/client/rpc/TestReadRetries.java | 10 +-- .../apache/hadoop/ozone/om/TestObjectStoreV1.java | 39 ++++++--- .../file/TestOMDirectoryCreateRequestV1.java | 20 ++--- .../om/request/key/TestOMKeyCreateRequestV1.java | 5 +- .../TestS3InitiateMultipartUploadRequestV1.java | 4 - .../file/TestOMDirectoryCreateResponseV1.java | 2 - .../response/file/TestOMFileCreateResponseV1.java | 6 +- .../om/response/key/TestOMKeyCommitResponse.java | 2 - .../om/response/key/TestOMKeyCreateResponseV1.java | 6 +- .../TestS3InitiateMultipartUploadResponseV1.java | 5 -- 13 files changed, 147 insertions(+), 121 deletions(-) diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java index 12dd51e..176d0c4 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileOps.java @@ -36,6 +36,7 @@ import org.apache.hadoop.ozone.om.helpers.OmBucketInfo; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.After; import org.junit.Before; @@ -155,12 +156,20 @@ public class TestOzoneFileOps { // trigger CommitKeyRequest outputStream.close(); - Assert.assertTrue("Failed to commit the open file:" + openFileKey, - omMgr.getOpenKeyTable().isEmpty()); - OmKeyInfo omKeyInfo = omMgr.getKeyTable().get(openFileKey); Assert.assertNotNull("Invalid Key!", omKeyInfo); verifyOMFileInfoFormat(omKeyInfo, file.getName(), d2ObjectID); + + // wait for DB updates + GenericTestUtils.waitFor(() -> { + try { + return omMgr.getOpenKeyTable().isEmpty(); + } catch (IOException e) { + LOG.error("DB failure!", e); + Assert.fail("DB failure!"); + return false; + } + }, 1000, 120000); } private void verifyOMFileInfoFormat(OmKeyInfo omKeyInfo, String fileName, diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index 685aade..a4af35b 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -20,6 +20,7 @@ package org.apache.hadoop.fs.ozone; import java.io.FileNotFoundException; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Arrays; import java.util.Collection; @@ -125,26 +126,16 @@ public class TestOzoneFileSystem { private static final Logger LOG = LoggerFactory.getLogger(TestOzoneFileSystem.class); - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static boolean isBucketFSOptimized = false; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static boolean enabledFileSystemPaths; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static boolean omRatisEnabled; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static MiniOzoneCluster cluster; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static FileSystem fs; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static OzoneFileSystem o3fs; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static String volumeName; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static String bucketName; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static int rootItemCount; - @SuppressWarnings("checkstyle:VisibilityModifier") - protected static Trash trash; + private static boolean isBucketFSOptimized = false; + private static boolean enabledFileSystemPaths; + private static boolean omRatisEnabled; + + private static MiniOzoneCluster cluster; + private static FileSystem fs; + private static OzoneFileSystem o3fs; + private static String volumeName; + private static String bucketName; + private static Trash trash; private void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); @@ -202,6 +193,26 @@ public class TestOzoneFileSystem { } } + public static FileSystem getFs() { + return fs; + } + + public static boolean isEnabledFileSystemPaths() { + return enabledFileSystemPaths; + } + + public static void setIsBucketFSOptimized(boolean isBucketFSO) { + isBucketFSOptimized = isBucketFSO; + } + + public static String getBucketName() { + return bucketName; + } + + public static String getVolumeName() { + return volumeName; + } + @Test public void testCreateFileShouldCheckExistenceOfDirWithSameName() throws Exception { @@ -605,7 +616,7 @@ public class TestOzoneFileSystem { // Added logs for debugging failures, to check any sub-path mismatches. Set<String> actualPaths = new TreeSet<>(); ArrayList<String> actualPathList = new ArrayList<>(); - if (rootItemCount != fileStatuses.length) { + if (numDirs != fileStatuses.length) { for (int i = 0; i < fileStatuses.length; i++) { boolean duplicate = actualPaths.add(fileStatuses[i].getPath().getName()); @@ -615,7 +626,7 @@ public class TestOzoneFileSystem { } actualPathList.add(fileStatuses[i].getPath().getName()); } - if (rootItemCount != actualPathList.size()) { + if (numDirs != actualPathList.size()) { LOG.info("actualPathsSize: {}", actualPaths.size()); LOG.info("actualPathListSize: {}", actualPathList.size()); actualPaths.removeAll(paths); @@ -642,8 +653,6 @@ public class TestOzoneFileSystem { Path root = new Path("/"); FileStatus[] fileStatuses = fs.listStatus(root); - rootItemCount = 0; // reset to zero - if (fileStatuses == null) { return; } @@ -722,7 +731,7 @@ public class TestOzoneFileSystem { public void testAllocateMoreThanOneBlock() throws IOException { Path file = new Path("/file"); String str = "TestOzoneFileSystemV1.testSeekOnFileLength"; - byte[] strBytes = str.getBytes(); + byte[] strBytes = str.getBytes(StandardCharsets.UTF_8); long numBlockAllocationsOrg = cluster.getOzoneManager().getMetrics().getNumBlockAllocates(); @@ -1051,7 +1060,6 @@ public class TestOzoneFileSystem { @Test public void testRenameDir() throws Exception { final String dir = "/root_dir/dir1"; - Path rootDir = new Path(fs.getUri().toString() + "/root_dir"); final Path source = new Path(fs.getUri().toString() + dir); final Path dest = new Path(source.toString() + ".renamed"); // Add a sub-dir to the directory to be moved. diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java index ffeb5a3..03846ae 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystemV1.java @@ -29,9 +29,7 @@ import org.junit.Assert; import org.junit.After; import org.junit.BeforeClass; import org.junit.Ignore; -import org.junit.Rule; import org.junit.Test; -import org.junit.rules.Timeout; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import org.slf4j.Logger; @@ -66,7 +64,7 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { @BeforeClass public static void init() { - isBucketFSOptimized = true; + setIsBucketFSOptimized(true); } public TestOzoneFileSystemV1(boolean setDefaultFs, boolean enableOMRatis) { @@ -85,12 +83,6 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { } } - /** - * Set a timeout for each test. - */ - @Rule - public Timeout timeout = new Timeout(300000); - private static final Logger LOG = LoggerFactory.getLogger(TestOzoneFileSystemV1.class); @@ -106,30 +98,32 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { * Op 7. create file -> /d1/d2/key1 */ Path key1 = new Path("/key1"); - try (FSDataOutputStream outputStream = fs.create(key1, false)) { + try (FSDataOutputStream outputStream = getFs().create(key1, + false)) { assertNotNull("Should be able to create file: key1", outputStream); } Path d1 = new Path("/d1"); Path dir1Key1 = new Path(d1, "key1"); - try (FSDataOutputStream outputStream = fs.create(dir1Key1, false)) { + try (FSDataOutputStream outputStream = getFs().create(dir1Key1, false)) { assertNotNull("Should be able to create file: " + dir1Key1, outputStream); } Path d2 = new Path("/d2"); Path dir2Key1 = new Path(d2, "key1"); - try (FSDataOutputStream outputStream = fs.create(dir2Key1, false)) { + try (FSDataOutputStream outputStream = getFs().create(dir2Key1, false)) { assertNotNull("Should be able to create file: " + dir2Key1, outputStream); } Path dir1Dir2 = new Path("/d1/d2/"); Path dir1Dir2Key1 = new Path(dir1Dir2, "key1"); - try (FSDataOutputStream outputStream = fs.create(dir1Dir2Key1, false)) { + try (FSDataOutputStream outputStream = getFs().create(dir1Dir2Key1, + false)) { assertNotNull("Should be able to create file: " + dir1Dir2Key1, outputStream); } Path d1Key2 = new Path(d1, "key2"); - try (FSDataOutputStream outputStream = fs.create(d1Key2, false)) { + try (FSDataOutputStream outputStream = getFs().create(d1Key2, false)) { assertNotNull("Should be able to create file: " + d1Key2, outputStream); } @@ -137,11 +131,14 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { Path dir1Dir3 = new Path("/d1/d3/"); Path dir1Dir4 = new Path("/d1/d4/"); - fs.mkdirs(dir1Dir3); - fs.mkdirs(dir1Dir4); + getFs().mkdirs(dir1Dir3); + getFs().mkdirs(dir1Dir4); + + String bucketName = getBucketName(); + String volumeName = getVolumeName(); // Root Directory - FileStatus[] fileStatusList = fs.listStatus(new Path("/")); + FileStatus[] fileStatusList = getFs().listStatus(new Path("/")); assertEquals("FileStatus should return files and directories", 3, fileStatusList.length); ArrayList<String> expectedPaths = new ArrayList<>(); @@ -155,7 +152,7 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { 0, expectedPaths.size()); // level-1 sub-dirs - fileStatusList = fs.listStatus(new Path("/d1")); + fileStatusList = getFs().listStatus(new Path("/d1")); assertEquals("FileStatus should return files and directories", 5, fileStatusList.length); expectedPaths = new ArrayList<>(); @@ -171,7 +168,7 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { 0, expectedPaths.size()); // level-2 sub-dirs - fileStatusList = fs.listStatus(new Path("/d1/d2")); + fileStatusList = getFs().listStatus(new Path("/d1/d2")); assertEquals("FileStatus should return files and directories", 1, fileStatusList.length); expectedPaths = new ArrayList<>(); @@ -184,7 +181,7 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { 0, expectedPaths.size()); // level-2 key2 - fileStatusList = fs.listStatus(new Path("/d1/d2/key1")); + fileStatusList = getFs().listStatus(new Path("/d1/d2/key1")); assertEquals("FileStatus should return files and directories", 1, fileStatusList.length); expectedPaths = new ArrayList<>(); @@ -198,13 +195,13 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { // invalid root key try { - fileStatusList = fs.listStatus(new Path("/key2")); + fileStatusList = getFs().listStatus(new Path("/key2")); fail("Should throw FileNotFoundException"); } catch (FileNotFoundException fnfe) { // ignore as its expected } try { - fileStatusList = fs.listStatus(new Path("/d1/d2/key2")); + fileStatusList = getFs().listStatus(new Path("/d1/d2/key2")); fail("Should throw FileNotFoundException"); } catch (FileNotFoundException fnfe) { // ignore as its expected @@ -221,30 +218,30 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { * Op 4. create dir -> /d1/d2/d1/d2/key1 */ Path dir1Dir1Dir2Key1 = new Path("/d1/d1/d2/key1"); - try (FSDataOutputStream outputStream = fs.create(dir1Dir1Dir2Key1, + try (FSDataOutputStream outputStream = getFs().create(dir1Dir1Dir2Key1, false)) { assertNotNull("Should be able to create file: " + dir1Dir1Dir2Key1, outputStream); } Path key1 = new Path("/key1"); - try (FSDataOutputStream outputStream = fs.create(key1, false)) { + try (FSDataOutputStream outputStream = getFs().create(key1, false)) { assertNotNull("Should be able to create file: " + key1, outputStream); } Path key2 = new Path("/key2"); - try (FSDataOutputStream outputStream = fs.create(key2, false)) { + try (FSDataOutputStream outputStream = getFs().create(key2, false)) { assertNotNull("Should be able to create file: key2", outputStream); } Path dir1Dir2Dir1Dir2Key1 = new Path("/d1/d2/d1/d2/key1"); - try (FSDataOutputStream outputStream = fs.create(dir1Dir2Dir1Dir2Key1, + try (FSDataOutputStream outputStream = getFs().create(dir1Dir2Dir1Dir2Key1, false)) { assertNotNull("Should be able to create file: " + dir1Dir2Dir1Dir2Key1, outputStream); } - RemoteIterator<LocatedFileStatus> fileStatusItr = fs.listFiles(new Path( - "/"), true); - String uriPrefix = "o3fs://" + bucketName + "." + volumeName; + RemoteIterator<LocatedFileStatus> fileStatusItr = getFs().listFiles( + new Path("/"), true); + String uriPrefix = "o3fs://" + getBucketName() + "." + getVolumeName(); ArrayList<String> expectedPaths = new ArrayList<>(); expectedPaths.add(uriPrefix + dir1Dir1Dir2Key1.toString()); expectedPaths.add(uriPrefix + key1.toString()); @@ -263,7 +260,7 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { expectedPaths.size()); // Recursive=false - fileStatusItr = fs.listFiles(new Path("/"), false); + fileStatusItr = getFs().listFiles(new Path("/"), false); expectedPaths.clear(); expectedPaths.add(uriPrefix + "/key1"); expectedPaths.add(uriPrefix + "/key2"); @@ -286,23 +283,23 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { @Test public void testRenameWithNonExistentSource() throws Exception { // Skip as this will run only in new layout - if (!enabledFileSystemPaths) { + if (!isEnabledFileSystemPaths()) { return; } final String root = "/root"; final String dir1 = root + "/dir1"; final String dir2 = root + "/dir2"; - final Path source = new Path(fs.getUri().toString() + dir1); - final Path destin = new Path(fs.getUri().toString() + dir2); + final Path source = new Path(getFs().getUri().toString() + dir1); + final Path destin = new Path(getFs().getUri().toString() + dir2); // creates destin - fs.mkdirs(destin); + getFs().mkdirs(destin); LOG.info("Created destin dir: {}", destin); LOG.info("Rename op-> source:{} to destin:{}}", source, destin); try { - fs.rename(source, destin); + getFs().rename(source, destin); Assert.fail("Should throw exception : Source doesn't exist!"); } catch (OMException ome) { // expected @@ -316,22 +313,22 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { @Test public void testRenameDirToItsOwnSubDir() throws Exception { // Skip as this will run only in new layout - if (!enabledFileSystemPaths) { + if (!isEnabledFileSystemPaths()) { return; } final String root = "/root"; final String dir1 = root + "/dir1"; - final Path dir1Path = new Path(fs.getUri().toString() + dir1); + final Path dir1Path = new Path(getFs().getUri().toString() + dir1); // Add a sub-dir1 to the directory to be moved. final Path subDir1 = new Path(dir1Path, "sub_dir1"); - fs.mkdirs(subDir1); + getFs().mkdirs(subDir1); LOG.info("Created dir1 {}", subDir1); - final Path sourceRoot = new Path(fs.getUri().toString() + root); + final Path sourceRoot = new Path(getFs().getUri().toString() + root); LOG.info("Rename op-> source:{} to destin:{}", sourceRoot, subDir1); try { - fs.rename(sourceRoot, subDir1); + getFs().rename(sourceRoot, subDir1); Assert.fail("Should throw exception : Cannot rename a directory to" + " its own subdirectory"); } catch (OMException ome) { @@ -348,20 +345,21 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { @Test public void testRenameDestinationParentDoesntExist() throws Exception { // Skip as this will run only in new layout - if (!enabledFileSystemPaths) { + if (!isEnabledFileSystemPaths()) { return; } final String root = "/root_dir"; final String dir1 = root + "/dir1"; final String dir2 = dir1 + "/dir2"; - final Path dir2SourcePath = new Path(fs.getUri().toString() + dir2); - fs.mkdirs(dir2SourcePath); + final Path dir2SourcePath = new Path(getFs().getUri().toString() + dir2); + getFs().mkdirs(dir2SourcePath); // (a) parent of dst does not exist. /root_dir/b/c - final Path destinPath = new Path(fs.getUri().toString() + root + "/b/c"); + final Path destinPath = new Path(getFs().getUri().toString() + + root + "/b/c"); try { - fs.rename(dir2SourcePath, destinPath); + getFs().rename(dir2SourcePath, destinPath); Assert.fail("Should fail as parent of dst does not exist!"); } catch (OMException ome) { // expected @@ -369,12 +367,12 @@ public class TestOzoneFileSystemV1 extends TestOzoneFileSystem { } // (b) parent of dst is a file. /root_dir/file1/c - Path filePath = new Path(fs.getUri().toString() + root + "/file1"); - ContractTestUtils.touch(fs, filePath); + Path filePath = new Path(getFs().getUri().toString() + root + "/file1"); + ContractTestUtils.touch(getFs(), filePath); Path newDestinPath = new Path(filePath, "c"); try { - fs.rename(dir2SourcePath, newDestinPath); + getFs().rename(dir2SourcePath, newDestinPath); Assert.fail("Should fail as parent of dst is a file!"); } catch (OMException ome) { // expected diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java index 36146a2..5af6c63 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestReadRetries.java @@ -87,11 +87,11 @@ public class TestReadRetries { @Rule public ExpectedException thrown = ExpectedException.none(); - private static MiniOzoneCluster cluster = null; - private static OzoneClient ozClient = null; - private static ObjectStore store = null; - private static OzoneManager ozoneManager; - private static StorageContainerLocationProtocolClientSideTranslatorPB + private MiniOzoneCluster cluster = null; + private OzoneClient ozClient = null; + private ObjectStore store = null; + private OzoneManager ozoneManager; + private StorageContainerLocationProtocolClientSideTranslatorPB storageContainerLocationClient; private static final String SCM_ID = UUID.randomUUID().toString(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java index d09020e..b877e29 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/om/TestObjectStoreV1.java @@ -40,6 +40,7 @@ import org.apache.hadoop.ozone.om.exceptions.OMException; import org.apache.hadoop.ozone.om.helpers.OmDirectoryInfo; import org.apache.hadoop.ozone.om.helpers.OmKeyInfo; import org.apache.hadoop.ozone.om.request.TestOMRequestUtils; +import org.apache.hadoop.test.GenericTestUtils; import org.apache.hadoop.util.StringUtils; import org.junit.Assert; import org.junit.AfterClass; @@ -50,8 +51,10 @@ import org.junit.Test; import org.junit.rules.Timeout; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.HashMap; import java.util.UUID; +import java.util.concurrent.TimeoutException; import static org.apache.hadoop.hdds.client.ReplicationFactor.ONE; import static org.apache.hadoop.hdds.client.ReplicationType.STAND_ALONE; @@ -175,7 +178,8 @@ public class TestObjectStoreV1 { verifyKeyInOpenFileTable(openFileTable, clientID, file, dirPathC.getObjectID(), false); - ozoneOutputStream.write(data.getBytes(), 0, data.length()); + ozoneOutputStream.write(data.getBytes(StandardCharsets.UTF_8), 0, + data.length()); ozoneOutputStream.close(); Table<String, OmKeyInfo> fileTable = @@ -227,7 +231,8 @@ public class TestObjectStoreV1 { verifyKeyInOpenFileTable(openFileTable, clientID, fileName, dirPathC.getObjectID(), false); - ozoneOutputStream.write(data.getBytes(), 0, data.length()); + ozoneOutputStream.write(data.getBytes(StandardCharsets.UTF_8), 0, + data.length()); // open key try { @@ -356,22 +361,21 @@ public class TestObjectStoreV1 { } private void assertKeyRenamedEx(OzoneBucket bucket, String keyName) - throws Exception { - OMException oe = null; + throws Exception { try { bucket.getKey(keyName); - } catch (OMException e) { - oe = e; + fail("Should throw KeyNotFound as the key got renamed!"); + } catch (OMException ome) { + Assert.assertEquals(KEY_NOT_FOUND, ome.getResult()); } - Assert.assertEquals(KEY_NOT_FOUND, oe.getResult()); } private void createTestKey(OzoneBucket bucket, String keyName, String keyValue) throws IOException { OzoneOutputStream out = bucket.createKey(keyName, - keyValue.getBytes().length, STAND_ALONE, + keyValue.getBytes(StandardCharsets.UTF_8).length, STAND_ALONE, ONE, new HashMap<>()); - out.write(keyValue.getBytes()); + out.write(keyValue.getBytes(StandardCharsets.UTF_8)); out.close(); OzoneKey key = bucket.getKey(keyName); Assert.assertEquals(keyName, key.getName()); @@ -416,13 +420,24 @@ public class TestObjectStoreV1 { private void verifyKeyInOpenFileTable(Table<String, OmKeyInfo> openFileTable, long clientID, String fileName, long parentID, boolean isEmpty) - throws IOException { + throws IOException, TimeoutException, InterruptedException { String dbOpenFileKey = parentID + OM_KEY_PREFIX + fileName + OM_KEY_PREFIX + clientID; - OmKeyInfo omKeyInfo = openFileTable.get(dbOpenFileKey); + if (isEmpty) { - Assert.assertNull("Table is not empty!", omKeyInfo); + // wait for DB updates + GenericTestUtils.waitFor(() -> { + try { + OmKeyInfo omKeyInfo = openFileTable.get(dbOpenFileKey); + return omKeyInfo == null; + } catch (IOException e) { + Assert.fail("DB failure!"); + return false; + } + + }, 1000, 120000); } else { + OmKeyInfo omKeyInfo = openFileTable.get(dbOpenFileKey); Assert.assertNotNull("Table is empty!", omKeyInfo); // used startsWith because the key format is, // <parentID>/fileName/<clientID> and clientID is not visible. diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestV1.java index 454cfbb..d8c8108 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/file/TestOMDirectoryCreateRequestV1.java @@ -25,6 +25,7 @@ import org.apache.hadoop.hdds.conf.OzoneConfiguration; import org.apache.hadoop.hdds.protocol.proto.HddsProtos; import org.apache.hadoop.hdds.utils.db.cache.CacheKey; import org.apache.hadoop.hdds.utils.db.cache.CacheValue; +import org.apache.hadoop.ozone.OzoneConsts; import org.apache.hadoop.ozone.audit.AuditLogger; import org.apache.hadoop.ozone.audit.AuditMessage; import org.apache.hadoop.ozone.om.OMConfigKeys; @@ -369,8 +370,9 @@ public class TestOMDirectoryCreateRequestV1 { bucketName, keyName, HddsProtos.ReplicationType.RATIS, HddsProtos.ReplicationFactor.THREE, objID++); String ozoneFileName = parentID + "/" + dirs.get(dirs.size() - 1); + ++txnID; omMetadataManager.getKeyTable().addCacheEntry(new CacheKey<>(ozoneFileName), - new CacheValue<>(Optional.of(omKeyInfo), ++txnID)); + new CacheValue<>(Optional.of(omKeyInfo), txnID)); omMetadataManager.getKeyTable().put(ozoneFileName, omKeyInfo); OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, @@ -435,10 +437,12 @@ public class TestOMDirectoryCreateRequestV1 { // Add a key in second level. OmKeyInfo omKeyInfo = TestOMRequestUtils.createOmKeyInfo(volumeName, bucketName, keyName, HddsProtos.ReplicationType.RATIS, - HddsProtos.ReplicationFactor.THREE, objID++); + HddsProtos.ReplicationFactor.THREE, objID); + String ozoneKey = parentID + "/" + dirs.get(1); + ++txnID; omMetadataManager.getKeyTable().addCacheEntry(new CacheKey<>(ozoneKey), - new CacheValue<>(Optional.of(omKeyInfo), ++txnID)); + new CacheValue<>(Optional.of(omKeyInfo), txnID)); omMetadataManager.getKeyTable().put(ozoneKey, omKeyInfo); OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, @@ -517,10 +521,6 @@ public class TestOMDirectoryCreateRequestV1 { // Add volume and bucket entries to DB. TestOMRequestUtils.addVolumeAndBucketToDB(volumeName, bucketName, omMetadataManager); - String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); - OmBucketInfo omBucketInfo = - omMetadataManager.getBucketTable().get(bucketKey); - long bucketID = omBucketInfo.getObjectID(); OMRequest omRequest = createDirectoryRequest(volumeName, bucketName, OzoneFSUtils.addTrailingSlashIfNeeded(keyName)); @@ -587,12 +587,14 @@ public class TestOMDirectoryCreateRequestV1 { private String createDirKey(List<String> dirs, int depth) { String keyName = RandomStringUtils.randomAlphabetic(5); dirs.add(keyName); + StringBuffer buf = new StringBuffer(keyName); for (int i = 0; i < depth; i++) { String dirName = RandomStringUtils.randomAlphabetic(5); dirs.add(dirName); - keyName += "/" + dirName; + buf.append(OzoneConsts.OM_KEY_PREFIX); + buf.append(dirName); } - return keyName; + return buf.toString(); } private void verifyDirectoriesInDB(List<String> dirs, long bucketID) diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java index b65443d..e545dc7 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCreateRequestV1.java @@ -75,7 +75,9 @@ public class TestOMKeyCreateRequestV1 extends TestOMKeyCreateRequest { long parentID = checkIntermediatePaths(keyPath); // Check open key entry - String fileName = keyPath.getFileName().toString(); + Path keyPathFileName = keyPath.getFileName(); + Assert.assertNotNull("Failed to find fileName", keyPathFileName); + String fileName = keyPathFileName.toString(); String openKey = omMetadataManager.getOpenFileName(parentID, fileName, omRequest.getCreateKeyRequest().getClientID()); OmKeyInfo omKeyInfo = omMetadataManager.getOpenKeyTable().get(openKey); @@ -88,6 +90,7 @@ public class TestOMKeyCreateRequestV1 extends TestOMKeyCreateRequest { String bucketKey = omMetadataManager.getBucketKey(volumeName, bucketName); OmBucketInfo omBucketInfo = omMetadataManager.getBucketTable().get(bucketKey); + Assert.assertNotNull("Bucket not found!", omBucketInfo); long lastKnownParentId = omBucketInfo.getObjectID(); Iterator<Path> elements = keyPath.iterator(); diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java index dac2efe..5fa75ba 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/s3/multipart/TestS3InitiateMultipartUploadRequestV1.java @@ -133,10 +133,6 @@ public class TestS3InitiateMultipartUploadRequestV1 Assert.assertEquals(OzoneManagerProtocolProtos.Status.BUCKET_NOT_FOUND, omClientResponse.getOMResponse().getStatus()); - String multipartKey = omMetadataManager.getMultipartKey(volumeName, - bucketName, keyName, modifiedRequest.getInitiateMultiPartUploadRequest() - .getKeyArgs().getMultipartUploadID()); - Assert.assertTrue(omMetadataManager.getOpenKeyTable().isEmpty()); Assert.assertTrue(omMetadataManager.getMultipartInfoTable().isEmpty()); } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseV1.java index 0a1114a..fb06581 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMDirectoryCreateResponseV1.java @@ -59,9 +59,7 @@ public class TestOMDirectoryCreateResponseV1 { @Test public void testAddToDBBatch() throws Exception { - String volumeName = UUID.randomUUID().toString(); String keyName = UUID.randomUUID().toString(); - String bucketName = UUID.randomUUID().toString(); long parentID = 100; OmDirectoryInfo omDirInfo = diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java index e1549e1..a8a0c99 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/file/TestOMFileCreateResponseV1.java @@ -30,6 +30,8 @@ import org.apache.hadoop.util.Time; import org.jetbrains.annotations.NotNull; import org.junit.Assert; +import java.util.ArrayList; + /** * Tests MKeyCreateResponse layout version V1. */ @@ -59,8 +61,8 @@ public class TestOMFileCreateResponseV1 extends TestOMKeyCreateResponse { protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, OmBucketInfo bucketInfo, OMResponse response) { - return new OMFileCreateResponseV1(response, keyInfo, null, clientID, - bucketInfo); + return new OMFileCreateResponseV1(response, keyInfo, new ArrayList<>(), + clientID, bucketInfo); } @NotNull diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java index 4d50337..e2a223c 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCommitResponse.java @@ -34,8 +34,6 @@ import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos; @SuppressWarnings("visibilitymodifier") public class TestOMKeyCommitResponse extends TestOMKeyResponse { - protected OmBucketInfo omBucketInfo; - @Test public void testAddToDBBatch() throws Exception { diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java index 6299639..834aafa 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/key/TestOMKeyCreateResponseV1.java @@ -28,6 +28,8 @@ import org.apache.hadoop.util.Time; import org.jetbrains.annotations.NotNull; import org.junit.Assert; +import java.util.ArrayList; + /** * Tests OMKeyCreateResponseV1. */ @@ -67,7 +69,7 @@ public class TestOMKeyCreateResponseV1 extends TestOMKeyCreateResponse { protected OMKeyCreateResponse getOmKeyCreateResponse(OmKeyInfo keyInfo, OmBucketInfo bucketInfo, OMResponse response) { - return new OMKeyCreateResponseV1(response, keyInfo, null, clientID, - bucketInfo); + return new OMKeyCreateResponseV1(response, keyInfo, new ArrayList<>(), + clientID, bucketInfo); } } diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponseV1.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponseV1.java index 31f9e5a..6dd6aaf 100644 --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponseV1.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/s3/multipart/TestS3InitiateMultipartUploadResponseV1.java @@ -40,11 +40,6 @@ public class TestS3InitiateMultipartUploadResponseV1 String volumeName = UUID.randomUUID().toString(); String bucketName = UUID.randomUUID().toString(); String prefix = "a/b/c/d/"; - List<String> dirs = new ArrayList<String>(); - dirs.add("a"); - dirs.add("b"); - dirs.add("c"); - dirs.add("d"); String fileName = UUID.randomUUID().toString(); String keyName = prefix + fileName; --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
