This is an automated email from the ASF dual-hosted git repository. avijayan pushed a commit to branch HDDS-3698-upgrade in repository https://gitbox.apache.org/repos/asf/ozone.git
commit bf7624bc2a4e0a4ad9901126b1baf924551cbedd Merge: 4384789 6cc4a43 Author: Aravindan Vijayan <[email protected]> AuthorDate: Mon Nov 23 16:08:37 2020 -0800 Merge remote-tracking branch 'upstream/master' into HDDS-3698-upgrade .../apache/hadoop/hdds/scm/OzoneClientConfig.java | 5 +- .../apache/hadoop/hdds/scm/XceiverClientRatis.java | 12 +- .../org/apache/hadoop/hdds/ratis/RatisHelper.java | 12 +- .../CreatePipelineCommandHandler.java | 2 +- .../server/ratis/ContainerStateMachine.java | 4 +- .../container/common/utils/ContainerCache.java | 2 +- .../container/keyvalue/helpers/BlockUtils.java | 12 +- .../keyvalue/helpers/KeyValueContainerUtil.java | 7 +- .../container/metadata/AbstractDatanodeStore.java | 6 +- .../metadata/DatanodeStoreSchemaOneImpl.java | 7 +- .../metadata/DatanodeStoreSchemaTwoImpl.java | 7 +- .../ozone/container/common/TestContainerCache.java | 2 +- .../TestCreatePipelineCommandHandler.java | 13 +- hadoop-hdds/docs/content/feature/HA.zh.md | 6 +- .../hdds/conf/DatanodeRatisServerConfig.java | 2 +- .../hadoop/hdds/utils/db/DBStoreBuilder.java | 9 +- .../org/apache/hadoop/hdds/utils/db/RDBStore.java | 14 +- .../hdds/scm/pipeline/RatisPipelineUtils.java | 4 +- .../ozone/client/io/BlockOutputStreamEntry.java | 4 - .../apache/hadoop/ozone/client/rpc/RpcClient.java | 2 +- .../org/apache/hadoop/ozone/RatisTestHelper.java | 3 +- .../ozone/client/rpc/TestBlockOutputStream.java | 12 +- .../rpc/TestBlockOutputStreamWithFailures.java | 11 +- .../rpc/TestOzoneClientRetriesOnException.java | 13 +- .../ozone/client/rpc/TestOzoneRpcClient.java | 2 - .../client/rpc/TestOzoneRpcClientWithRatis.java | 2 - .../ozone/om/ratis/OzoneManagerRatisServer.java | 15 +- .../ozone/om/ratis/OzoneManagerStateMachine.java | 2 +- .../om/response/file/OMFileCreateResponse.java | 5 + .../ozone/om/response/key/OMKeyCreateResponse.java | 3 +- .../om/ratis/TestOzoneManagerStateMachine.java | 10 +- .../ozone/om/response/TestCleanupTableInfo.java | 343 +++++++++++++++++++-- .../hadoop/ozone/s3/endpoint/BucketEndpoint.java | 8 +- .../ozone/s3/endpoint/ListObjectResponse.java | 11 + .../freon/FollowerAppendLogEntryGenerator.java | 11 +- .../ozone/freon/LeaderAppendLogEntryGenerator.java | 20 +- pom.xml | 2 +- 37 files changed, 472 insertions(+), 133 deletions(-) diff --cc hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java index f813000,a59a4e1..90c08cb --- a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java +++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/response/TestCleanupTableInfo.java @@@ -55,41 -138,234 +138,239 @@@ public class TestCleanupTableInfo } @Test - public void checkAllWriteResponseHasCleanupTableAnnotation() { - getResponseClasses().forEach(aClass -> { - Assert.assertTrue( - aClass + "does not have annotation of CleanupTableInfo", - aClass.isAnnotationPresent(CleanupTableInfo.class)); - }); - } + public void checkAnnotationAndTableName() { + OMMetadataManager omMetadataManager = om.getMetadataManager(); - @Test - public void checkWriteResponseIsAnnotatedWithKnownTableNames() - throws Exception { - OMMetadataManager omMetadataManager = new OmMetadataManagerImpl(conf); Set<String> tables = omMetadataManager.listTableNames(); - - getResponseClasses().forEach(aClass -> { - + Set<Class<? extends OMClientResponse>> subTypes = responseClasses(); + subTypes.forEach(aClass -> { + Assert.assertTrue(aClass + "does not have annotation of" + + " CleanupTableInfo", + aClass.isAnnotationPresent(CleanupTableInfo.class)); - String[] cleanupTables = - aClass.getAnnotation(CleanupTableInfo.class).cleanupTables(); - Assert.assertTrue(cleanupTables.length >=1); - for (String tableName : cleanupTables) { - Assert.assertTrue(tables.contains(tableName)); + CleanupTableInfo annotation = + aClass.getAnnotation(CleanupTableInfo.class); + String[] cleanupTables = annotation.cleanupTables(); + boolean cleanupAll = annotation.cleanupAll(); - + if (cleanupTables.length >= 1) { - assertTrue( ++ Assert.assertTrue( + Arrays.stream(cleanupTables).allMatch(tables::contains) + ); + } else { - assertTrue(cleanupAll); ++ Assert.assertTrue(cleanupAll); } - }); } - private Set<Class<? extends OMClientResponse>> getResponseClasses() { - Reflections reflections = - new Reflections("org.apache.hadoop.ozone.om.response"); ++ + private Set<Class<? extends OMClientResponse>> responseClasses() { + Reflections reflections = new Reflections(OM_RESPONSE_PACKAGE); return reflections.getSubTypesOf(OMClientResponse.class); } - } + + @Test + public void testFileCreateRequestSetsAllTouchedTableCachesForEviction() { + OMFileCreateRequest request = anOMFileCreateRequest(); + Map<String, Integer> cacheItemCount = recordCacheItemCounts(); + + request.validateAndUpdateCache(om, 1, dbh); + + assertCacheItemCounts(cacheItemCount, OMFileCreateResponse.class); + verify(omMetrics, times(1)).incNumCreateFile(); + } + + @Test + public void testKeyCreateRequestSetsAllTouchedTableCachesForEviction() { + OMKeyCreateRequest request = anOMKeyCreateRequest(); + when(om.getEnableFileSystemPaths()).thenReturn(true); + + Map<String, Integer> cacheItemCount = recordCacheItemCounts(); + + request.validateAndUpdateCache(om, 1, dbh); + + assertCacheItemCounts(cacheItemCount, OMKeyCreateResponse.class); + verify(omMetrics, times(1)).incNumKeyAllocates(); + } + - - + private Map<String, Integer> recordCacheItemCounts() { + Map<String, Integer> cacheItemCount = new HashMap<>(); + for (String tableName : om.getMetadataManager().listTableNames()) { + cacheItemCount.put( + tableName, + Iterators.size( + om.getMetadataManager().getTable(tableName).cacheIterator() + ) + ); + } + return cacheItemCount; + } + - private void assertCacheItemCounts(Map<String, Integer> cacheItemCount, ++ private void assertCacheItemCounts( ++ Map<String, Integer> cacheItemCount, + Class<? extends OMClientResponse> responseClass + ) { + CleanupTableInfo ann = responseClass.getAnnotation(CleanupTableInfo.class); + List<String> cleanup = Arrays.asList(ann.cleanupTables()); + for (String tableName : om.getMetadataManager().listTableNames()) { + if (!cleanup.contains(tableName)) { + assertEquals( + "Cache item count of table " +tableName, + cacheItemCount.get(tableName).intValue(), + Iterators.size( + om.getMetadataManager().getTable(tableName).cacheIterator() + ) + ); + } + } + } + + /** + * Adds the volume info to the volumeTable in the MetadataManager, and also + * add the value to the table's cache. + * + * @param volumeArgs the OMVolumeArgs object specifying the volume propertes + * @throws IOException if an IO issue occurs while wrtiing to RocksDB + */ + private void addVolumeToMetaTable(OmVolumeArgs volumeArgs) + throws IOException { + String volumeKey = om.getMetadataManager().getVolumeKey(TEST_VOLUME_NAME); + om.getMetadataManager().getVolumeTable().put(volumeKey, volumeArgs); + om.getMetadataManager().getVolumeTable().addCacheEntry( + new CacheKey<>(volumeKey), + new CacheValue<>(Optional.of(volumeArgs), 2) + ); + } + + /** + * Adds the bucket info to the bucketTable in the MetadataManager, and also + * adds the value to the table's cache. + * + * @param bucketInfo the OMBucketInfo object specifying the bucket properties + * @throws IOException if an IO issue occurs while writing to RocksDB + */ + private void addBucketToMetaTable(OmBucketInfo bucketInfo) + throws IOException { + String bucketKey = om.getMetadataManager() + .getBucketKey(bucketInfo.getVolumeName(), bucketInfo.getBucketName()); + om.getMetadataManager().getBucketTable().put(bucketKey, bucketInfo); + om.getMetadataManager().getBucketTable().addCacheEntry( + new CacheKey<>(bucketKey), + new CacheValue<>(Optional.of(bucketInfo), 1) + ); + } + + /** + * Creates a spy object over an instantiated OMMetadataManager, giving the + * possibility to redefine behaviour. In the current implementation + * there isn't any behaviour which is redefined. + * + * @return the OMMetadataManager spy instance created. + * @throws IOException if I/O error occurs in setting up data store for the + * metadata manager. + */ + private OMMetadataManager createOMMetadataManagerSpy() throws IOException { + OzoneConfiguration conf = new OzoneConfiguration(); + File newFolder = folder.newFolder(); + if (!newFolder.exists()) { + Assert.assertTrue(newFolder.mkdirs()); + } + ServerUtils.setOzoneMetaDirPath(conf, newFolder.toString()); + return spy(new OmMetadataManagerImpl(conf)); + } + + private OMFileCreateRequest anOMFileCreateRequest() { + OMRequest protoRequest = mock(OMRequest.class); + when(protoRequest.getCreateFileRequest()).thenReturn(aCreateFileRequest()); + when(protoRequest.getCmdType()).thenReturn(Type.CreateFile); + when(protoRequest.getTraceID()).thenReturn(""); + return new OMFileCreateRequest(protoRequest); + } + + private OMKeyCreateRequest anOMKeyCreateRequest(){ + OMRequest protoRequest = mock(OMRequest.class); + when(protoRequest.getCreateKeyRequest()).thenReturn(aKeyCreateRequest()); + when(protoRequest.getCmdType()).thenReturn(Type.CreateKey); + when(protoRequest.getTraceID()).thenReturn(""); + return new OMKeyCreateRequest(protoRequest); + } + + private OmBucketInfo aBucketInfo() { + return OmBucketInfo.newBuilder() + .setVolumeName(TEST_VOLUME_NAME) + .setBucketName(TEST_BUCKET_NAME) + .setAcls(Collections.emptyList()) + .setIsVersionEnabled(false) + .setStorageType(StorageType.DEFAULT) + .build(); + } + + private OmVolumeArgs aVolumeArgs() { + return OmVolumeArgs.newBuilder() + .setAdminName("admin") + .setOwnerName("owner") + .setVolume(TEST_VOLUME_NAME) + .build(); + } + + private CreateFileRequest aCreateFileRequest() { + return CreateFileRequest.newBuilder() + .setKeyArgs(aKeyArgs()) + .setIsRecursive(true) + .setIsOverwrite(false) + .setClientID(1L) + .build(); + } + + private CreateKeyRequest aKeyCreateRequest() { + return CreateKeyRequest.newBuilder() + .setKeyArgs(aKeyArgs()) + .setClientID(1L) + .build(); + } + + private KeyArgs aKeyArgs() { + return KeyArgs.newBuilder() + .setVolumeName(TEST_VOLUME_NAME) + .setBucketName(TEST_BUCKET_NAME) + .setKeyName(TEST_KEY) + .setDataSize(512L) + .addKeyLocations(aKeyLocation(TEST_BLOCK_ID)) + .addKeyLocations(aKeyLocation(TEST_BLOCK_ID)) + .addKeyLocations(aKeyLocation(TEST_BLOCK_ID)) + .build(); + } + + private KeyLocation aKeyLocation( + HddsProtos.BlockID blockID) { + return KeyLocation.newBuilder() + .setBlockID(blockID) + .setOffset(0) + .setLength(512) + .setCreateVersion(0) + .setPipeline(aPipeline()) + .build(); + } + + private Pipeline aPipeline() { + return Pipeline.newBuilder() + .setId(aPipelineID()) + .addMembers(aDatanodeDetailsProto("192.168.1.1", "host1")) + .addMembers(aDatanodeDetailsProto("192.168.1.2", "host2")) + .addMembers(aDatanodeDetailsProto("192.168.1.3", "host3")) + .build(); + } + + private DatanodeDetailsProto aDatanodeDetailsProto(String s, - String host1) { ++ String host1) { + return DatanodeDetailsProto.newBuilder() + .setUuid(UUID.randomUUID().toString()) + .setIpAddress(s) + .setHostName(host1) + .build(); + } + + private HddsProtos.PipelineID aPipelineID() { + return HddsProtos.PipelineID.newBuilder() + .setId(UUID.randomUUID().toString()) + .build(); + } -} ++} --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
