anuragp010 commented on code in PR #12022:
URL: https://github.com/apache/ozone/pull/12022#discussion_r4582530796
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneStoragePolicy.java:
##########
@@ -188,6 +204,117 @@ public void testKeyWithoutAnyPolicyUsesClusterDefault()
throws Exception {
OzoneStoragePolicy.getDefaultPolicy().getCreationTier());
}
+ /**
+ * Creating a file with a storage policy works for every file bucket layout.
All
+ * layouts share one cluster, because the scenarios differ only in the bucket
+ * they use.
+ */
+ @Test
+ public void testCreateFileWithStoragePolicy() throws Exception {
+ startCluster(allTierTopology(RATIS_THREE.getRequiredNodes()));
+ OzoneManagerProtocol omClientSpy = spyOmClient();
+
+ for (BucketLayout bucketLayout : FILE_BUCKET_LAYOUTS) {
+ assertFileStoragePolicyIsHonoured(bucketLayout);
+ assertFileInheritsBucketPolicyOrClusterDefault(bucketLayout);
+ assertMultiBlockFileKeepsPolicy(bucketLayout, omClientSpy);
+ assertOverwriteUpdatesPolicyAndPlacement(bucketLayout);
+ }
+ }
+
+ /**
+ * A file created with an explicit storage policy records that policy and
lands
+ * on the policy's creation tier, in SCM and on the datanodes.
+ */
+ private void assertFileStoragePolicyIsHonoured(BucketLayout bucketLayout)
+ throws IOException {
+ for (OzoneStoragePolicy storagePolicy : OzoneStoragePolicy.values()) {
+ OzoneBucket bucket = createBucket(null, bucketLayout);
+ OmKeyInfo keyInfo = createFileAndLookup(bucket, storagePolicy);
+
+ assertKeyInfo(keyInfo, storagePolicy);
+ assertScmContainerAndPipeline(keyInfo, storagePolicy.getCreationTier());
+ assertDatanodeContainerAndBlock(keyInfo,
storagePolicy.getCreationTier());
+ }
+ }
+
+ /**
+ * A file created without a storage policy inherits the bucket's policy, and
+ * without a bucket policy either it gets the cluster default.
+ */
+ private void assertFileInheritsBucketPolicyOrClusterDefault(BucketLayout
bucketLayout)
+ throws IOException {
+ OzoneStoragePolicy bucketPolicy = OzoneStoragePolicy.COLD;
+ OmKeyInfo inherited = createFileAndLookup(createBucket(bucketPolicy,
bucketLayout), null);
+ assertKeyInfo(inherited, bucketPolicy);
+ assertScmContainerAndPipeline(inherited, bucketPolicy.getCreationTier());
+ assertDatanodeContainerAndBlock(inherited, bucketPolicy.getCreationTier());
+
+ OzoneStoragePolicy defaultPolicy = OzoneStoragePolicy.getDefaultPolicy();
+ OmKeyInfo defaulted = createFileAndLookup(createBucket(null,
bucketLayout), null);
+ assertKeyInfo(defaulted, defaultPolicy);
+ assertScmContainerAndPipeline(defaulted, defaultPolicy.getCreationTier());
+ assertDatanodeContainerAndBlock(defaulted,
defaultPolicy.getCreationTier());
+ }
+
+ /**
+ * Tests whether a file that needs more blocks than OM pre-allocates keeps
its policy on
+ * every block, and the client sends that policy with every later
allocateBlock call.
+ *
+ * <p>The file asks for COLD while its bucket is WARM, so a policy that is
lost
+ * on the way makes the later blocks land on DISK, or makes the call carry
none.
+ * Creating the file with size 0 makes OM pre-allocate one block, so the
others
+ * come from allocateBlock calls.
+ */
+ private void assertMultiBlockFileKeepsPolicy(BucketLayout bucketLayout,
OzoneManagerProtocol omClientSpy)
+ throws IOException {
+ OzoneBucket bucket = createBucket(OzoneStoragePolicy.WARM, bucketLayout);
+ String keyName = UUID.randomUUID().toString();
+ clearInvocations(omClientSpy);
+
+ writeFile(bucket, keyName, multiBlockData(), 0, false,
OzoneStoragePolicy.COLD);
+
+ OmKeyInfo keyInfo = lookupKey(bucket, keyName);
+ assertThat(keyInfo.getStoragePolicy()).as("layout %s",
bucketLayout).isEqualTo(OzoneStoragePolicy.COLD);
+ List<OmKeyLocationInfo> blocks =
keyInfo.getLatestVersionLocations().getBlocksLatestVersionOnly();
+ assertThat(blocks.size())
+ .withFailMessage("Expected more than one block so a later allocation
is exercised, "
+ + "but the file in a %s bucket has %d", bucketLayout,
blocks.size())
+ .isGreaterThan(1);
+
+ ArgumentCaptor<OmKeyArgs> allocateBlockArgs =
ArgumentCaptor.forClass(OmKeyArgs.class);
+ verify(omClientSpy,
atLeastOnce()).allocateBlock(allocateBlockArgs.capture(), anyLong(), any());
Review Comment:
@smengcl alright, yes. Both make sense and I have updated
`testCreateFileWithStoragePolicy` to use a single helper which verifies
multi-block writes along with overwrite of the file.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]