anuragp010 commented on code in PR #12022:
URL: https://github.com/apache/ozone/pull/12022#discussion_r4585921714
##########
hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestOzoneStoragePolicy.java:
##########
@@ -188,6 +191,88 @@ public void testKeyWithoutAnyPolicyUsesClusterDefault()
throws Exception {
OzoneStoragePolicy.getDefaultPolicy().getCreationTier());
}
+ /**
+ * Creating a file with a storage policy works for every file bucket layout.
All
+ * layouts share one cluster, because the scenarios differ only in the bucket
+ * they use.
+ */
+ @Test
+ public void testCreateFileWithStoragePolicy() throws Exception {
+ startCluster(allTierTopology(RATIS_THREE.getRequiredNodes()));
+
+ for (BucketLayout bucketLayout : FILE_BUCKET_LAYOUTS) {
+ assertFileStoragePolicyIsHonoured(bucketLayout);
+ assertFileInheritsBucketPolicyOrClusterDefault(bucketLayout);
+ assertMultiBlockFileAndOverwriteHonourPolicy(bucketLayout);
+ }
+ }
+
+ /**
+ * A file created with an explicit storage policy records that policy and
lands
+ * on the policy's creation tier, in SCM and on the datanodes.
+ */
+ private void assertFileStoragePolicyIsHonoured(BucketLayout bucketLayout)
+ throws IOException {
+ for (OzoneStoragePolicy storagePolicy : OzoneStoragePolicy.values()) {
+ OzoneBucket bucket = createBucket(null, bucketLayout);
+ OmKeyInfo keyInfo = createFileAndLookup(bucket, storagePolicy);
+
+ assertKeyInfo(keyInfo, storagePolicy);
+ assertScmContainerAndPipeline(keyInfo, storagePolicy.getCreationTier());
+ assertDatanodeContainerAndBlock(keyInfo,
storagePolicy.getCreationTier());
+ }
+ }
+
+ /**
+ * A file created without a storage policy inherits the bucket's policy, and
+ * without a bucket policy either it gets the cluster default.
+ */
+ private void assertFileInheritsBucketPolicyOrClusterDefault(BucketLayout
bucketLayout)
+ throws IOException {
+ OzoneStoragePolicy bucketPolicy = OzoneStoragePolicy.COLD;
+ OmKeyInfo inherited = createFileAndLookup(createBucket(bucketPolicy,
bucketLayout), null);
+ assertKeyInfo(inherited, bucketPolicy);
+ assertScmContainerAndPipeline(inherited, bucketPolicy.getCreationTier());
+ assertDatanodeContainerAndBlock(inherited, bucketPolicy.getCreationTier());
+
+ OzoneStoragePolicy defaultPolicy = OzoneStoragePolicy.getDefaultPolicy();
+ OmKeyInfo defaulted = createFileAndLookup(createBucket(null,
bucketLayout), null);
+ assertKeyInfo(defaulted, defaultPolicy);
+ assertScmContainerAndPipeline(defaulted, defaultPolicy.getCreationTier());
+ assertDatanodeContainerAndBlock(defaulted,
defaultPolicy.getCreationTier());
+ }
+
+ /**
+ * Tests two scenarios on a file that spans several blocks.
+ *
+ * <p>Multi-block write: Tests whether a file that needs more blocks than OM
pre-allocates keeps its policy on
+ * every block. Creating the file with size 0 makes OM pre-allocate one
block, so the other blocks come from
+ * allocateBlock calls. Every block, not just the first, must sit on the
COLD tier.
+ *
+ * <p>Overwrite: Overwriting a file with a different storage policy updates
the recorded
+ * policy and moves the new blocks to the new tier.
+ */
+ private void assertMultiBlockFileAndOverwriteHonourPolicy(BucketLayout
bucketLayout)
+ throws IOException {
+ OzoneBucket bucket = createBucket(OzoneStoragePolicy.WARM, bucketLayout);
+ String keyName = UUID.randomUUID().toString();
+
+ writeFile(bucket, keyName, multiBlockData(), 0, false,
OzoneStoragePolicy.COLD);
+ OmKeyInfo original = lookupKey(bucket, keyName);
+ assertThat(original.getStoragePolicy()).as("layout %s",
bucketLayout).isEqualTo(OzoneStoragePolicy.COLD);
+ List<OmKeyLocationInfo> blocks =
original.getLatestVersionLocations().getBlocksLatestVersionOnly();
+ assertThat(blocks.size())
+ .withFailMessage("Expected more than one block so a later allocation
is exercised, "
+ + "but the file in a %s bucket has %d", bucketLayout,
blocks.size())
+ .isGreaterThan(1);
+ assertAllBlocksOnTier(original, OzoneStoragePolicy.COLD.getCreationTier());
Review Comment:
Yes we can. I have added `assertAllBlocksOnDatanodes` which verifies if all
the blocks of a file are on the correct volume.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]