This is an automated email from the ASF dual-hosted git repository. msingh pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/ozone.git
commit f83fe886e15b0aba87024db180643136d0adf340 Author: Sadanand Shenoy <[email protected]> AuthorDate: Tue Mar 16 11:54:51 2021 +0530 HDDS-4971. Define Ozone specific trash-interval and checkpoint interval (#2031) --- .../java/org/apache/hadoop/ozone/om/OMConfigKeys.java | 10 ++++++++++ .../org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java | 6 ++++-- .../apache/hadoop/ozone/TestOzoneConfigurationFields.java | 2 ++ .../java/org/apache/hadoop/ozone/om/OzoneManager.java | 7 ++++++- .../java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java | 15 +++++++++++++-- 5 files changed, 35 insertions(+), 5 deletions(-) diff --git a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java index 3ad4ab9..2eead63 100644 --- a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java +++ b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/OMConfigKeys.java @@ -234,4 +234,14 @@ public final class OMConfigKeys { false; public static final String OZONE_OM_HA_PREFIX = "ozone.om.ha"; + + public static final String OZONE_FS_TRASH_INTERVAL_KEY = + "ozone.fs.trash.interval"; + + public static final long OZONE_FS_TRASH_INTERVAL_DEFAULT = 0; + + public static final String OZONE_FS_TRASH_CHECKPOINT_INTERVAL_KEY = + "ozone.fs.trash.checkpoint.interval"; + + public static final long OZONE_FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT = 0; } diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java index 03a22e3..6f13b0d 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestOzoneFileSystem.java @@ -133,7 +133,8 @@ public class TestOzoneFileSystem { private void init() throws Exception { OzoneConfiguration conf = new OzoneConfiguration(); - conf.setInt(FS_TRASH_INTERVAL_KEY, 1); + conf.setInt(FS_TRASH_INTERVAL_KEY, 2); + conf.setInt(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, 1); conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled); conf.setBoolean(OZONE_ACL_ENABLED, true); conf.setBoolean(OMConfigKeys.OZONE_OM_ENABLE_FILESYSTEM_PATHS, @@ -801,7 +802,8 @@ public class TestOzoneFileSystem { Assert.assertTrue(trash.getConf().getClass( "fs.trash.classname", TrashPolicy.class). isAssignableFrom(TrashPolicyOzone.class)); - Assert.assertEquals(trash.getConf().getInt(FS_TRASH_INTERVAL_KEY, 0), 1); + Assert.assertEquals(1, trash.getConf(). + getInt(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, 0)); // Call moveToTrash. We can't call protected fs.rename() directly trash.moveToTrash(path); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java index c3ed747..5a9c796 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestOzoneConfigurationFields.java @@ -67,6 +67,8 @@ public class TestOzoneConfigurationFields extends TestConfigurationFieldsBase { HddsConfigKeys.HDDS_SECURITY_PROVIDER, HddsConfigKeys.HDDS_X509_CRL_NAME, // HDDS-2873 OMConfigKeys.OZONE_OM_NODES_KEY, + OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, + OMConfigKeys.OZONE_FS_TRASH_CHECKPOINT_INTERVAL_KEY, OzoneConfigKeys.OZONE_ACL_AUTHORIZER_CLASS_NATIVE, OzoneConfigKeys.OZONE_S3_AUTHINFO_MAX_LIFETIME_KEY, ReconServerConfigKeys.OZONE_RECON_SCM_DB_DIR, diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java index 0b4b8ea..fffcf80 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/OzoneManager.java @@ -1270,8 +1270,13 @@ public final class OzoneManager extends ServiceRuntimeInfoImpl * @throws IOException */ private void startTrashEmptier(Configuration conf) throws IOException { + long hadoopTrashInterval = + conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT); + // check whether user has configured ozone specific trash-interval + // if not fall back to hadoop configuration long trashInterval = - conf.getLong(FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT); + conf.getLong(OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, + hadoopTrashInterval); if (trashInterval == 0) { LOG.info("Trash Interval set to 0. Files deleted will not move to trash"); return; diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java index edb4a40..e751a98 100644 --- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java +++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/TrashPolicyOzone.java @@ -82,11 +82,22 @@ public class TrashPolicyOzone extends TrashPolicyDefault { public void initialize(Configuration conf, FileSystem fs) { this.fs = fs; this.configuration = conf; + float hadoopTrashInterval = conf.getFloat( + FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT); + // check whether user has configured ozone specific trash-interval + // if not fall back to hadoop configuration this.deletionInterval = (long)(conf.getFloat( - FS_TRASH_INTERVAL_KEY, FS_TRASH_INTERVAL_DEFAULT) + OMConfigKeys.OZONE_FS_TRASH_INTERVAL_KEY, hadoopTrashInterval) * MSECS_PER_MINUTE); + float hadoopCheckpointInterval = conf.getFloat( + FS_TRASH_CHECKPOINT_INTERVAL_KEY, + FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT); + // check whether user has configured ozone specific + // trash- checkpoint-interval + // if not fall back to hadoop configuration this.emptierInterval = (long)(conf.getFloat( - FS_TRASH_CHECKPOINT_INTERVAL_KEY, FS_TRASH_CHECKPOINT_INTERVAL_DEFAULT) + OMConfigKeys.OZONE_FS_TRASH_CHECKPOINT_INTERVAL_KEY, + hadoopCheckpointInterval) * MSECS_PER_MINUTE); if (deletionInterval < 0) { LOG.warn("Invalid value {} for deletion interval," --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
