This is an automated email from the ASF dual-hosted git repository.
weichiu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new b23981cbda HDDS-11342. [hsync] Add a config as HBase-related features
master switch (#7126)
b23981cbda is described below
commit b23981cbdaf27134d9a622548977f10256e41d44
Author: Siyao Meng <[email protected]>
AuthorDate: Thu Sep 5 09:34:16 2024 -0700
HDDS-11342. [hsync] Add a config as HBase-related features master switch
(#7126)
---
.../apache/hadoop/hdds/scm/OzoneClientConfig.java | 73 +++++++++++++++++++---
.../hadoop/hdds/scm/storage/BlockOutputStream.java | 48 ++++++++++++--
.../hadoop/hdds/scm/TestOzoneClientConfig.java | 40 ++++++++++++
.../org/apache/hadoop/ozone/OzoneConfigKeys.java | 8 +++
.../common/src/main/resources/ozone-default.xml | 19 +++++-
.../ozone/client/io/KeyOutputStreamSemaphore.java | 2 +-
.../apache/hadoop/ozone/client/rpc/RpcClient.java | 9 +--
.../TestBlockOutputStreamIncrementalPutBlock.java | 2 +
.../hadoop/ozone/om/helpers/OzoneFSUtils.java | 31 +++++++++
.../hadoop/ozone/om/helpers/TestOzoneFsUtils.java | 30 +++++++++
.../fs/ozone/AbstractOzoneFileSystemTest.java | 2 +
.../ozone/AbstractRootedOzoneFileSystemTest.java | 2 +
.../java/org/apache/hadoop/fs/ozone/TestHSync.java | 2 +
.../apache/hadoop/fs/ozone/TestHSyncUpgrade.java | 2 +
.../apache/hadoop/fs/ozone/TestLeaseRecovery.java | 2 +
.../ozone/contract/AbstractOzoneContractTest.java | 3 +
.../client/rpc/AbstractTestECKeyOutputStream.java | 2 +
.../ozone/client/rpc/TestBlockOutputStream.java | 4 ++
.../ozone/client/rpc/TestSecureOzoneRpcClient.java | 2 +
.../hadoop/ozone/debug/TestLeaseRecoverer.java | 2 +
.../hadoop/ozone/shell/TestOzoneShellHA.java | 5 ++
.../ozone/shell/TestOzoneShellHAWithFSO.java | 2 +
.../ozone/om/request/key/OMKeyCommitRequest.java | 5 +-
.../om/request/key/TestOMKeyCommitRequest.java | 20 +++++-
.../ozone/om/request/key/TestOMKeyRequest.java | 1 +
.../om/service/TestOpenKeyCleanupService.java | 2 +
.../hadoop/fs/ozone/BasicOzoneFileSystem.java | 5 +-
.../fs/ozone/BasicRootedOzoneFileSystem.java | 5 +-
28 files changed, 295 insertions(+), 35 deletions(-)
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
index 873f6f6734..5426bbc498 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/OzoneClientConfig.java
@@ -247,28 +247,49 @@ public class OzoneClientConfig {
tags = ConfigTag.CLIENT)
private String fsDefaultBucketLayout = "FILE_SYSTEM_OPTIMIZED";
+ // ozone.client.hbase.enhancements.allowed
+ @Config(key = "hbase.enhancements.allowed",
+ defaultValue = "false",
+ description = "When set to false, client-side HBase enhancement-related
Ozone (experimental) features " +
+ "are disabled (not allowed to be enabled) regardless of whether
those configs are set.\n" +
+ "\n" +
+ "Here is the list of configs and values overridden when this config
is set to false:\n" +
+ "1. ozone.fs.hsync.enabled = false\n" +
+ "2. ozone.client.incremental.chunk.list = false\n" +
+ "3. ozone.client.stream.putblock.piggybacking = false\n" +
+ "4. ozone.client.key.write.concurrency = 1\n" +
+ "\n" +
+ "A warning message will be printed if any of the above configs are
overridden by this.",
+ tags = ConfigTag.CLIENT)
+ private boolean hbaseEnhancementsAllowed = false;
+
+ // ozone.client.incremental.chunk.list
@Config(key = "incremental.chunk.list",
- defaultValue = "true",
+ defaultValue = "false",
type = ConfigType.BOOLEAN,
description = "Client PutBlock request can choose incremental chunk " +
"list rather than full chunk list to optimize performance. " +
- "Critical to HBase. EC does not support this feature.",
+ "Critical to HBase. EC does not support this feature. " +
+ "Can be enabled only when ozone.client.hbase.enhancements.allowed =
true",
tags = ConfigTag.CLIENT)
- private boolean incrementalChunkList = true;
+ private boolean incrementalChunkList = false;
+ // ozone.client.stream.putblock.piggybacking
@Config(key = "stream.putblock.piggybacking",
- defaultValue = "true",
+ defaultValue = "false",
type = ConfigType.BOOLEAN,
- description = "Allow PutBlock to be piggybacked in WriteChunk " +
- "requests if the chunk is small.",
+ description = "Allow PutBlock to be piggybacked in WriteChunk
requests if the chunk is small. " +
+ "Can be enabled only when
ozone.client.hbase.enhancements.allowed = true",
tags = ConfigTag.CLIENT)
- private boolean enablePutblockPiggybacking = true;
+ private boolean enablePutblockPiggybacking = false;
+ // ozone.client.key.write.concurrency
@Config(key = "key.write.concurrency",
defaultValue = "1",
description = "Maximum concurrent writes allowed on each key. " +
"Defaults to 1 which matches the behavior before HDDS-9844. " +
- "For unlimited write concurrency, set this to -1 or any negative
integer value.",
+ "For unlimited write concurrency, set this to -1 or any negative
integer value. " +
+ "Any value other than 1 is effective only when
ozone.client.hbase.enhancements.allowed = true",
tags = ConfigTag.CLIENT)
private int maxConcurrentWritePerKey = 1;
@@ -298,6 +319,34 @@ public class OzoneClientConfig {
OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE;
}
+ // Verify client configs related to HBase enhancements
+ // Enforce check on ozone.client.hbase.enhancements.allowed
+ if (!hbaseEnhancementsAllowed) {
+ // ozone.client.hbase.enhancements.allowed = false
+ if (incrementalChunkList) {
+ LOG.warn("Ignoring ozone.client.incremental.chunk.list = true " +
+ "because HBase enhancements are disallowed. " +
+ "To enable it, set ozone.client.hbase.enhancements.allowed =
true.");
+ incrementalChunkList = false;
+ LOG.debug("Final ozone.client.incremental.chunk.list = {}",
incrementalChunkList);
+ }
+ if (enablePutblockPiggybacking) {
+ LOG.warn("Ignoring ozone.client.stream.putblock.piggybacking = true " +
+ "because HBase enhancements are disallowed. " +
+ "To enable it, set ozone.client.hbase.enhancements.allowed =
true.");
+ enablePutblockPiggybacking = false;
+ LOG.debug("Final ozone.client.stream.putblock.piggybacking = {}",
enablePutblockPiggybacking);
+ }
+ if (maxConcurrentWritePerKey != 1) {
+ LOG.warn("Ignoring ozone.client.key.write.concurrency = {} " +
+ "because HBase enhancements are disallowed. " +
+ "To enable it, set ozone.client.hbase.enhancements.allowed =
true.",
+ maxConcurrentWritePerKey);
+ maxConcurrentWritePerKey = 1;
+ LOG.debug("Final ozone.client.key.write.concurrency = {}",
maxConcurrentWritePerKey);
+ }
+ // Note: ozone.fs.hsync.enabled is enforced by
OzoneFSUtils#canEnableHsync, not here
+ }
}
public long getStreamBufferFlushSize() {
@@ -486,6 +535,14 @@ public class OzoneClientConfig {
this.datastreamPipelineMode = datastreamPipelineMode;
}
+ public void setHBaseEnhancementsAllowed(boolean isHBaseEnhancementsEnabled) {
+ this.hbaseEnhancementsAllowed = isHBaseEnhancementsEnabled;
+ }
+
+ public boolean getHBaseEnhancementsAllowed() {
+ return this.hbaseEnhancementsAllowed;
+ }
+
public void setIncrementalChunkList(boolean enable) {
this.incrementalChunkList = enable;
}
diff --git
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
index ca3e4e5374..59795dd0f0 100644
---
a/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
+++
b/hadoop-hdds/client/src/main/java/org/apache/hadoop/hdds/scm/storage/BlockOutputStream.java
@@ -198,9 +198,7 @@ public class BlockOutputStream extends OutputStream {
blkIDBuilder.build()).addMetadata(keyValue);
this.pipeline = pipeline;
// tell DataNode I will send incremental chunk list
- // EC does not support incremental chunk list.
- this.supportIncrementalChunkList = config.getIncrementalChunkList() &&
- this instanceof RatisBlockOutputStream &&
allDataNodesSupportPiggybacking();
+ this.supportIncrementalChunkList = canEnableIncrementalChunkList();
LOG.debug("incrementalChunkList is {}", supportIncrementalChunkList);
if (supportIncrementalChunkList) {
this.containerBlockData.addMetadata(INCREMENTAL_CHUNK_LIST_KV);
@@ -237,11 +235,51 @@ public class BlockOutputStream extends OutputStream {
config.getBytesPerChecksum());
this.clientMetrics = clientMetrics;
this.streamBufferArgs = streamBufferArgs;
- this.allowPutBlockPiggybacking = config.getEnablePutblockPiggybacking() &&
- allDataNodesSupportPiggybacking();
+ this.allowPutBlockPiggybacking = canEnablePutblockPiggybacking();
LOG.debug("PutBlock piggybacking is {}", allowPutBlockPiggybacking);
}
+ /**
+ * Helper method to check if incremental chunk list can be enabled.
+ * Prints debug messages if it cannot be enabled.
+ */
+ private boolean canEnableIncrementalChunkList() {
+ boolean confEnableIncrementalChunkList = config.getIncrementalChunkList();
+ if (!confEnableIncrementalChunkList) {
+ return false;
+ }
+
+ if (!(this instanceof RatisBlockOutputStream)) {
+ // Note: EC does not support incremental chunk list
+ LOG.debug("Unable to enable incrementalChunkList because
BlockOutputStream is not a RatisBlockOutputStream");
+ return false;
+ }
+ if (!allDataNodesSupportPiggybacking()) {
+ // Not all datanodes support piggybacking and incremental chunk list.
+ LOG.debug("Unable to enable incrementalChunkList because not all
datanodes support piggybacking");
+ return false;
+ }
+ return confEnableIncrementalChunkList;
+ }
+
+ /**
+ * Helper method to check if PutBlock piggybacking can be enabled.
+ * Prints debug message if it cannot be enabled.
+ */
+ private boolean canEnablePutblockPiggybacking() {
+ boolean confEnablePutblockPiggybacking =
config.getEnablePutblockPiggybacking();
+ if (!confEnablePutblockPiggybacking) {
+ return false;
+ }
+
+ if (!allDataNodesSupportPiggybacking()) {
+ // Not all datanodes support piggybacking and incremental chunk list.
+ LOG.debug("Unable to enable PutBlock piggybacking because not all
datanodes support piggybacking");
+ return false;
+ }
+ return confEnablePutblockPiggybacking;
+ }
+
private boolean allDataNodesSupportPiggybacking() {
// return true only if all DataNodes in the pipeline are on a version
// that supports PutBlock piggybacking.
diff --git
a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java
b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java
index 0dd29cb50a..920d1e19e1 100644
---
a/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java
+++
b/hadoop-hdds/client/src/test/java/org/apache/hadoop/hdds/scm/TestOzoneClientConfig.java
@@ -22,6 +22,8 @@ import org.junit.jupiter.api.Test;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE;
import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertTrue;
class TestOzoneClientConfig {
@@ -36,4 +38,42 @@ class TestOzoneClientConfig {
assertEquals(OZONE_CLIENT_BYTES_PER_CHECKSUM_MIN_SIZE,
subject.getBytesPerChecksum());
}
+
+ @Test
+ void testClientHBaseEnhancementsAllowedTrue() {
+ // When ozone.client.hbase.enhancements.allowed = true,
+ // related client configs should be effective as-is.
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
+
+ // Note: ozone.fs.hsync.enabled is checked by
OzoneFSUtils.canEnableHsync(), thus not checked here
+ conf.setBoolean("ozone.client.incremental.chunk.list", true);
+ conf.setBoolean("ozone.client.stream.putblock.piggybacking", true);
+ conf.setInt("ozone.client.key.write.concurrency", -1);
+
+ OzoneClientConfig subject = conf.getObject(OzoneClientConfig.class);
+
+ assertTrue(subject.getIncrementalChunkList());
+ assertTrue(subject.getEnablePutblockPiggybacking());
+ assertEquals(-1, subject.getMaxConcurrentWritePerKey());
+ }
+
+ @Test
+ void testClientHBaseEnhancementsAllowedFalse() {
+ // When ozone.client.hbase.enhancements.allowed = false,
+ // related client configs should be reverted back to default.
+ OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", false);
+
+ // Note: ozone.fs.hsync.enabled is checked by
OzoneFSUtils.canEnableHsync(), thus not checked here
+ conf.setBoolean("ozone.client.incremental.chunk.list", true);
+ conf.setBoolean("ozone.client.stream.putblock.piggybacking", true);
+ conf.setInt("ozone.client.key.write.concurrency", -1);
+
+ OzoneClientConfig subject = conf.getObject(OzoneClientConfig.class);
+
+ assertFalse(subject.getIncrementalChunkList());
+ assertFalse(subject.getEnablePutblockPiggybacking());
+ assertEquals(1, subject.getMaxConcurrentWritePerKey());
+ }
}
diff --git
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
index c2cdb4d0d8..df0fdc59a4 100644
---
a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
+++
b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java
@@ -120,6 +120,14 @@ public final class OzoneConfigKeys {
public static final String OZONE_FS_DATASTREAM_AUTO_THRESHOLD_DEFAULT
= "4MB";
+ /**
+ * Flag to allow server-side HBase-related features and enhancements to be
enabled.
+ */
+ public static final String OZONE_HBASE_ENHANCEMENTS_ALLOWED
+ = "ozone.hbase.enhancements.allowed";
+ public static final boolean OZONE_HBASE_ENHANCEMENTS_ALLOWED_DEFAULT
+ = false;
+
/**
* Flag to enable hsync/hflush.
*/
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index e72b718e9f..3b845bfe92 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -4216,12 +4216,27 @@
</description>
</property>
+ <property>
+ <name>ozone.hbase.enhancements.allowed</name>
+ <value>false</value>
+ <tag>OZONE, OM</tag>
+ <description>
+ When set to false, server-side HBase enhancement-related Ozone
(experimental) features
+ are disabled (not allowed to be enabled) regardless of whether those
configs are set.
+
+ Here is the list of configs and values overridden when this config is
set to false:
+ 1. ozone.fs.hsync.enabled = false
+
+ A warning message will be printed if any of the above configs are
overridden by this.
+ </description>
+ </property>
<property>
<name>ozone.fs.hsync.enabled</name>
<value>false</value>
- <tag>OZONE, CLIENT</tag>
+ <tag>OZONE, CLIENT, OM</tag>
<description>
- Enable hsync/hflush. By default they are disabled.
+ Enable hsync/hflush on the Ozone Manager and/or client side. Disabled by
default.
+ Can be enabled only when ozone.hbase.enhancements.allowed = true
</description>
</property>
<property>
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStreamSemaphore.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStreamSemaphore.java
index 36031a9cf4..dc85fffe1c 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStreamSemaphore.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/io/KeyOutputStreamSemaphore.java
@@ -32,7 +32,7 @@ public class KeyOutputStreamSemaphore {
private final Semaphore requestSemaphore;
KeyOutputStreamSemaphore(int maxConcurrentWritePerKey) {
- LOG.info("Initializing semaphore with maxConcurrentWritePerKey = {}",
maxConcurrentWritePerKey);
+ LOG.debug("Initializing semaphore with maxConcurrentWritePerKey = {}",
maxConcurrentWritePerKey);
if (maxConcurrentWritePerKey > 0) {
requestSemaphore = new Semaphore(maxConcurrentWritePerKey);
} else if (maxConcurrentWritePerKey == 0) {
diff --git
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
index 35db51b3e4..15babfde69 100644
---
a/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
+++
b/hadoop-ozone/client/src/main/java/org/apache/hadoop/ozone/client/rpc/RpcClient.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.crypto.CryptoOutputStream;
import org.apache.hadoop.crypto.key.KeyProvider;
import org.apache.hadoop.fs.FileEncryptionInfo;
import org.apache.hadoop.fs.Syncable;
+import org.apache.hadoop.ozone.om.helpers.OzoneFSUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.hdds.client.DefaultReplicationConfig;
import org.apache.hadoop.hdds.client.ECReplicationConfig;
@@ -2489,9 +2490,7 @@ public class RpcClient implements ClientProtocol {
private OzoneOutputStream createOutputStream(OpenKeySession openKey,
KeyOutputStream keyOutputStream)
throws IOException {
- boolean enableHsync = conf.getBoolean(
- OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED,
- OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED_DEFAULT);
+ boolean enableHsync = OzoneFSUtils.canEnableHsync(conf, true);
keyOutputStream
.addPreallocateBlocks(openKey.getKeyInfo().getLatestVersionLocations(),
openKey.getOpenVersion());
@@ -2503,9 +2502,7 @@ public class RpcClient implements ClientProtocol {
private OzoneOutputStream createSecureOutputStream(OpenKeySession openKey,
OutputStream keyOutputStream, Syncable syncable) throws IOException {
- boolean enableHsync = conf.getBoolean(
- OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED,
- OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED_DEFAULT);
+ boolean enableHsync = OzoneFSUtils.canEnableHsync(conf, true);
final FileEncryptionInfo feInfo =
openKey.getKeyInfo().getFileEncryptionInfo();
if (feInfo != null) {
diff --git
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestBlockOutputStreamIncrementalPutBlock.java
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestBlockOutputStreamIncrementalPutBlock.java
index 5f2b80bdef..361dcb1fd0 100644
---
a/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestBlockOutputStreamIncrementalPutBlock.java
+++
b/hadoop-ozone/client/src/test/java/org/apache/hadoop/ozone/client/TestBlockOutputStreamIncrementalPutBlock.java
@@ -71,6 +71,8 @@ public class TestBlockOutputStreamIncrementalPutBlock {
((InMemoryConfiguration)config).setFromObject(clientConfig);
+ ((InMemoryConfiguration) config).setBoolean(
+ OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
((InMemoryConfiguration) config).setBoolean(
OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
diff --git
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
index 74effbd80a..aa7d06e2a9 100644
---
a/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
+++
b/hadoop-ozone/common/src/main/java/org/apache/hadoop/ozone/om/helpers/OzoneFSUtils.java
@@ -18,10 +18,15 @@
package org.apache.hadoop.ozone.om.helpers;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdds.conf.ConfigurationSource;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Time;
import jakarta.annotation.Nonnull;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import java.nio.file.Paths;
import java.util.UUID;
@@ -32,6 +37,7 @@ import static
org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
* Utility class for OzoneFileSystem.
*/
public final class OzoneFSUtils {
+ static final Logger LOG = LoggerFactory.getLogger(OzoneFSUtils.class);
private OzoneFSUtils() { }
@@ -292,4 +298,29 @@ public final class OzoneFSUtils {
}
return res;
}
+
+ /**
+ * Helper method to return whether Hsync can be enabled.
+ * And print warning when the config is ignored.
+ */
+ public static boolean canEnableHsync(ConfigurationSource conf, boolean
isClient) {
+ final String confKey = isClient ?
+ "ozone.client.hbase.enhancements.allowed" :
+ OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED;
+
+ boolean confHBaseEnhancementsAllowed = conf.getBoolean(
+ confKey, OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED_DEFAULT);
+
+ boolean confHsyncEnabled = conf.getBoolean(
+ OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED,
OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED_DEFAULT);
+
+ if (confHBaseEnhancementsAllowed) {
+ return confHsyncEnabled;
+ } else {
+ LOG.warn("Ignoring {} = {} because HBase enhancements are disallowed. To
enable it, set {} = true as well.",
+ OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, confHsyncEnabled,
+ confKey);
+ return false;
+ }
+ }
}
diff --git
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOzoneFsUtils.java
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOzoneFsUtils.java
index 84ad208cf9..f8363af375 100644
---
a/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOzoneFsUtils.java
+++
b/hadoop-ozone/common/src/test/java/org/apache/hadoop/ozone/om/helpers/TestOzoneFsUtils.java
@@ -18,8 +18,13 @@
package org.apache.hadoop.ozone.om.helpers;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
+import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertTrue;
import static org.junit.jupiter.api.Assertions.assertFalse;
@@ -38,4 +43,29 @@ public class TestOzoneFsUtils {
assertFalse(OzoneFSUtils.isValidName("/a:/b"));
assertFalse(OzoneFSUtils.isValidName("/a//b"));
}
+
+ /**
+ * In these scenarios below, OzoneFSUtils.canEnableHsync() should return
false:
+ * 1. ozone.hbase.enhancements.allowed = false, ozone.fs.hsync.enabled =
false
+ * 2. ozone.hbase.enhancements.allowed = false, ozone.fs.hsync.enabled = true
+ * 3. ozone.hbase.enhancements.allowed = true, ozone.fs.hsync.enabled =
false
+ * <p>
+ * The only case where OzoneFSUtils.canEnableHsync() would return true:
+ * 4. ozone.hbase.enhancements.allowed = true, ozone.fs.hsync.enabled = true
+ */
+ @ParameterizedTest
+ @CsvSource({"false,false,false,false", "false,false,true,false",
"false,true,false,false", "true,true,true,false",
+ "false,false,false,true", "false,false,true,true",
"false,true,false,true", "true,true,true,true"})
+ void testCanEnableHsync(boolean canEnableHsync,
+ boolean hbaseEnhancementsEnabled, boolean
fsHsyncEnabled,
+ boolean isClient) {
+ OzoneConfiguration conf = new OzoneConfiguration();
+ final String confKey = isClient ?
+ "ozone.client.hbase.enhancements.allowed" :
+ OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED;
+ conf.setBoolean(confKey, hbaseEnhancementsEnabled);
+ conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, fsHsyncEnabled);
+
+ assertEquals(canEnableHsync, OzoneFSUtils.canEnableHsync(conf, isClient));
+ }
}
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
index 78b67f99f1..69242d2b1f 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractOzoneFileSystemTest.java
@@ -187,6 +187,8 @@ abstract class AbstractOzoneFileSystemTest {
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
conf.setBoolean(OZONE_ACL_ENABLED, true);
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
if (!bucketLayout.equals(FILE_SYSTEM_OPTIMIZED)) {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
index 32a785a95a..cfc9029019 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/AbstractRootedOzoneFileSystemTest.java
@@ -237,6 +237,8 @@ abstract class AbstractRootedOzoneFileSystemTest {
conf.setFloat(FS_TRASH_INTERVAL_KEY, TRASH_INTERVAL);
conf.setFloat(FS_TRASH_CHECKPOINT_INTERVAL_KEY, TRASH_INTERVAL / 2);
conf.setBoolean(OMConfigKeys.OZONE_OM_RATIS_ENABLE_KEY, omRatisEnabled);
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
if (bucketLayout == BucketLayout.FILE_SYSTEM_OPTIMIZED) {
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
index 72978f8181..d0287aaee8 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSync.java
@@ -181,6 +181,8 @@ public class TestHSync {
CONF.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false);
CONF.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name());
+ CONF.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ CONF.setBoolean("ozone.client.hbase.enhancements.allowed", true);
CONF.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
CONF.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
// Reduce KeyDeletingService interval
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
index 917ce57fe7..624b5e02c1 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestHSyncUpgrade.java
@@ -107,6 +107,8 @@ public class TestHSyncUpgrade {
conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false);
conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name());
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
// Reduce KeyDeletingService interval
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
index a4a9bcff47..6a3a0eb5b6 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/TestLeaseRecovery.java
@@ -120,6 +120,8 @@ public class TestLeaseRecovery {
final BucketLayout layout = BucketLayout.FILE_SYSTEM_OPTIMIZED;
conf.setBoolean(OZONE_OM_RATIS_ENABLE_KEY, false);
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
conf.set(OZONE_DEFAULT_BUCKET_LAYOUT, layout.name());
conf.setInt(OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/AbstractOzoneContractTest.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/AbstractOzoneContractTest.java
index b79c9a870e..bce9625187 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/AbstractOzoneContractTest.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/fs/ozone/contract/AbstractOzoneContractTest.java
@@ -47,6 +47,7 @@ import java.time.Duration;
import static org.apache.hadoop.fs.contract.ContractTestUtils.cleanup;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT;
import static org.assertj.core.api.Assumptions.assumeThat;
@@ -93,6 +94,8 @@ abstract class AbstractOzoneContractTest {
conf.addResource(CONTRACT_XML);
+ conf.setBoolean(OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OZONE_FS_HSYNC_ENABLED, true);
return conf;
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java
index 766ed09bcc..3063e2587e 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/AbstractTestECKeyOutputStream.java
@@ -126,6 +126,8 @@ abstract class AbstractTestECKeyOutputStream {
zeroCopyEnabled);
conf.setInt(ScmConfigKeys.OZONE_SCM_RATIS_PIPELINE_LIMIT, 10);
// "Enable" hsync to verify that hsync would be blocked by
ECKeyOutputStream
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
ClientConfigForTesting.newBuilder(StorageUnit.BYTES)
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
index 4c978683de..eb3709c9a8 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestBlockOutputStream.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hdds.scm.storage.BufferPool;
import org.apache.hadoop.hdds.scm.storage.RatisBlockOutputStream;
import org.apache.hadoop.ozone.ClientConfigForTesting;
import org.apache.hadoop.ozone.MiniOzoneCluster;
+import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.client.ObjectStore;
import org.apache.hadoop.ozone.client.OzoneClient;
import org.apache.hadoop.ozone.client.OzoneClientFactory;
@@ -95,6 +96,9 @@ class TestBlockOutputStream {
conf.setStorageSize(OZONE_SCM_BLOCK_SIZE, 4, StorageUnit.MB);
conf.setInt(OZONE_DATANODE_PIPELINE_LIMIT, 3);
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
+
DatanodeRatisServerConfig ratisServerConfig =
conf.getObject(DatanodeRatisServerConfig.class);
ratisServerConfig.setRequestTimeOut(Duration.ofSeconds(3));
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
index 958a37380c..b2766599ae 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/client/rpc/TestSecureOzoneRpcClient.java
@@ -121,6 +121,8 @@ class TestSecureOzoneRpcClient extends OzoneRpcClientTests {
// constructed.
conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
OMConfigKeys.OZONE_BUCKET_LAYOUT_OBJECT_STORE);
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
keyProviderUri);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
index c24cf748dd..29f91821eb 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/debug/TestLeaseRecoverer.java
@@ -70,6 +70,8 @@ public class TestLeaseRecoverer {
@BeforeAll
public static void init() throws Exception {
conf = new OzoneConfiguration();
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
OzoneClientConfig clientConfig = conf.getObject(OzoneClientConfig.class);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
index 89f068cded..4c5325edab 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHA.java
@@ -82,6 +82,7 @@ import static
org.apache.hadoop.fs.FileSystem.FS_DEFAULT_NAME_KEY;
import static org.apache.hadoop.fs.FileSystem.TRASH_PREFIX;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_FS_LISTING_PAGE_SIZE;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED;
import static org.apache.hadoop.ozone.OzoneConsts.OM_KEY_PREFIX;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_OFS_URI_SCHEME;
import static org.apache.hadoop.ozone.OzoneConsts.OZONE_URI_DELIMITER;
@@ -163,6 +164,7 @@ public class TestOzoneShellHA {
@BeforeAll
public static void init() throws Exception {
OzoneConfiguration conf = new OzoneConfiguration();
+ conf.setBoolean(OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
conf.setBoolean(OZONE_FS_HSYNC_ENABLED, true);
startKMS();
startCluster(conf);
@@ -590,6 +592,7 @@ public class TestOzoneShellHA {
final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf);
+ clientConf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
clientConf.setBoolean(OZONE_FS_HSYNC_ENABLED, true);
FileSystem fs = FileSystem.get(clientConf);
@@ -709,6 +712,7 @@ public class TestOzoneShellHA {
final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf);
+ clientConf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
clientConf.setBoolean(OZONE_FS_HSYNC_ENABLED, true);
FileSystem fs = FileSystem.get(clientConf);
@@ -825,6 +829,7 @@ public class TestOzoneShellHA {
final String hostPrefix = OZONE_OFS_URI_SCHEME + "://" + omServiceId;
OzoneConfiguration clientConf = getClientConfForOFS(hostPrefix, conf);
+ clientConf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
clientConf.setBoolean(OZONE_FS_HSYNC_ENABLED, true);
FileSystem fs = FileSystem.get(clientConf);
diff --git
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java
index a0ad35500c..b1dcbc0576 100644
---
a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java
+++
b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/shell/TestOzoneShellHAWithFSO.java
@@ -38,6 +38,8 @@ public class TestOzoneShellHAWithFSO extends TestOzoneShellHA
{
OzoneConfiguration conf = new OzoneConfiguration();
conf.set(OMConfigKeys.OZONE_DEFAULT_BUCKET_LAYOUT,
OMConfigKeys.OZONE_BUCKET_LAYOUT_FILE_SYSTEM_OPTIMIZED);
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
startKMS();
startCluster(conf);
diff --git
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
index f2ea5fbfeb..cb507cd0f3 100644
---
a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/key/OMKeyCommitRequest.java
@@ -31,7 +31,6 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.ozone.OzoneManagerVersion;
import org.apache.ratis.server.protocol.TermIndex;
import org.apache.hadoop.ozone.OmUtils;
-import org.apache.hadoop.ozone.OzoneConfigKeys;
import org.apache.hadoop.ozone.OzoneConsts;
import org.apache.hadoop.ozone.om.OMConfigKeys;
import org.apache.hadoop.ozone.om.helpers.KeyValueUtil;
@@ -111,9 +110,7 @@ public class OMKeyCommitRequest extends OMKeyRequest {
}
boolean isHsync = commitKeyRequest.hasHsync() &&
commitKeyRequest.getHsync();
boolean isRecovery = commitKeyRequest.hasRecovery() &&
commitKeyRequest.getRecovery();
- boolean enableHsync = ozoneManager.getConfiguration().getBoolean(
- OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED,
- OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED_DEFAULT);
+ boolean enableHsync =
OzoneFSUtils.canEnableHsync(ozoneManager.getConfiguration(), false);
// If hsynced is called for a file, then this file is hsynced, otherwise
it's not hsynced.
// Currently, file lease recovery by design only supports recover hsynced
file
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
index 13f0191b29..1fc0cb6eba 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyCommitRequest.java
@@ -56,6 +56,8 @@ import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.CommitK
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyArgs;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.KeyLocation;
import
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.CsvSource;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.KEY_NOT_FOUND;
import static
org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
@@ -375,10 +377,19 @@ public class TestOMKeyCommitRequest extends
TestOMKeyRequest {
}
- @Test
- public void testRejectHsyncIfNotEnabled() throws Exception {
+ /**
+ * In these scenarios below, OM should reject key commit with HSync
requested from a client:
+ * 1. ozone.hbase.enhancements.allowed = false, ozone.fs.hsync.enabled =
false
+ * 2. ozone.hbase.enhancements.allowed = false, ozone.fs.hsync.enabled = true
+ * 3. ozone.hbase.enhancements.allowed = true, ozone.fs.hsync.enabled =
false
+ */
+ @ParameterizedTest
+ @CsvSource({"false,false", "false,true", "true,false"})
+ public void testRejectHsyncIfNotEnabled(boolean hbaseEnhancementsEnabled,
boolean fsHsyncEnabled) throws Exception {
OzoneConfiguration conf = ozoneManager.getConfiguration();
- conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, false);
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED,
hbaseEnhancementsEnabled);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
+ conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, fsHsyncEnabled);
BucketLayout bucketLayout = getBucketLayout();
OMRequestTestUtils.addVolumeAndBucketToDB(volumeName, bucketName,
@@ -392,6 +403,9 @@ public class TestOMKeyCommitRequest extends
TestOMKeyRequest {
// Regular key commit should still work
doKeyCommit(false, allocatedKeyLocationList.subList(0, 5));
+ // Restore config after this test run
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
}
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
index c1b0e45e6d..f636152c35 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/key/TestOMKeyRequest.java
@@ -132,6 +132,7 @@ public class TestOMKeyRequest {
folder.toAbsolutePath().toString());
ozoneConfiguration.set(OzoneConfigKeys.OZONE_METADATA_DIRS,
folder.toAbsolutePath().toString());
+
ozoneConfiguration.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED,
true);
ozoneConfiguration.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED,
true);
omMetadataManager = new OmMetadataManagerImpl(ozoneConfiguration,
ozoneManager);
diff --git
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
index eeb6f2c71e..014865f919 100644
---
a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
+++
b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/service/TestOpenKeyCleanupService.java
@@ -118,6 +118,8 @@ class TestOpenKeyCleanupService {
conf.setTimeDuration(OZONE_OM_LEASE_HARD_LIMIT,
EXPIRE_THRESHOLD_MS, TimeUnit.MILLISECONDS);
conf.set(OzoneConfigKeys.OZONE_OM_LEASE_SOFT_LIMIT, "0s");
+ conf.setBoolean(OzoneConfigKeys.OZONE_HBASE_ENHANCEMENTS_ALLOWED, true);
+ conf.setBoolean("ozone.client.hbase.enhancements.allowed", true);
conf.setBoolean(OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED, true);
conf.setQuietMode(false);
OmTestManagers omTestManagers = new OmTestManagers(conf);
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
index f25d901147..ed5574af32 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java
@@ -147,9 +147,6 @@ public class BasicOzoneFileSystem extends FileSystem {
OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD,
OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD_DEFAULT,
StorageUnit.BYTES);
- hsyncEnabled = conf.getBoolean(
- OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED,
- OZONE_FS_HSYNC_ENABLED_DEFAULT);
setConf(conf);
Preconditions.checkNotNull(name.getScheme(),
"No scheme provided in %s", name);
@@ -197,6 +194,8 @@ public class BasicOzoneFileSystem extends FileSystem {
LOG.trace("Ozone URI for ozfs initialization is {}", uri);
ConfigurationSource source = getConfSource();
+ this.hsyncEnabled = OzoneFSUtils.canEnableHsync(source, true);
+ LOG.debug("hsyncEnabled = {}", hsyncEnabled);
this.adapter =
createAdapter(source, bucketStr,
volumeStr, omHost, omPort);
diff --git
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
index eb346b5edc..3e0a373062 100644
---
a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
+++
b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java
@@ -153,9 +153,6 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD,
OzoneConfigKeys.OZONE_FS_DATASTREAM_AUTO_THRESHOLD_DEFAULT,
StorageUnit.BYTES);
- hsyncEnabled = conf.getBoolean(
- OzoneConfigKeys.OZONE_FS_HSYNC_ENABLED,
- OZONE_FS_HSYNC_ENABLED_DEFAULT);
setConf(conf);
Preconditions.checkNotNull(name.getScheme(),
"No scheme provided in %s", name);
@@ -192,6 +189,8 @@ public class BasicRootedOzoneFileSystem extends FileSystem {
LOG.trace("Ozone URI for OFS initialization is " + uri);
ConfigurationSource source = getConfSource();
+ this.hsyncEnabled = OzoneFSUtils.canEnableHsync(source, true);
+ LOG.debug("hsyncEnabled = {}", hsyncEnabled);
this.adapter = createAdapter(source, omHostOrServiceId, omPort);
this.adapterImpl = (BasicRootedOzoneClientAdapterImpl) this.adapter;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
