This is an automated email from the ASF dual-hosted git repository. szetszwo pushed a commit to branch HDDS-4454 in repository https://gitbox.apache.org/repos/asf/ozone.git
commit 4c63b2360a04ffa341c2695776ebe477c794a4d0 Author: Tsz-Wo Nicholas Sze <[email protected]> AuthorDate: Fri Oct 28 14:27:38 2022 +0800 HDDS-7431. [Ozone-Streaming] Disable data steam by default. (#3900) (cherry picked from commit 53af9086bb8b8a8d8d446c4eada4e19891750d4f) --- .../org/apache/hadoop/ozone/OzoneConfigKeys.java | 15 +++++++------- .../common/src/main/resources/ozone-default.xml | 9 ++++---- .../transport/server/ratis/XceiverServerRatis.java | 4 ++-- .../container/common/TestDatanodeStateMachine.java | 2 ++ .../upgrade/TestDatanodeUpgradeToSchemaV3.java | 2 ++ .../ozone/container/common/TestEndPoint.java | 4 ++++ .../org/apache/hadoop/ozone/MiniOzoneCluster.java | 1 + .../apache/hadoop/ozone/MiniOzoneClusterImpl.java | 2 ++ .../apache/hadoop/ozone/TestMiniOzoneCluster.java | 2 ++ .../server/TestSecureContainerServer.java | 2 ++ .../hadoop/fs/ozone/BasicOzoneFileSystem.java | 4 ++-- .../fs/ozone/BasicRootedOzoneFileSystem.java | 4 ++-- .../hadoop/ozone/client/ClientProtocolStub.java | 24 ++++++++++++++++++++++ 13 files changed, 58 insertions(+), 17 deletions(-) diff --git a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java index aab528a1b1..70c22eebfc 100644 --- a/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java +++ b/hadoop-hdds/common/src/main/java/org/apache/hadoop/ozone/OzoneConfigKeys.java @@ -88,10 +88,10 @@ public final class OzoneConfigKeys { /** * Ratis Port where containers listen to datastream requests. */ - public static final String DFS_CONTAINER_RATIS_DATASTREAM_ENABLE - = "dfs.container.ratis.datastream.enable"; - public static final boolean DFS_CONTAINER_RATIS_DATASTREAM_ENABLE_DEFAULT - = true; + public static final String DFS_CONTAINER_RATIS_DATASTREAM_ENABLED + = "dfs.container.ratis.datastream.enabled"; + public static final boolean DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT + = false; public static final String DFS_CONTAINER_RATIS_DATASTREAM_PORT = "dfs.container.ratis.datastream.port"; public static final int DFS_CONTAINER_RATIS_DATASTREAM_PORT_DEFAULT @@ -100,9 +100,10 @@ public final class OzoneConfigKeys { /** * Flag to enable ratis streaming on filesystem writes. */ - public static final String OZONE_FS_DATASTREAM_ENABLE = - "ozone.fs.datastream.enable"; - public static final boolean OZONE_FS_DATASTREAM_ENABLE_DEFAULT = false; + public static final String OZONE_FS_DATASTREAM_ENABLED + = "ozone.fs.datastream.enabled"; + public static final boolean OZONE_FS_DATASTREAM_ENABLED_DEFAULT + = false; /** * When set to true, allocate a random free port for ozone container, so that diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml b/hadoop-hdds/common/src/main/resources/ozone-default.xml index b9671b4fa9..a3263f75e1 100644 --- a/hadoop-hdds/common/src/main/resources/ozone-default.xml +++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml @@ -53,11 +53,12 @@ <tag>OZONE, CONTAINER, MANAGEMENT</tag> <description>The ipc port number of container.</description> </property> + <property> - <name>dfs.container.ratis.datastream.enable</name> - <value>true</value> + <name>dfs.container.ratis.datastream.enabled</name> + <value>false</value> <tag>OZONE, CONTAINER, RATIS, DATASTREAM</tag> - <description>If enable datastream ipc of container.</description> + <description>It specifies whether to enable data stream of container.</description> </property> <property> <name>dfs.container.ratis.datastream.port</name> @@ -3454,7 +3455,7 @@ </property> <property> - <name>ozone.fs.datastream.enable</name> + <name>ozone.fs.datastream.enabled</name> <value>false</value> <tag>OZONE, DATANODE</tag> <description> diff --git a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java index 6b0ad0e41e..89e5047b85 100644 --- a/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java +++ b/hadoop-hdds/container-service/src/main/java/org/apache/hadoop/ozone/container/common/transport/server/ratis/XceiverServerRatis.java @@ -161,8 +161,8 @@ public final class XceiverServerRatis implements XceiverServerSpi { datanodeDetails = dd; assignPorts(); this.streamEnable = conf.getBoolean( - OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLE, - OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLE_DEFAULT); + OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED_DEFAULT); RaftProperties serverProperties = newRaftProperties(); this.context = context; this.dispatcher = dispatcher; diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java index 8aa1222233..f2faeaa3a3 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/common/TestDatanodeStateMachine.java @@ -81,6 +81,8 @@ public class TestDatanodeStateMachine { TimeUnit.MILLISECONDS); conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, true); conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, true); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + true); conf.setBoolean( OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true); serverAddresses = new ArrayList<>(); diff --git a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java index c004b38843..7af39e5e7f 100644 --- a/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java +++ b/hadoop-hdds/container-service/src/test/java/org/apache/hadoop/ozone/container/upgrade/TestDatanodeUpgradeToSchemaV3.java @@ -105,6 +105,8 @@ public class TestDatanodeUpgradeToSchemaV3 { conf = new OzoneConfiguration(); conf.setBoolean(DatanodeConfiguration.CONTAINER_SCHEMA_V3_ENABLED, this.schemaV3Enabled); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + true); conf.setBoolean( OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true); } diff --git a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java index 56a04de02c..b620ace2a9 100644 --- a/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java +++ b/hadoop-hdds/server-scm/src/test/java/org/apache/hadoop/ozone/container/common/TestEndPoint.java @@ -144,6 +144,8 @@ public class TestEndPoint { try (EndpointStateMachine rpcEndPoint = createEndpoint(conf, serverAddress, 1000)) { DatanodeDetails datanodeDetails = randomDatanodeDetails(); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + true); conf.setBoolean( OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true); OzoneContainer ozoneContainer = new OzoneContainer( @@ -170,6 +172,8 @@ public class TestEndPoint { true); conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, true); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + true); conf.setBoolean( OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true); conf.setFromObject(new ReplicationConfig().setPort(0)); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java index 4c485d805c..e7a4cf0319 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneCluster.java @@ -321,6 +321,7 @@ public interface MiniOzoneCluster { protected Boolean randomContainerPort = true; protected Boolean randomContainerStreamPort = true; + protected Boolean enableContainerDatastream = true; protected Optional<String> datanodeReservedSpace = Optional.empty(); protected Optional<Integer> chunkSize = Optional.empty(); protected OptionalInt streamBufferSize = OptionalInt.empty(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java index 39f3552b36..d0c897b488 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/MiniOzoneClusterImpl.java @@ -933,6 +933,8 @@ public class MiniOzoneClusterImpl implements MiniOzoneCluster { randomContainerPort); conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, randomContainerPort); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + enableContainerDatastream); conf.setBoolean(DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, randomContainerStreamPort); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java index 7da2bc8e17..4d4e9fb8b6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/TestMiniOzoneCluster.java @@ -216,6 +216,8 @@ public class TestMiniOzoneCluster { ozoneConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_IPC_RANDOM_PORT, true); ozoneConf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_RANDOM_PORT, true); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + true); ozoneConf.setBoolean( OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true); List<DatanodeStateMachine> stateMachines = new ArrayList<>(); diff --git a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java index 51d617a4e0..62e7ecbda6 100644 --- a/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java +++ b/hadoop-ozone/integration-test/src/test/java/org/apache/hadoop/ozone/container/server/TestSecureContainerServer.java @@ -216,6 +216,8 @@ public class TestSecureContainerServer { DatanodeDetails dn, OzoneConfiguration conf) throws IOException { conf.setInt(OzoneConfigKeys.DFS_CONTAINER_RATIS_IPC_PORT, dn.getPort(DatanodeDetails.Port.Name.RATIS).getValue()); + conf.setBoolean(OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_ENABLED, + true); conf.setBoolean( OzoneConfigKeys.DFS_CONTAINER_RATIS_DATASTREAM_RANDOM_PORT, true); final String dir = TEST_DIR + dn.getUuid(); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java index e3289e0c57..c274b89a14 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicOzoneFileSystem.java @@ -272,8 +272,8 @@ public class BasicOzoneFileSystem extends FileSystem { private FSDataOutputStream createOutputStream(String key, short replication, boolean overwrite, boolean recursive) throws IOException { boolean isRatisStreamingEnabled = getConf().getBoolean( - OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE, - OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE_DEFAULT); + OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED, + OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED_DEFAULT); if (isRatisStreamingEnabled) { return new FSDataOutputStream(adapter.createStreamFile(key, replication, overwrite, recursive), statistics); diff --git a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java index 28c8772138..e61f65e4fa 100644 --- a/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java +++ b/hadoop-ozone/ozonefs-common/src/main/java/org/apache/hadoop/fs/ozone/BasicRootedOzoneFileSystem.java @@ -253,8 +253,8 @@ public class BasicRootedOzoneFileSystem extends FileSystem { private FSDataOutputStream createOutputStream(String key, short replication, boolean overwrite, boolean recursive) throws IOException { boolean isRatisStreamingEnabled = getConf().getBoolean( - OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE, - OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLE_DEFAULT); + OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED, + OzoneConfigKeys.OZONE_FS_DATASTREAM_ENABLED_DEFAULT); if (isRatisStreamingEnabled) { return new FSDataOutputStream(adapter.createStreamFile(key, replication, overwrite, recursive), statistics); diff --git a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java index 7c1daeca0b..e8da93b366 100644 --- a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java +++ b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/client/ClientProtocolStub.java @@ -27,6 +27,7 @@ import org.apache.hadoop.hdds.protocol.DatanodeDetails; import org.apache.hadoop.hdds.protocol.StorageType; import org.apache.hadoop.io.Text; import org.apache.hadoop.ozone.OzoneAcl; +import org.apache.hadoop.ozone.client.io.OzoneDataStreamOutput; import org.apache.hadoop.ozone.client.io.OzoneInputStream; import org.apache.hadoop.ozone.client.io.OzoneOutputStream; import org.apache.hadoop.ozone.client.protocol.ClientProtocol; @@ -583,4 +584,27 @@ public class ClientProtocolStub implements ClientProtocol { String volumeName, String bucketName, String keyName) throws IOException { return null; } + + @Override + public OzoneDataStreamOutput createStreamKey( + String volumeName, String bucketName, String keyName, long size, + ReplicationConfig replicationConfig, Map<String, String> metadata) + throws IOException { + return null; + } + + @Override + public OzoneDataStreamOutput createMultipartStreamKey( + String volumeName, String bucketName, String keyName, long size, + int partNumber, String uploadID) throws IOException { + return null; + } + + @Override + public OzoneDataStreamOutput createStreamFile( + String volumeName, String bucketName, String keyName, long size, + ReplicationConfig replicationConf, boolean overWrite, boolean recursive) + throws IOException { + return null; + } } --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
