This is an automated email from the ASF dual-hosted git repository.
smengcl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 8ca44914771 HDDS-15087. S3 Gateway and Local S3 UX (#10759)
8ca44914771 is described below
commit 8ca449147710bb69be1b77b22d24d0f70709f72c
Author: Chun-Hung Tseng <[email protected]>
AuthorDate: Fri Oct 9 06:46:43 2026 +0200
HDDS-15087. S3 Gateway and Local S3 UX (#10759)
---
.../common/src/main/resources/ozone-default.xml | 11 ++
hadoop-hdds/docs/content/tools/_index.md | 4 +-
.../hadoop/hdds/server/http/HttpServer2.java | 3 +
hadoop-ozone/integration-test-s3/pom.xml | 5 +
.../hadoop/ozone/local/TestLocalOzoneS3.java | 95 ++++++++++++++
.../java/org/apache/hadoop/ozone/s3/Gateway.java | 1 -
.../hadoop/ozone/s3/OzoneConfigurationHolder.java | 3 -
.../hadoop/ozone/s3/S3GatewayConfigKeys.java | 8 ++
.../hadoop/ozone/s3/endpoint/EndpointBase.java | 7 +-
.../org/apache/hadoop/ozone/s3/util/S3Utils.java | 8 +-
.../apache/hadoop/ozone/s3/util/TestS3Utils.java | 16 ++-
hadoop-ozone/tools/pom.xml | 4 +
.../hadoop/ozone/local/LocalOzoneCluster.java | 137 ++++++++++++++++++---
.../ozone/local/LocalOzoneClusterConfig.java | 59 ++-------
.../hadoop/ozone/local/LocalOzoneRuntime.java | 4 +-
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 46 ++-----
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 111 +++++++++++++++++
.../ozone/local/TestLocalOzoneClusterConfig.java | 11 +-
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 76 +++++++-----
19 files changed, 451 insertions(+), 158 deletions(-)
diff --git a/hadoop-hdds/common/src/main/resources/ozone-default.xml
b/hadoop-hdds/common/src/main/resources/ozone-default.xml
index e4be1210fae..4ba267b677c 100644
--- a/hadoop-hdds/common/src/main/resources/ozone-default.xml
+++ b/hadoop-hdds/common/src/main/resources/ozone-default.xml
@@ -2101,6 +2101,17 @@
</description>
</property>
+ <property>
+ <name>ozone.s3g.standard.storage-class.use-client-default</name>
+ <value>false</value>
+ <tag>OZONE, S3GATEWAY</tag>
+ <description>
+ When true, a request with x-amz-storage-class STANDARD uses
ozone.replication and
+ ozone.replication.type, if ozone.replication is set, instead of
RATIS/THREE.
+ The ozone local runtime sets it to true.
+ </description>
+ </property>
+
<property>
<name>ozone.s3g.secret.http.enabled</name>
<value>false</value>
diff --git a/hadoop-hdds/docs/content/tools/_index.md
b/hadoop-hdds/docs/content/tools/_index.md
index 3a8bf08e826..adbd4740f87 100644
--- a/hadoop-hdds/docs/content/tools/_index.md
+++ b/hadoop-hdds/docs/content/tools/_index.md
@@ -43,8 +43,8 @@ Client commands:
* **sh** - Primary command line interface for ozone to manage
volumes/buckets/keys.
* **fs** - Runs a command on ozone file system (similar to `hdfs dfs`)
- * **local** - Runs a single-node local Ozone cluster (SCM, OM, and
datanodes)
- in one process for development.
+ * **local** - Runs a single-node local Ozone cluster (SCM, OM, datanodes,
+ and optional S3 Gateway) in one process for development.
* **version** - Prints the version of Ozone and HDDS.
diff --git
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
index 775e1bc1cf9..1568d26fcfe 100644
---
a/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
+++
b/hadoop-hdds/framework/src/main/java/org/apache/hadoop/hdds/server/http/HttpServer2.java
@@ -785,6 +785,9 @@ private static WebAppContext createWebAppContext(Builder b,
AccessControlList adminsAcl, final String appDir) {
WebAppContext ctx = new WebAppContext();
ctx.setDefaultsDescriptor(null);
+ // S3 Gateway starts Weld through its web.xml listener; other services do
not use CDI.
+ ctx.setAttribute("org.eclipse.jetty.containerInitializerExclusionPattern",
+ "org.jboss.weld.environment.servlet.EnhancedListener");
ServletHolder holder = new ServletHolder(new DefaultServlet());
Map<String, String> params = ImmutableMap.<String, String>builder()
.put("acceptRanges", "true")
diff --git a/hadoop-ozone/integration-test-s3/pom.xml
b/hadoop-ozone/integration-test-s3/pom.xml
index 1f7c884d6f7..9629f2ba31c 100644
--- a/hadoop-ozone/integration-test-s3/pom.xml
+++ b/hadoop-ozone/integration-test-s3/pom.xml
@@ -149,6 +149,11 @@
<artifactId>ozone-s3gateway</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-tools</artifactId>
+ <scope>test</scope>
+ </dependency>
<dependency>
<groupId>org.apache.ratis</groupId>
<artifactId>ratis-common</artifactId>
diff --git
a/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneS3.java
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneS3.java
new file mode 100644
index 00000000000..4cdaa7137f9
--- /dev/null
+++
b/hadoop-ozone/integration-test-s3/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneS3.java
@@ -0,0 +1,95 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.local;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.net.URI;
+import java.nio.file.Path;
+import java.util.UUID;
+import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.StorageUnit;
+import org.apache.hadoop.ozone.ClientConfigForTesting;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
+import software.amazon.awssdk.auth.credentials.AwsBasicCredentials;
+import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider;
+import software.amazon.awssdk.core.sync.RequestBody;
+import software.amazon.awssdk.regions.Region;
+import software.amazon.awssdk.services.s3.S3Client;
+import software.amazon.awssdk.services.s3.model.StorageClass;
+
+/**
+ * Integration tests for the S3 Gateway of the {@code ozone local} runtime.
+ */
+class TestLocalOzoneS3 {
+
+ @TempDir
+ private Path tempDir;
+
+ /**
+ * Security is off, so the S3 Gateway accepts credentials it has never seen.
The object carries an explicit STANDARD
+ * storage class, which succeeds on the single default datanode only because
ozone local sets
+ * {@code ozone.s3g.standard.storage-class.use-client-default}; otherwise
S3StorageType maps it to RATIS/THREE.
+ */
+ @Test
+ void s3GatewayServesRequests() throws Exception {
+ LocalOzoneClusterConfig config =
LocalOzoneClusterConfig.builder(tempDir.resolve("local-ozone-s3")).build();
+ OzoneConfiguration conf = new OzoneConfiguration();
+ // Match the test module's 128 MB containers; production blocks default to
256 MB.
+ ClientConfigForTesting.newBuilder(StorageUnit.MB).applyTo(conf);
+
+ try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, conf)) {
+ cluster.start();
+
+ assertTrue(cluster.getS3gPort() > 0);
+ assertTrue(cluster.getS3gBoundAddress().getAddress().isLoopbackAddress(),
+ () -> cluster.getS3gBoundAddress().toString());
+
+ assertBucketRoundTrip(cluster.getS3Endpoint(), "local-smoke",
+ LocalOzoneClusterConfig.LOCAL_S3_ACCESS_KEY,
LocalOzoneClusterConfig.LOCAL_S3_SECRET_KEY);
+ String unknown = UUID.randomUUID().toString().replace("-", "");
+ assertBucketRoundTrip(cluster.getS3Endpoint(), "local-smoke-" + unknown,
unknown, unknown);
+
+ try (S3Client s3 = s3Client(cluster.getS3Endpoint(),
+ LocalOzoneClusterConfig.LOCAL_S3_ACCESS_KEY,
LocalOzoneClusterConfig.LOCAL_S3_SECRET_KEY)) {
+ s3.putObject(request ->
request.bucket("local-smoke").key("object").storageClass(StorageClass.STANDARD),
+ RequestBody.fromString("payload"));
+ assertEquals("payload",
+ s3.getObjectAsBytes(request ->
request.bucket("local-smoke").key("object")).asUtf8String());
+ }
+ }
+ }
+
+ private static void assertBucketRoundTrip(String endpoint, String bucket,
String accessKey, String secretKey) {
+ try (S3Client s3 = s3Client(endpoint, accessKey, secretKey)) {
+ s3.createBucket(request -> request.bucket(bucket));
+ assertTrue(s3.listBuckets().buckets().stream().anyMatch(b ->
bucket.equals(b.name())));
+ }
+ }
+
+ private static S3Client s3Client(String endpoint, String accessKey, String
secretKey) {
+ return S3Client.builder()
+ .endpointOverride(URI.create(endpoint))
+ .region(Region.of(LocalOzoneClusterConfig.LOCAL_S3_REGION))
+
.credentialsProvider(StaticCredentialsProvider.create(AwsBasicCredentials.create(accessKey,
secretKey)))
+ .forcePathStyle(true)
+ .build();
+ }
+}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
index 41e89fbde35..0b738d01eb8 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/Gateway.java
@@ -160,7 +160,6 @@ private static void loginS3GUser(OzoneConfiguration conf)
}
}
- @VisibleForTesting
public InetSocketAddress getHttpAddress() {
return this.httpServer.getHttpAddress();
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java
index 88652f0457a..9a35d293913 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/OzoneConfigurationHolder.java
@@ -17,7 +17,6 @@
package org.apache.hadoop.ozone.s3;
-import com.google.common.annotations.VisibleForTesting;
import javax.enterprise.inject.Produces;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
@@ -39,7 +38,6 @@ public static OzoneConfiguration configuration() {
return configuration;
}
- @VisibleForTesting
public static void setConfiguration(
OzoneConfiguration conf) {
// Nullity check is used in case the configuration was already set
@@ -49,7 +47,6 @@ public static void setConfiguration(
}
}
- @VisibleForTesting
public static void resetConfiguration() {
configuration = null;
}
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
index cc674f4b0d6..468dd018099 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/S3GatewayConfigKeys.java
@@ -102,6 +102,14 @@ public final class S3GatewayConfigKeys {
public static final String OZONE_S3G_LIST_MAX_KEYS_LIMIT =
"ozone.s3g.list.max.keys.limit";
public static final int OZONE_S3G_LIST_MAX_KEYS_LIMIT_DEFAULT = 1000;
+ /**
+ * Configuration key that makes an explicit {@code x-amz-storage-class:
STANDARD} use the {@code ozone.replication}
+ * setting, when set, instead of {@link
org.apache.hadoop.ozone.s3.util.S3StorageType#STANDARD}.
+ */
+ public static final String
OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_KEY =
+ "ozone.s3g.standard.storage-class.use-client-default";
+ public static final boolean
OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_DEFAULT = false;
+
/**
* Never constructed.
*/
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
index b7b02b5fc53..74a90f1fc6e 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/endpoint/EndpointBase.java
@@ -32,6 +32,8 @@
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_CLIENT_BUFFER_SIZE_KEY;
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED;
import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_FSO_DIRECTORY_CREATION_ENABLED_DEFAULT;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_DEFAULT;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_KEY;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_EXISTS;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.BUCKET_ALREADY_OWNED_BY_YOU;
import static
org.apache.hadoop.ozone.s3.exception.S3ErrorTable.INVALID_ARGUMENT;
@@ -820,9 +822,12 @@ protected ReplicationConfig
getReplicationConfig(OzoneBucket ozoneBucket) throws
ReplicationConfig clientConfiguredReplicationConfig =
OzoneClientUtils.getClientConfiguredReplicationConfig(getOzoneConfiguration());
+ boolean standardUsesClientDefault = getOzoneConfiguration().getBoolean(
+ OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_KEY,
+ OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_DEFAULT);
return S3Utils.resolveS3ClientSideReplicationConfig(storageType,
storageConfig,
- clientConfiguredReplicationConfig, ozoneBucket.getReplicationConfig());
+ clientConfiguredReplicationConfig, ozoneBucket.getReplicationConfig(),
standardUsesClientDefault);
}
/**
diff --git
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
index b72c4adb2d2..6acd1ef9e0f 100644
---
a/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
+++
b/hadoop-ozone/s3gateway/src/main/java/org/apache/hadoop/ozone/s3/util/S3Utils.java
@@ -105,17 +105,23 @@ private S3Utils() {
* config.
* @param clientConfiguredReplConfig - Client side configured replication
* config.
+ * @param standardUsesClientDefault - whether STANDARD resolves to a
non-null
+ * clientConfiguredReplConfig.
* @return client resolved replication config.
*/
public static ReplicationConfig resolveS3ClientSideReplicationConfig(
String s3StorageTypeHeader, String s3StorageConfigHeader,
ReplicationConfig clientConfiguredReplConfig,
- ReplicationConfig bucketReplConfig)
+ ReplicationConfig bucketReplConfig, boolean standardUsesClientDefault)
throws OS3Exception {
// If user provided s3 storage type header is not null then map it
// to ozone replication config
if (!StringUtils.isEmpty(s3StorageTypeHeader)) {
+ if (standardUsesClientDefault && clientConfiguredReplConfig != null
+ && S3StorageType.STANDARD.name().equals(s3StorageTypeHeader)) {
+ return clientConfiguredReplConfig;
+ }
return toReplicationConfig(s3StorageTypeHeader, s3StorageConfigHeader);
}
diff --git
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
index f178c507ac9..fe086d57020 100644
---
a/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
+++
b/hadoop-ozone/s3gateway/src/test/java/org/apache/hadoop/ozone/s3/util/TestS3Utils.java
@@ -82,7 +82,10 @@ public static List<Arguments> validS3ReplicationConfigs() {
for (String s3StorageConfig : S3STORAGECONFIG) {
for (ReplicationConfig clientReplConfig : REPLICATIONS) {
for (ReplicationConfig bucketReplConfig: REPLICATIONS) {
- args.add(Arguments.of(s3StorageType, s3StorageConfig,
clientReplConfig, bucketReplConfig));
+ for (boolean standardUsesClientDefault : new boolean[] {false,
true}) {
+ args.add(Arguments.of(s3StorageType, s3StorageConfig,
clientReplConfig, bucketReplConfig,
+ standardUsesClientDefault));
+ }
}
}
}
@@ -93,11 +96,11 @@ public static List<Arguments> validS3ReplicationConfigs() {
@ParameterizedTest
@MethodSource("validS3ReplicationConfigs")
public void testValidResolveS3ClientSideReplicationConfig(String
s3StorageType, String s3StorageConfig,
- ReplicationConfig clientConfiguredReplConfig, ReplicationConfig
bucketReplConfig)
- throws OS3Exception {
+ ReplicationConfig clientConfiguredReplConfig, ReplicationConfig
bucketReplConfig,
+ boolean standardUsesClientDefault) throws OS3Exception {
ReplicationConfig replicationConfig = S3Utils
.resolveS3ClientSideReplicationConfig(s3StorageType, s3StorageConfig,
- clientConfiguredReplConfig, bucketReplConfig);
+ clientConfiguredReplConfig, bucketReplConfig,
standardUsesClientDefault);
final ReplicationConfig expectedReplConfig;
if (!StringUtils.isEmpty(s3StorageType)) {
@@ -108,7 +111,8 @@ public void
testValidResolveS3ClientSideReplicationConfig(String s3StorageType,
expectedReplConfig = EC32REPLICATIONCONFIG;
}
} else if (S3StorageType.STANDARD.name().equals(s3StorageType)) {
- expectedReplConfig = RATIS3REPLICATIONCONFIG;
+ expectedReplConfig = standardUsesClientDefault &&
clientConfiguredReplConfig != null
+ ? clientConfiguredReplConfig : RATIS3REPLICATIONCONFIG;
} else {
expectedReplConfig = RATIS1REPLICATIONCONFIG;
}
@@ -146,7 +150,7 @@ public void
testResolveRepConfWhenUserPassedIsInvalid(String s3StorageType, Stri
throws OS3Exception {
OS3Exception exception = assertThrows(OS3Exception.class, () -> S3Utils.
resolveS3ClientSideReplicationConfig(
- s3StorageType, s3StorageConfig, clientConfiguredReplConfig,
bucketReplConfig));
+ s3StorageType, s3StorageConfig, clientConfiguredReplConfig,
bucketReplConfig, false));
assertEquals(S3ErrorTable.INVALID_STORAGE_CLASS.getCode(),
exception.getCode());
}
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index 8429437e5ad..bdd362d9ba4 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -87,6 +87,10 @@
<groupId>org.apache.ozone</groupId>
<artifactId>ozone-manager</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-s3gateway</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.ratis</groupId>
<artifactId>ratis-common</artifactId>
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java
index fdab4179bd8..8bb057f601b 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java
@@ -54,6 +54,8 @@
import static
org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_IPC_PORT;
import static
org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_RATIS_SERVER_PORT;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_BASEDIR;
+import static
org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_POLICY_DEFAULT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_POLICY_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
@@ -72,10 +74,21 @@
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_SNAPSHOT_DIFF_DB_DIR;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_BIND_HOST_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_BIND_HOST_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ENABLED_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTPS_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTPS_BIND_HOST_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_WEBADMIN_HTTP_BIND_HOST_KEY;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
+import java.net.InetSocketAddress;
import java.net.ServerSocket;
import java.nio.file.Files;
import java.nio.file.Path;
@@ -103,6 +116,7 @@
import org.apache.hadoop.hdds.scm.proxy.SCMClientConfig;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
+import org.apache.hadoop.hdds.server.http.HttpConfig;
import org.apache.hadoop.hdds.utils.IOUtils;
import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
import org.apache.hadoop.ozone.HddsDatanodeService;
@@ -111,13 +125,14 @@
import org.apache.hadoop.ozone.container.replication.ReplicationServer;
import org.apache.hadoop.ozone.om.OMStorage;
import org.apache.hadoop.ozone.om.OzoneManager;
+import org.apache.hadoop.ozone.s3.Gateway;
+import org.apache.hadoop.ozone.s3.OzoneConfigurationHolder;
+import org.apache.ratis.util.function.CheckedSupplier;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
/**
- * Starts the SCM, OM, and datanode portion of the {@code ozone local} runtime.
- *
- * <p>S3 Gateway and Recon are added by later local runtime tickets.</p>
+ * Starts the SCM, OM, datanode, and optional S3 Gateway portion of the {@code
ozone local} runtime.
*/
public final class LocalOzoneCluster implements LocalOzoneRuntime {
@@ -145,6 +160,8 @@ public final class LocalOzoneCluster implements
LocalOzoneRuntime {
private static final String OM_HTTP_PORT_KEY = "om.http";
private static final String OM_HTTPS_PORT_KEY = "om.https";
private static final String OM_RATIS_PORT_KEY = "om.ratis";
+ private static final String S3G_HTTP_PORT_KEY = "s3g.http";
+ private static final String S3G_WEBADMIN_HTTP_PORT_KEY = "s3g.web.http";
private static final String DATANODE_PORT_KEY_PREFIX = "dn.";
private static final String DATANODE_HTTP_PORT_KEY_SUFFIX = "http";
private static final String DATANODE_CLIENT_PORT_KEY_SUFFIX = "client";
@@ -205,6 +222,7 @@ public final class LocalOzoneCluster implements
LocalOzoneRuntime {
private PreparedConfiguration preparedConfiguration;
private StorageContainerManager scm;
private OzoneManager om;
+ private Gateway s3Gateway;
private final List<HddsDatanodeService> datanodes = new ArrayList<>();
private boolean previousMetricsMiniClusterMode;
private boolean metricsMiniClusterModeEnabled;
@@ -236,6 +254,9 @@ public void start() throws Exception {
startOm(prepared.getConfiguration());
startDatanodes(prepared.getDatanodeConfigurations());
waitForClusterReadiness(config.getStartupTimeout());
+ if (config.isS3gEnabled()) {
+ startS3Gateway(prepared.getConfiguration());
+ }
} catch (Exception ex) {
// Roll back without latching closed: the caller's close() still owns the
// ephemeral data dir lifecycle.
@@ -257,23 +278,23 @@ PreparedConfiguration prepareConfiguration() throws
IOException {
configureLocalDefaults(conf);
PersistedPortState persistedPorts = loadPersistedPortState();
- PortAllocator portAllocator = new PortAllocator();
+ PortAllocator portAllocator = new PortAllocator(persistedPorts.ports());
int scmPort = configureScm(conf, persistedPorts, portAllocator);
int omPort = configureOm(conf, persistedPorts, portAllocator);
+ int s3gPort = configureS3Gateway(conf, persistedPorts, portAllocator);
List<OzoneConfiguration> datanodeConfigurations =
configureDatanodes(conf, persistedPorts, portAllocator);
prepareStorageLayout();
createServiceDirectories();
persistedPorts.store();
- preparedConfiguration = new PreparedConfiguration(conf, scmPort, omPort,
- datanodeConfigurations);
+ preparedConfiguration = new PreparedConfiguration(conf, scmPort, omPort,
s3gPort, datanodeConfigurations);
return preparedConfiguration;
}
@Override
public String getDisplayHost() {
- return LocalOzoneClusterConfig.DEFAULT_BIND_HOST.equals(config.getHost())
+ return LocalOzoneClusterConfig.WILDCARD_HOST.equals(config.getHost())
? LocalOzoneClusterConfig.DEFAULT_HOST : config.getHost();
}
@@ -296,12 +317,19 @@ public int getDatanodeCount() {
@Override
public int getS3gPort() {
- return -1;
+ InetSocketAddress address = getS3gBoundAddress();
+ return address != null ? address.getPort() : -1;
}
@Override
public String getS3Endpoint() {
- return "";
+ int port = getS3gPort();
+ return port > 0 ? "http://"; + address(getDisplayHost(), port) : "";
+ }
+
+ /** Returns the bound S3 Gateway HTTP address, or null when the S3 Gateway
is not running. */
+ InetSocketAddress getS3gBoundAddress() {
+ return s3Gateway != null ? s3Gateway.getHttpAddress() : null;
}
@Override
@@ -330,12 +358,20 @@ private void stopServices() {
try {
// Logged rather than closeQuietly: stopServices() is also the start()
rollback, where a silent
// failure hides leaked threads and ports.
- IOUtils.close(LOG, this::stopDatanodes, this::stopOm, this::stopScm);
+ IOUtils.close(LOG, this::stopS3Gateway, this::stopDatanodes,
this::stopOm, this::stopScm);
} finally {
restoreSameJvmMetricsMode();
}
}
+ private void stopS3Gateway() throws Exception {
+ Gateway service = s3Gateway;
+ s3Gateway = null;
+ if (service != null) {
+ service.stop();
+ }
+ }
+
private void stopDatanodes() {
List<AutoCloseable> stoppers = new ArrayList<>();
for (int i = datanodes.size() - 1; i >= 0; i--) {
@@ -610,6 +646,37 @@ private void configureOmStorage(OzoneConfiguration conf) {
omMetadataDir.toString());
}
+ private int configureS3Gateway(OzoneConfiguration conf, PersistedPortState
persistedPorts,
+ PortAllocator portAllocator) throws IOException {
+ if (!config.isS3gEnabled()) {
+ return -1;
+ }
+ // getS3Endpoint() advertises the HTTP listener, which a policy without
HTTP leaves unbound.
+ HttpConfig.Policy httpPolicy =
+ HttpConfig.Policy.fromString(conf.get(OZONE_HTTP_POLICY_KEY,
OZONE_HTTP_POLICY_DEFAULT));
+ if (httpPolicy != null && !httpPolicy.isHttpEnabled()) {
+ rejectUserConfigured(conf, OZONE_HTTP_POLICY_KEY, "HTTP_ONLY or
HTTP_AND_HTTPS");
+ }
+ // getS3gPort() reads the port back off this listener.
+ setLocalOverride(conf, OZONE_S3G_HTTP_ENABLED_KEY, true);
+ // S3StorageType maps an explicit STANDARD to RATIS/THREE regardless of
the local STAND_ALONE/ONE replication.
+ setLocalOverride(conf,
OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_KEY, true);
+
+ int s3gHttpPort = reservePort(portAllocator, persistedPorts,
S3G_HTTP_PORT_KEY, config.getS3gPort());
+ int s3gWebHttpPort = reservePort(portAllocator, persistedPorts,
S3G_WEBADMIN_HTTP_PORT_KEY, 0);
+
+ // HTTPS addresses stay on port 0: the printed endpoint is HTTP, so HTTPS
needs no stable port.
+ conf.set(OZONE_S3G_HTTP_ADDRESS_KEY, address(config.getHost(),
s3gHttpPort));
+ conf.set(OZONE_S3G_HTTP_BIND_HOST_KEY, config.getBindHost());
+ conf.set(OZONE_S3G_HTTPS_ADDRESS_KEY, address(config.getHost(), 0));
+ conf.set(OZONE_S3G_HTTPS_BIND_HOST_KEY, config.getBindHost());
+ conf.set(OZONE_S3G_WEBADMIN_HTTP_ADDRESS_KEY, address(config.getHost(),
s3gWebHttpPort));
+ conf.set(OZONE_S3G_WEBADMIN_HTTP_BIND_HOST_KEY, config.getBindHost());
+ conf.set(OZONE_S3G_WEBADMIN_HTTPS_ADDRESS_KEY, address(config.getHost(),
0));
+ conf.set(OZONE_S3G_WEBADMIN_HTTPS_BIND_HOST_KEY, config.getBindHost());
+ return s3gHttpPort;
+ }
+
private List<OzoneConfiguration> configureDatanodes(OzoneConfiguration conf,
PersistedPortState persistedPorts, PortAllocator portAllocator)
throws IOException {
@@ -765,6 +832,18 @@ private void startDatanodes(List<OzoneConfiguration>
datanodeConfigurations) {
}
}
+ private void startS3Gateway(OzoneConfiguration conf) throws Exception {
+ // Gateway reads the static holder; setConfiguration() keeps the first
value.
+ OzoneConfigurationHolder.resetConfiguration();
+ OzoneConfigurationHolder.setConfiguration(new OzoneConfiguration(conf));
+ // Assigned first so start()'s stopServices() rollback stops a failed
call().
+ s3Gateway = new Gateway();
+ // call(), not execute(): GenericCli#printError drops the cause and can
exit the JVM.
+ // parseArgs() sets the parse result Gateway#start reads.
+ s3Gateway.getCmd().parseArgs();
+ s3Gateway.call();
+ }
+
private void waitForClusterReadiness(Duration timeout) throws Exception {
waitForReadiness(this::clusterReadinessBlocker, "Ozone cluster", timeout);
}
@@ -938,6 +1017,10 @@ private PersistedPortState loadPersistedPortState()
throws IOException {
private String[] requiredPersistedPortKeys() {
List<String> keys =
new ArrayList<>(Arrays.asList(REQUIRED_PERSISTED_PORT_KEYS));
+ if (config.isS3gEnabled()) {
+ keys.add(S3G_HTTP_PORT_KEY);
+ keys.add(S3G_WEBADMIN_HTTP_PORT_KEY);
+ }
for (int index = 0; index < config.getDatanodes(); index++) {
for (String suffix : DATANODE_PORT_KEY_SUFFIXES) {
keys.add(datanodePortKey(index, suffix));
@@ -1000,14 +1083,16 @@ static final class PreparedConfiguration {
private final OzoneConfiguration configuration;
private final int scmPort;
private final int omPort;
+ private final int s3gPort;
private final List<OzoneConfiguration> datanodeConfigurations;
- PreparedConfiguration(OzoneConfiguration configuration, int scmPort,
- int omPort, List<OzoneConfiguration> datanodeConfigurations) {
+ PreparedConfiguration(OzoneConfiguration configuration, int scmPort, int
omPort, int s3gPort,
+ List<OzoneConfiguration> datanodeConfigurations) {
this.configuration = Objects.requireNonNull(configuration,
"configuration");
this.scmPort = scmPort;
this.omPort = omPort;
+ this.s3gPort = s3gPort;
this.datanodeConfigurations = Collections.unmodifiableList(
new ArrayList<>(Objects.requireNonNull(datanodeConfigurations,
"datanodeConfigurations")));
@@ -1025,6 +1110,10 @@ int getOmPort() {
return omPort;
}
+ int getS3gPort() {
+ return s3gPort;
+ }
+
List<OzoneConfiguration> getDatanodeConfigurations() {
return datanodeConfigurations;
}
@@ -1035,6 +1124,17 @@ List<OzoneConfiguration> getDatanodeConfigurations() {
*/
static final class PortAllocator {
private final Set<Integer> reserved = new HashSet<>();
+ private final Set<Integer> persisted;
+ private final CheckedSupplier<Integer, IOException> freePorts;
+
+ PortAllocator(Set<Integer> persisted) {
+ this(persisted, PortAllocator::nextFreePort);
+ }
+
+ PortAllocator(Set<Integer> persisted, CheckedSupplier<Integer,
IOException> freePorts) {
+ this.persisted = persisted;
+ this.freePorts = freePorts;
+ }
int reserve(int preferredPort) throws IOException {
if (preferredPort > 0) {
@@ -1042,8 +1142,9 @@ int reserve(int preferredPort) throws IOException {
}
while (true) {
- int candidate = nextFreePort();
- if (reserved.add(candidate)) {
+ int candidate = freePorts.get();
+ // A persisted port is free while the cluster is down, but its key may
reserve it later in this run.
+ if (!persisted.contains(candidate) && reserved.add(candidate)) {
return candidate;
}
}
@@ -1120,6 +1221,14 @@ int get(String key) throws IOException {
}
}
+ Set<Integer> ports() throws IOException {
+ Set<Integer> ports = new HashSet<>();
+ for (String key : properties.stringPropertyNames()) {
+ ports.add(get(key));
+ }
+ return ports;
+ }
+
void requireKeys(String[] keys) throws IOException {
for (String key : keys) {
if (get(key) <= 0) {
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java
index 4a5d9d7ea20..e2d62d2f860 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java
@@ -56,7 +56,7 @@ public final class LocalOzoneClusterConfig {
static final int DEFAULT_DATANODES =
Integer.parseInt(DEFAULT_DATANODES_VALUE);
static final String DEFAULT_HOST = "127.0.0.1";
- static final String DEFAULT_BIND_HOST = "0.0.0.0";
+ static final String WILDCARD_HOST = "0.0.0.0";
static final int DEFAULT_PORT = Integer.parseInt(DEFAULT_PORT_VALUE);
static final boolean DEFAULT_S3G_ENABLED =
Boolean.parseBoolean(DEFAULT_S3G_ENABLED_VALUE);
@@ -64,9 +64,10 @@ public final class LocalOzoneClusterConfig {
Boolean.parseBoolean(DEFAULT_EPHEMERAL_VALUE);
static final Duration DEFAULT_STARTUP_TIMEOUT =
Duration.parse(DEFAULT_STARTUP_TIMEOUT_VALUE);
- static final String DEFAULT_S3_ACCESS_KEY = "admin";
- static final String DEFAULT_S3_SECRET_KEY = "admin123";
- static final String DEFAULT_S3_REGION = "us-east-1";
+ // Printed for S3 clients; not checked while security is off.
+ static final String LOCAL_S3_ACCESS_KEY = "admin";
+ static final String LOCAL_S3_SECRET_KEY = "admin123";
+ static final String LOCAL_S3_REGION = "us-east-1";
private final Path dataDir;
private final FormatMode formatMode;
@@ -79,9 +80,6 @@ public final class LocalOzoneClusterConfig {
private final boolean s3gEnabled;
private final boolean ephemeral;
private final Duration startupTimeout;
- private final String s3AccessKey;
- private final String s3SecretKey;
- private final String s3Region;
private LocalOzoneClusterConfig(Builder builder) {
dataDir = Objects.requireNonNull(builder.dataDir, "dataDir")
@@ -90,7 +88,8 @@ private LocalOzoneClusterConfig(Builder builder) {
formatMode = Objects.requireNonNull(builder.formatMode, "formatMode");
datanodes = builder.datanodes;
host = Objects.requireNonNull(builder.host, "host");
- bindHost = Objects.requireNonNull(builder.bindHost, "bindHost");
+ // Defaults to host rather than WILDCARD_HOST: the local runtime runs
without security by default.
+ bindHost = builder.bindHost != null ? builder.bindHost : host;
scmPort = builder.scmPort;
omPort = builder.omPort;
s3gPort = builder.s3gPort;
@@ -98,9 +97,6 @@ private LocalOzoneClusterConfig(Builder builder) {
ephemeral = builder.ephemeral;
startupTimeout = Objects.requireNonNull(builder.startupTimeout,
"startupTimeout");
- s3AccessKey = Objects.requireNonNull(builder.s3AccessKey, "s3AccessKey");
- s3SecretKey = Objects.requireNonNull(builder.s3SecretKey, "s3SecretKey");
- s3Region = Objects.requireNonNull(builder.s3Region, "s3Region");
}
public Path getDataDir() {
@@ -170,27 +166,6 @@ public Duration getStartupTimeout() {
return startupTimeout;
}
- /**
- * Returns the suggested local-only S3 access key printed for client setup.
- */
- public String getS3AccessKey() {
- return s3AccessKey;
- }
-
- /**
- * Returns the suggested local-only S3 secret key printed for client setup.
- */
- public String getS3SecretKey() {
- return s3SecretKey;
- }
-
- /**
- * Returns the suggested local-only S3 region printed for client setup.
- */
- public String getS3Region() {
- return s3Region;
- }
-
public static Builder builder() {
return new Builder(DEFAULT_DATA_DIR);
}
@@ -240,16 +215,13 @@ public static final class Builder {
private FormatMode formatMode = DEFAULT_FORMAT_MODE;
private int datanodes = DEFAULT_DATANODES;
private String host = DEFAULT_HOST;
- private String bindHost = DEFAULT_BIND_HOST;
+ private String bindHost;
private int scmPort = DEFAULT_PORT;
private int omPort = DEFAULT_PORT;
private int s3gPort = DEFAULT_PORT;
private boolean s3gEnabled = DEFAULT_S3G_ENABLED;
private boolean ephemeral = DEFAULT_EPHEMERAL;
private Duration startupTimeout = DEFAULT_STARTUP_TIMEOUT;
- private String s3AccessKey = DEFAULT_S3_ACCESS_KEY;
- private String s3SecretKey = DEFAULT_S3_SECRET_KEY;
- private String s3Region = DEFAULT_S3_REGION;
private Builder(Path dataDir) {
this.dataDir = dataDir;
@@ -305,21 +277,6 @@ public Builder setStartupTimeout(Duration value) {
return this;
}
- public Builder setS3AccessKey(String value) {
- s3AccessKey = value;
- return this;
- }
-
- public Builder setS3SecretKey(String value) {
- s3SecretKey = value;
- return this;
- }
-
- public Builder setS3Region(String value) {
- s3Region = value;
- return this;
- }
-
public LocalOzoneClusterConfig build() {
return new LocalOzoneClusterConfig(this);
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
index 3ed93904417..ed0176bd234 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
@@ -60,14 +60,14 @@ public interface LocalOzoneRuntime extends AutoCloseable {
/**
* Returns the S3 Gateway HTTP port for this local runtime.
*
- * @return S3 Gateway port
+ * @return S3 Gateway port, or -1 when the S3 Gateway is not running
*/
int getS3gPort();
/**
* Returns the full S3 Gateway endpoint shown to users.
*
- * @return S3 Gateway endpoint, including scheme, host, and port
+ * @return S3 Gateway endpoint, including scheme, host, and port; empty when
the S3 Gateway is not running
*/
String getS3Endpoint();
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
index 79b50efc729..f2565361148 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
@@ -62,9 +62,6 @@ public class OzoneLocal extends GenericCli {
static final String ENV_S3G_PORT = "OZONE_LOCAL_S3G_PORT";
static final String ENV_EPHEMERAL = "OZONE_LOCAL_EPHEMERAL";
static final String ENV_STARTUP_TIMEOUT = "OZONE_LOCAL_STARTUP_TIMEOUT";
- static final String ENV_S3_ACCESS_KEY = "OZONE_LOCAL_S3_ACCESS_KEY";
- static final String ENV_S3_SECRET_KEY = "OZONE_LOCAL_S3_SECRET_KEY";
- static final String ENV_S3_REGION = "OZONE_LOCAL_S3_REGION";
private static final String DEFAULT_DATA_DIR_VALUE = "${env:" + ENV_DATA_DIR
+ ":-" + LocalOzoneClusterConfig.DEFAULT_DATA_DIR_VALUE + "}";
@@ -75,8 +72,7 @@ public class OzoneLocal extends GenericCli {
+ "}";
private static final String DEFAULT_HOST_VALUE = "${env:" + ENV_HOST
+ ":-" + LocalOzoneClusterConfig.DEFAULT_HOST + "}";
- private static final String DEFAULT_BIND_HOST_VALUE = "${env:"
- + ENV_BIND_HOST + ":-" + LocalOzoneClusterConfig.DEFAULT_BIND_HOST + "}";
+ private static final String DEFAULT_BIND_HOST_VALUE = "${env:" +
ENV_BIND_HOST + "}";
private static final String DEFAULT_SCM_PORT_VALUE = "${env:"
+ ENV_SCM_PORT + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE
+ "}";
@@ -93,14 +89,6 @@ public class OzoneLocal extends GenericCli {
private static final String DEFAULT_STARTUP_TIMEOUT_VALUE = "${env:"
+ ENV_STARTUP_TIMEOUT + ":-"
+ LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE + "}";
- private static final String DEFAULT_S3_ACCESS_KEY_VALUE = "${env:"
- + ENV_S3_ACCESS_KEY + ":-"
- + LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY + "}";
- private static final String DEFAULT_S3_SECRET_KEY_VALUE = "${env:"
- + ENV_S3_SECRET_KEY + ":-"
- + LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY + "}";
- private static final String DEFAULT_S3_REGION_VALUE = "${env:"
- + ENV_S3_REGION + ":-" + LocalOzoneClusterConfig.DEFAULT_S3_REGION + "}";
public OzoneLocal() {
super();
@@ -162,7 +150,7 @@ private StartupFailureException(Throwable cause) {
}
@Command(name = "run",
- description = "Start SCM, OM, and datanodes in one local process")
+ description = "Start SCM, OM, datanodes, and optional S3 Gateway in one
local process")
static class RunCommand extends AbstractSubcommand implements Callable<Void>
{
@Option(names = "--data-dir",
@@ -188,7 +176,7 @@ static class RunCommand extends AbstractSubcommand
implements Callable<Void> {
@Option(names = "--bind-host",
defaultValue = DEFAULT_BIND_HOST_VALUE,
- description = "Bind host for HTTP and RPC listeners")
+ description = "Bind host for HTTP and RPC listeners (default: --host)")
private String bindHost;
@Option(names = "--scm-port",
@@ -226,21 +214,6 @@ static class RunCommand extends AbstractSubcommand
implements Callable<Void> {
description = "How long to wait for the local cluster to become ready")
private Duration startupTimeout;
- @Option(names = "--s3-access-key",
- defaultValue = DEFAULT_S3_ACCESS_KEY_VALUE,
- description = "Suggested local AWS access key to print on startup")
- private String s3AccessKey;
-
- @Option(names = "--s3-secret-key",
- defaultValue = DEFAULT_S3_SECRET_KEY_VALUE,
- description = "Suggested local AWS secret key to print on startup")
- private String s3SecretKey;
-
- @Option(names = "--s3-region",
- defaultValue = DEFAULT_S3_REGION_VALUE,
- description = "Suggested local AWS region to print on startup")
- private String s3Region;
-
@Override
public Void call() throws Exception {
LocalOzoneClusterConfig config = resolveConfig();
@@ -270,6 +243,16 @@ private void printSummary(LocalOzoneRuntime runtime,
LocalOzoneClusterConfig con
writer.println("Local Ozone is running from " + config.getDataDir());
writer.println("SCM RPC: " + runtime.getDisplayHost() + ":" +
runtime.getScmPort());
writer.println("OM RPC: " + runtime.getDisplayHost() + ":" +
runtime.getOmPort());
+ if (config.isS3gEnabled()) {
+ writer.println("S3 endpoint: " + runtime.getS3Endpoint());
+ writer.println("Suggested local AWS settings:");
+ writer.println("AWS_ACCESS_KEY_ID=" +
LocalOzoneClusterConfig.LOCAL_S3_ACCESS_KEY);
+ writer.println("AWS_SECRET_ACCESS_KEY=" +
LocalOzoneClusterConfig.LOCAL_S3_SECRET_KEY);
+ writer.println("AWS_REGION=" +
LocalOzoneClusterConfig.LOCAL_S3_REGION);
+ writer.println("AWS_ENDPOINT_URL_S3=" + runtime.getS3Endpoint());
+ writer.println("Use path-style addressing (AWS CLI: aws configure set
default.s3.addressing_style path).");
+ writer.println("With security off (the default), the S3 Gateway
accepts any credentials.");
+ }
writer.println("Press Ctrl+C to stop.");
writer.flush();
}
@@ -322,9 +305,6 @@ LocalOzoneClusterConfig resolveConfig() {
.setS3gEnabled(s3gEnabled)
.setEphemeral(ephemeral)
.setStartupTimeout(startupTimeout)
- .setS3AccessKey(s3AccessKey)
- .setS3SecretKey(s3SecretKey)
- .setS3Region(s3Region)
.build();
}
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java
index c0d2e19aea5..6da5f287e24 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.ozone.local;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_DATANODE_CLIENT_BIND_HOST_KEY;
import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION;
@@ -25,18 +26,27 @@
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
+import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_BIND_HOST_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_NAMES;
import static org.apache.hadoop.ozone.OzoneConfigKeys.HDDS_CONTAINER_IPC_PORT;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_HTTP_POLICY_KEY;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_METADATA_DIRS;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_HTTP_BIND_HOST_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTPS_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_BIND_HOST_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_HTTP_ENABLED_KEY;
+import static
org.apache.hadoop.ozone.s3.S3GatewayConfigKeys.OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotEquals;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertSame;
import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
@@ -48,6 +58,7 @@
import java.time.Duration;
import java.util.Arrays;
import java.util.Collections;
+import java.util.Iterator;
import java.util.Properties;
import java.util.concurrent.TimeoutException;
import java.util.concurrent.atomic.AtomicInteger;
@@ -55,8 +66,11 @@
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SafeModeRuleStatusProto;
+import org.apache.hadoop.hdds.server.http.HttpConfig;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.EnumSource;
/**
* Tests for {@link LocalOzoneCluster}.
@@ -83,9 +97,23 @@ void prepareConfigurationExposesPreparedPorts() throws
Exception {
assertTrue(Files.isRegularFile(portStateFile(dataDir)));
assertTrue(prepared.getScmPort() > 0);
assertTrue(prepared.getOmPort() > 0);
+ assertTrue(prepared.getS3gPort() > 0);
+ }
+ }
+
+ @Test
+ void prepareConfigurationSkipsS3GatewayWhenDisabled() throws Exception {
+ Path dataDir = tempDir.resolve("local-ozone");
+ LocalOzoneClusterConfig config =
LocalOzoneClusterConfig.builder(dataDir).setS3gEnabled(false).build();
+
+ try (LocalOzoneCluster cluster = newCluster(config)) {
+ LocalOzoneCluster.PreparedConfiguration prepared =
cluster.prepareConfiguration();
+
+ assertEquals(-1, prepared.getS3gPort());
assertEquals(-1, cluster.getS3gPort());
assertEquals("", cluster.getS3Endpoint());
}
+
assertFalse(loadPortState(dataDir).stringPropertyNames().stream().anyMatch(key
-> key.startsWith("s3g.")));
}
@Test
@@ -95,6 +123,7 @@ void prepareConfigurationCreatesBaseLayoutAndLocalDefaults()
LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir)
.setScmPort(9860)
.setOmPort(9862)
+ .setS3gPort(9878)
.setDatanodes(2)
.build();
@@ -116,10 +145,67 @@ void
prepareConfigurationCreatesBaseLayoutAndLocalDefaults()
assertEquals(2, conf.getInt(HDDS_SCM_SAFEMODE_MIN_DATANODE, 0));
assertTrue(conf.get(OZONE_SCM_CLIENT_ADDRESS_KEY).endsWith(":9860"));
assertTrue(conf.get(OZONE_OM_ADDRESS_KEY).endsWith(":9862"));
+ assertTrue(conf.get(OZONE_S3G_HTTP_ADDRESS_KEY).endsWith(":9878"));
+ assertTrue(conf.get(OZONE_S3G_HTTPS_ADDRESS_KEY).endsWith(":0"));
+
assertTrue(conf.getBoolean(OZONE_S3G_STANDARD_STORAGE_CLASS_USE_CLIENT_DEFAULT_KEY,
false));
assertTrue(conf.getTrimmedStringCollection(OZONE_SCM_NAMES).iterator()
.next().contains(":"));
assertEquals(9860, prepared.getScmPort());
assertEquals(9862, prepared.getOmPort());
+ assertEquals(9878, prepared.getS3gPort());
+ }
+
+ @Test
+ void s3AccessorsReportNoListenerBeforeStart() throws Exception {
+ LocalOzoneClusterConfig config = defaultConfig();
+
+ try (LocalOzoneCluster cluster = newCluster(config)) {
+ assertTrue(config.isS3gEnabled());
+ assertEquals(-1, cluster.getS3gPort());
+ assertEquals("", cluster.getS3Endpoint());
+ assertNull(cluster.getS3gBoundAddress());
+ }
+ }
+
+ @Test
+ void prepareConfigurationRejectsDisabledS3GatewayHttpListener() {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.setBoolean(OZONE_S3G_HTTP_ENABLED_KEY, false);
+
+ IOException error = assertPrepareFails(seed);
+
+ assertMessageContains(error, OZONE_S3G_HTTP_ENABLED_KEY);
+ }
+
+ @ParameterizedTest
+ @EnumSource(HttpConfig.Policy.class)
+ void prepareConfigurationChecksHttpPolicyForS3Gateway(HttpConfig.Policy
policy) throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_HTTP_POLICY_KEY, policy.name(), "test-ozone-site.xml");
+
+ if (policy.isHttpEnabled()) {
+ prepared(seed);
+ } else {
+ IOException error = assertPrepareFails(seed);
+ assertMessageContains(error, OZONE_HTTP_POLICY_KEY);
+ assertMessageContains(error, "test-ozone-site.xml");
+ }
+ LocalOzoneClusterConfig withoutS3g =
+
LocalOzoneClusterConfig.builder(tempDir.resolve("without-s3g")).setS3gEnabled(false).build();
+ try (LocalOzoneCluster cluster = new LocalOzoneCluster(withoutS3g, seed)) {
+ cluster.prepareConfiguration();
+ }
+ }
+
+ @Test
+ void prepareConfigurationBindsListenersToLoopbackByDefault() throws
Exception {
+ LocalOzoneCluster.PreparedConfiguration prepared =
prepare(defaultConfig());
+ OzoneConfiguration conf = prepared.getConfiguration();
+
+ assertEquals("127.0.0.1", conf.get(OZONE_SCM_CLIENT_BIND_HOST_KEY));
+ assertEquals("127.0.0.1", conf.get(OZONE_OM_HTTP_BIND_HOST_KEY));
+ assertEquals("127.0.0.1", conf.get(OZONE_S3G_HTTP_BIND_HOST_KEY));
+ assertEquals("127.0.0.1",
prepared.getDatanodeConfigurations().get(0).get(HDDS_DATANODE_CLIENT_BIND_HOST_KEY));
}
@Test
@@ -312,10 +398,35 @@ void persistedPortFileContainsDistinctAllocatedPorts()
throws Exception {
assertPositivePort(properties, "om.rpc");
assertPositivePort(properties, "om.http");
assertPositivePort(properties, "om.ratis");
+ assertPositivePort(properties, "s3g.http");
+ assertPositivePort(properties, "s3g.web.http");
assertNotEquals(properties.getProperty("scm.client"),
properties.getProperty("om.rpc"));
}
+ @Test
+ void freshPortSkipsPersistedPorts() throws Exception {
+ Iterator<Integer> freePorts = Arrays.asList(40000, 40001).iterator();
+ LocalOzoneCluster.PortAllocator allocator =
+ new LocalOzoneCluster.PortAllocator(Collections.singleton(40000),
freePorts::next);
+
+ assertEquals(40001, allocator.reserve(0));
+ assertEquals(40000, allocator.reserve(40000));
+ }
+
+ @Test
+ void formatNeverRejectsPortStateMissingS3GatewayPorts() throws Exception {
+ Path dataDir = tempDir.resolve("local-ozone");
+
prepare(LocalOzoneClusterConfig.builder(dataDir).setS3gEnabled(false).build());
+ LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir)
+ .setFormatMode(LocalOzoneClusterConfig.FormatMode.NEVER)
+ .build();
+
+ IOException error = assertPrepareFails(config);
+
+ assertMessageContains(error, "s3g.");
+ }
+
@Test
void prepareConfigurationCreatesDatanodeConfigurations() throws Exception {
Path dataDir = tempDir.resolve("local-ozone");
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java
index 7e7f8248024..1f9fddbb965 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java
@@ -41,16 +41,13 @@ void builderProvidesLocalClusterDefaults() {
config.getFormatMode());
assertEquals(1, config.getDatanodes());
assertEquals("127.0.0.1", config.getHost());
- assertEquals("0.0.0.0", config.getBindHost());
+ assertEquals("127.0.0.1", config.getBindHost());
assertEquals(0, config.getScmPort());
assertEquals(0, config.getOmPort());
assertEquals(0, config.getS3gPort());
assertTrue(config.isS3gEnabled());
assertFalse(config.isEphemeral());
assertEquals(Duration.ofMinutes(2), config.getStartupTimeout());
- assertEquals("admin", config.getS3AccessKey());
- assertEquals("admin123", config.getS3SecretKey());
- assertEquals("us-east-1", config.getS3Region());
}
@Test
@@ -88,9 +85,6 @@ void builderAcceptsExplicitOverrides() {
.setS3gEnabled(false)
.setEphemeral(true)
.setStartupTimeout(Duration.ofSeconds(45))
- .setS3AccessKey("dev")
- .setS3SecretKey("secret")
- .setS3Region("ap-south-1")
.build();
assertEquals(Paths.get("target", "custom-local-ozone")
@@ -106,9 +100,6 @@ void builderAcceptsExplicitOverrides() {
assertFalse(config.isS3gEnabled());
assertTrue(config.isEphemeral());
assertEquals(Duration.ofSeconds(45), config.getStartupTimeout());
- assertEquals("dev", config.getS3AccessKey());
- assertEquals("secret", config.getS3SecretKey());
- assertEquals("ap-south-1", config.getS3Region());
}
@Test
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
index c91c30d563a..9427be1d502 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
@@ -101,7 +101,7 @@ void rootHelpListsRunCommand() throws Exception {
@Test
void runCommandStartsRuntimeAndPrintsStartupSummary() throws Exception {
ByteArrayOutputStream out = new ByteArrayOutputStream();
- StubRuntime runtime = new StubRuntime("localhost", 9860, 9862);
+ StubRuntime runtime = new StubRuntime("localhost", 9860, 9862,
"http://localhost:9878";);
TestableRunCommand command = new TestableRunCommand(runtime);
CommandLine commandLine = new CommandLine(command);
commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8),
@@ -117,12 +117,36 @@ void runCommandStartsRuntimeAndPrintsStartupSummary()
throws Exception {
assertTrue(text.contains("SCM RPC: localhost:9860"), text);
assertTrue(text.contains("OM RPC: localhost:9862"), text);
assertFalse(text.contains("Datanodes:"), text);
+ assertTrue(text.contains("S3 endpoint: http://localhost:9878";), text);
+ assertTrue(text.contains("AWS_ACCESS_KEY_ID=" +
LocalOzoneClusterConfig.LOCAL_S3_ACCESS_KEY), text);
+ assertTrue(text.contains("AWS_SECRET_ACCESS_KEY=" +
LocalOzoneClusterConfig.LOCAL_S3_SECRET_KEY), text);
+ assertTrue(text.contains("AWS_REGION=" +
LocalOzoneClusterConfig.LOCAL_S3_REGION), text);
+ assertTrue(text.contains("AWS_ENDPOINT_URL_S3=http://localhost:9878";),
text);
+ assertTrue(text.contains("aws configure set default.s3.addressing_style
path"), text);
+ assertTrue(text.contains("accepts any credentials"), text);
+ assertTrue(text.contains("Press Ctrl+C to stop."), text);
+ }
+
+ @Test
+ void runCommandOmitsS3SummaryWhenS3gDisabled() throws Exception {
+ ByteArrayOutputStream out = new ByteArrayOutputStream();
+ StubRuntime runtime = new StubRuntime("localhost", 9860, 9862, "");
+ TestableRunCommand command = new TestableRunCommand(runtime);
+ CommandLine commandLine = new CommandLine(command);
+ commandLine.setOut(new PrintWriter(new OutputStreamWriter(out, UTF_8),
true));
+
+ int exitCode = commandLine.execute("--no-s3g");
+
+ assertEquals(0, exitCode);
+ String text = out.toString(UTF_8.name());
+ assertFalse(text.contains("S3 endpoint:"), text);
+ assertFalse(text.contains("AWS_ACCESS_KEY_ID="), text);
assertTrue(text.contains("Press Ctrl+C to stop."), text);
}
@Test
void runCommandClosesRuntimeWhenStartupFails() {
- StubRuntime runtime = new StubRuntime("localhost", 9860, 9862);
+ StubRuntime runtime = new StubRuntime("localhost", 9860, 9862, "");
runtime.failStart = true;
TestableRunCommand command = new TestableRunCommand(runtime);
@@ -135,7 +159,7 @@ void runCommandClosesRuntimeWhenStartupFails() {
@Test
void runCommandPreservesStartupFailureAsTheCause() throws Exception {
ByteArrayOutputStream err = new ByteArrayOutputStream();
- StubRuntime runtime = new StubRuntime("localhost", 9860, 9862);
+ StubRuntime runtime = new StubRuntime("localhost", 9860, 9862, "");
runtime.failStart = true;
TestableRunCommand command = new TestableRunCommand(runtime);
CommandLine commandLine = new CommandLine(command);
@@ -243,8 +267,8 @@ void runCommandOptionsUseEnvironmentDefaults() throws
Exception {
LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE);
assertEnvDefault("host", OzoneLocal.ENV_HOST,
LocalOzoneClusterConfig.DEFAULT_HOST);
- assertEnvDefault("bindHost", OzoneLocal.ENV_BIND_HOST,
- LocalOzoneClusterConfig.DEFAULT_BIND_HOST);
+ assertEquals("${env:" + OzoneLocal.ENV_BIND_HOST + "}",
+
OzoneLocal.RunCommand.class.getDeclaredField("bindHost").getAnnotation(Option.class).defaultValue());
assertEnvDefault("scmPort", OzoneLocal.ENV_SCM_PORT,
LocalOzoneClusterConfig.DEFAULT_PORT_VALUE);
assertEnvDefault("omPort", OzoneLocal.ENV_OM_PORT,
@@ -257,12 +281,6 @@ void runCommandOptionsUseEnvironmentDefaults() throws
Exception {
LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE);
assertEnvDefault("startupTimeout", OzoneLocal.ENV_STARTUP_TIMEOUT,
LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE);
- assertEnvDefault("s3AccessKey", OzoneLocal.ENV_S3_ACCESS_KEY,
- LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY);
- assertEnvDefault("s3SecretKey", OzoneLocal.ENV_S3_SECRET_KEY,
- LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY);
- assertEnvDefault("s3Region", OzoneLocal.ENV_S3_REGION,
- LocalOzoneClusterConfig.DEFAULT_S3_REGION);
}
@Test
@@ -275,16 +293,18 @@ void resolveConfigUsesPicocliDefaults() {
config.getFormatMode());
assertEquals(1, config.getDatanodes());
assertEquals("127.0.0.1", config.getHost());
- assertEquals("0.0.0.0", config.getBindHost());
+ assertEquals("127.0.0.1", config.getBindHost());
assertEquals(0, config.getScmPort());
assertEquals(0, config.getOmPort());
assertEquals(0, config.getS3gPort());
assertTrue(config.isS3gEnabled());
assertFalse(config.isEphemeral());
assertEquals(Duration.ofMinutes(2), config.getStartupTimeout());
- assertEquals("admin", config.getS3AccessKey());
- assertEquals("admin123", config.getS3SecretKey());
- assertEquals("us-east-1", config.getS3Region());
+ }
+
+ @Test
+ void resolveConfigBindsToHostWithoutBindHost() {
+ assertEquals("cli-host", resolve("--host", "cli-host").getBindHost());
}
@Test
@@ -294,16 +314,13 @@ void resolveConfigUsesCliOverrides() {
"--format", "always",
"--datanodes", "3",
"--host", "cli-host",
- "--bind-host", "127.0.0.1",
+ "--bind-host", "0.0.0.0",
"--scm-port", "200",
"--om-port", "201",
"--s3g-port", "202",
"--no-s3g",
"--ephemeral",
- "--startup-timeout", "45s",
- "--s3-access-key", "cli-access",
- "--s3-secret-key", "cli-secret",
- "--s3-region", "cli-region");
+ "--startup-timeout", "45s");
assertEquals(Paths.get("target/cli-local").toAbsolutePath().normalize(),
config.getDataDir());
@@ -311,16 +328,13 @@ void resolveConfigUsesCliOverrides() {
config.getFormatMode());
assertEquals(3, config.getDatanodes());
assertEquals("cli-host", config.getHost());
- assertEquals("127.0.0.1", config.getBindHost());
+ assertEquals("0.0.0.0", config.getBindHost());
assertEquals(200, config.getScmPort());
assertEquals(201, config.getOmPort());
assertEquals(202, config.getS3gPort());
assertFalse(config.isS3gEnabled());
assertTrue(config.isEphemeral());
assertEquals(Duration.ofSeconds(45), config.getStartupTimeout());
- assertEquals("cli-access", config.getS3AccessKey());
- assertEquals("cli-secret", config.getS3SecretKey());
- assertEquals("cli-region", config.getS3Region());
}
@Test
@@ -484,14 +498,16 @@ private static final class StubRuntime implements
LocalOzoneRuntime {
private final String displayHost;
private final int scmPort;
private final int omPort;
+ private final String s3Endpoint;
private boolean failStart;
private boolean started;
private boolean closed;
- private StubRuntime(String displayHost, int scmPort, int omPort) {
+ private StubRuntime(String displayHost, int scmPort, int omPort, String
s3Endpoint) {
this.displayHost = displayHost;
this.scmPort = scmPort;
this.omPort = omPort;
+ this.s3Endpoint = s3Endpoint;
}
@Override
@@ -524,7 +540,7 @@ public int getS3gPort() {
@Override
public String getS3Endpoint() {
- return "";
+ return s3Endpoint;
}
@Override
@@ -550,8 +566,6 @@ public String defaultValue(ArgSpec argSpec) {
return LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE;
} else if ("--host".equals(option)) {
return LocalOzoneClusterConfig.DEFAULT_HOST;
- } else if ("--bind-host".equals(option)) {
- return LocalOzoneClusterConfig.DEFAULT_BIND_HOST;
} else if ("--scm-port".equals(option)
|| "--om-port".equals(option)
|| "--s3g-port".equals(option)) {
@@ -562,12 +576,6 @@ public String defaultValue(ArgSpec argSpec) {
return LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE;
} else if ("--startup-timeout".equals(option)) {
return LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE;
- } else if ("--s3-access-key".equals(option)) {
- return LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY;
- } else if ("--s3-secret-key".equals(option)) {
- return LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY;
- } else if ("--s3-region".equals(option)) {
- return LocalOzoneClusterConfig.DEFAULT_S3_REGION;
}
return null;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
