This is an automated email from the ASF dual-hosted git repository.
peterxcli pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new e5da18a959c HDDS-15082. Add User facing Config Contract for `ozone
local` (#10147)
e5da18a959c is described below
commit e5da18a959c30aa19f5c68500ac9f810abc36b14
Author: Chun-Hung Tseng <[email protected]>
AuthorDate: Mon May 11 21:06:35 2026 +0800
HDDS-15082. Add User facing Config Contract for `ozone local` (#10147)
---
.../ozone/local/LocalOzoneClusterConfig.java | 327 +++++++++++++++++++++
.../hadoop/ozone/local/LocalOzoneRuntime.java | 44 ++-
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 234 ++++++++++++++-
.../apache/hadoop/ozone/local/package-info.java | 2 +-
.../ozone/local/TestLocalOzoneClusterConfig.java | 135 +++++++++
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 273 ++++++++++++++++-
6 files changed, 1005 insertions(+), 10 deletions(-)
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java
new file mode 100644
index 00000000000..4a5d9d7ea20
--- /dev/null
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneClusterConfig.java
@@ -0,0 +1,327 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.local;
+
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.Locale;
+import java.util.Objects;
+
+/**
+ * Configuration for a local Ozone cluster runtime.
+ *
+ * <p>The datanode count describes how many local datanode services should run
+ * on the same host.
+ */
+public final class LocalOzoneClusterConfig {
+
+ private static final String DEFAULT_DATA_DIR_PARENT = ".ozone";
+ private static final String DEFAULT_DATA_DIR_NAME = "local";
+
+ // Picocli annotation defaults require compile-time strings, so the CLI uses
+ // this expression while the typed default below uses the same path
fragments.
+ static final String DEFAULT_DATA_DIR_VALUE =
+ "${sys:user.home}${sys:file.separator}" + DEFAULT_DATA_DIR_PARENT
+ + "${sys:file.separator}" + DEFAULT_DATA_DIR_NAME;
+ static final String DEFAULT_FORMAT_MODE_VALUE = "if-needed";
+ static final String DEFAULT_DATANODES_VALUE = "1";
+ static final String DEFAULT_PORT_VALUE = "0";
+ static final String DEFAULT_S3G_ENABLED_VALUE = "true";
+ static final String DEFAULT_EPHEMERAL_VALUE = "false";
+ static final String DEFAULT_STARTUP_TIMEOUT_VALUE = "PT2M";
+
+ static final Path DEFAULT_DATA_DIR =
+ Paths.get(System.getProperty("user.home"), DEFAULT_DATA_DIR_PARENT,
+ DEFAULT_DATA_DIR_NAME)
+ .toAbsolutePath()
+ .normalize();
+ static final FormatMode DEFAULT_FORMAT_MODE =
+ FormatMode.fromString(DEFAULT_FORMAT_MODE_VALUE);
+ static final int DEFAULT_DATANODES =
+ Integer.parseInt(DEFAULT_DATANODES_VALUE);
+ static final String DEFAULT_HOST = "127.0.0.1";
+ static final String DEFAULT_BIND_HOST = "0.0.0.0";
+ static final int DEFAULT_PORT = Integer.parseInt(DEFAULT_PORT_VALUE);
+ static final boolean DEFAULT_S3G_ENABLED =
+ Boolean.parseBoolean(DEFAULT_S3G_ENABLED_VALUE);
+ static final boolean DEFAULT_EPHEMERAL =
+ Boolean.parseBoolean(DEFAULT_EPHEMERAL_VALUE);
+ static final Duration DEFAULT_STARTUP_TIMEOUT =
+ Duration.parse(DEFAULT_STARTUP_TIMEOUT_VALUE);
+ static final String DEFAULT_S3_ACCESS_KEY = "admin";
+ static final String DEFAULT_S3_SECRET_KEY = "admin123";
+ static final String DEFAULT_S3_REGION = "us-east-1";
+
+ private final Path dataDir;
+ private final FormatMode formatMode;
+ private final int datanodes;
+ private final String host;
+ private final String bindHost;
+ private final int scmPort;
+ private final int omPort;
+ private final int s3gPort;
+ private final boolean s3gEnabled;
+ private final boolean ephemeral;
+ private final Duration startupTimeout;
+ private final String s3AccessKey;
+ private final String s3SecretKey;
+ private final String s3Region;
+
+ private LocalOzoneClusterConfig(Builder builder) {
+ dataDir = Objects.requireNonNull(builder.dataDir, "dataDir")
+ .toAbsolutePath()
+ .normalize();
+ formatMode = Objects.requireNonNull(builder.formatMode, "formatMode");
+ datanodes = builder.datanodes;
+ host = Objects.requireNonNull(builder.host, "host");
+ bindHost = Objects.requireNonNull(builder.bindHost, "bindHost");
+ scmPort = builder.scmPort;
+ omPort = builder.omPort;
+ s3gPort = builder.s3gPort;
+ s3gEnabled = builder.s3gEnabled;
+ ephemeral = builder.ephemeral;
+ startupTimeout = Objects.requireNonNull(builder.startupTimeout,
+ "startupTimeout");
+ s3AccessKey = Objects.requireNonNull(builder.s3AccessKey, "s3AccessKey");
+ s3SecretKey = Objects.requireNonNull(builder.s3SecretKey, "s3SecretKey");
+ s3Region = Objects.requireNonNull(builder.s3Region, "s3Region");
+ }
+
+ public Path getDataDir() {
+ return dataDir;
+ }
+
+ public FormatMode getFormatMode() {
+ return formatMode;
+ }
+
+ public int getDatanodes() {
+ return datanodes;
+ }
+
+ public String getHost() {
+ return host;
+ }
+
+ public String getBindHost() {
+ return bindHost;
+ }
+
+ /**
+ * Returns the SCM client RPC port. Port {@code 0} asks the runtime to choose
+ * an available local port.
+ */
+ public int getScmPort() {
+ return scmPort;
+ }
+
+ /**
+ * Returns the OM RPC port. Port {@code 0} asks the runtime to choose
+ * an available local port.
+ */
+ public int getOmPort() {
+ return omPort;
+ }
+
+ /**
+ * Returns the S3 Gateway HTTP port. Port {@code 0} asks the runtime to
+ * choose an available local port.
+ */
+ public int getS3gPort() {
+ return s3gPort;
+ }
+
+ /**
+ * Returns whether the local runtime should include S3 Gateway.
+ */
+ public boolean isS3gEnabled() {
+ return s3gEnabled;
+ }
+
+ /**
+ * Returns whether the local runtime should remove its data directory when it
+ * shuts down.
+ */
+ public boolean isEphemeral() {
+ return ephemeral;
+ }
+
+ /**
+ * Returns how long the launcher should wait for local services to become
+ * ready before failing startup.
+ */
+ public Duration getStartupTimeout() {
+ return startupTimeout;
+ }
+
+ /**
+ * Returns the suggested local-only S3 access key printed for client setup.
+ */
+ public String getS3AccessKey() {
+ return s3AccessKey;
+ }
+
+ /**
+ * Returns the suggested local-only S3 secret key printed for client setup.
+ */
+ public String getS3SecretKey() {
+ return s3SecretKey;
+ }
+
+ /**
+ * Returns the suggested local-only S3 region printed for client setup.
+ */
+ public String getS3Region() {
+ return s3Region;
+ }
+
+ public static Builder builder() {
+ return new Builder(DEFAULT_DATA_DIR);
+ }
+
+ public static Builder builder(Path dataDir) {
+ return new Builder(dataDir);
+ }
+
+ /**
+ * Storage initialization mode for the local runtime.
+ */
+ public enum FormatMode {
+ /**
+ * Initialize storage only when local metadata is missing or unformatted.
+ * Existing local data is reused.
+ */
+ IF_NEEDED,
+
+ /**
+ * Always format local storage before startup. Existing local data may be
+ * discarded.
+ */
+ ALWAYS,
+
+ /**
+ * Never format local storage. Startup should fail later if required
storage
+ * is not already initialized.
+ */
+ NEVER;
+
+ public static FormatMode fromString(String value) {
+ if (value == null) {
+ throw new IllegalArgumentException("Format mode must not be null.");
+ }
+ String normalized = value.trim().toUpperCase(Locale.ROOT)
+ .replace('-', '_');
+ return valueOf(normalized);
+ }
+ }
+
+ /**
+ * Builder for {@link LocalOzoneClusterConfig}.
+ */
+ public static final class Builder {
+
+ private final Path dataDir;
+ private FormatMode formatMode = DEFAULT_FORMAT_MODE;
+ private int datanodes = DEFAULT_DATANODES;
+ private String host = DEFAULT_HOST;
+ private String bindHost = DEFAULT_BIND_HOST;
+ private int scmPort = DEFAULT_PORT;
+ private int omPort = DEFAULT_PORT;
+ private int s3gPort = DEFAULT_PORT;
+ private boolean s3gEnabled = DEFAULT_S3G_ENABLED;
+ private boolean ephemeral = DEFAULT_EPHEMERAL;
+ private Duration startupTimeout = DEFAULT_STARTUP_TIMEOUT;
+ private String s3AccessKey = DEFAULT_S3_ACCESS_KEY;
+ private String s3SecretKey = DEFAULT_S3_SECRET_KEY;
+ private String s3Region = DEFAULT_S3_REGION;
+
+ private Builder(Path dataDir) {
+ this.dataDir = dataDir;
+ }
+
+ public Builder setFormatMode(FormatMode value) {
+ formatMode = value;
+ return this;
+ }
+
+ public Builder setDatanodes(int value) {
+ datanodes = value;
+ return this;
+ }
+
+ public Builder setHost(String value) {
+ host = value;
+ return this;
+ }
+
+ public Builder setBindHost(String value) {
+ bindHost = value;
+ return this;
+ }
+
+ public Builder setScmPort(int value) {
+ scmPort = value;
+ return this;
+ }
+
+ public Builder setOmPort(int value) {
+ omPort = value;
+ return this;
+ }
+
+ public Builder setS3gPort(int value) {
+ s3gPort = value;
+ return this;
+ }
+
+ public Builder setS3gEnabled(boolean value) {
+ s3gEnabled = value;
+ return this;
+ }
+
+ public Builder setEphemeral(boolean value) {
+ ephemeral = value;
+ return this;
+ }
+
+ public Builder setStartupTimeout(Duration value) {
+ startupTimeout = value;
+ return this;
+ }
+
+ public Builder setS3AccessKey(String value) {
+ s3AccessKey = value;
+ return this;
+ }
+
+ public Builder setS3SecretKey(String value) {
+ s3SecretKey = value;
+ return this;
+ }
+
+ public Builder setS3Region(String value) {
+ s3Region = value;
+ return this;
+ }
+
+ public LocalOzoneClusterConfig build() {
+ return new LocalOzoneClusterConfig(this);
+ }
+ }
+}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
index 54a08e476ff..3ed93904417 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneRuntime.java
@@ -18,22 +18,64 @@
package org.apache.hadoop.ozone.local;
/**
- * Runtime contract for local single-node Ozone commands.
+ * Runtime contract for local Ozone cluster commands.
*/
public interface LocalOzoneRuntime extends AutoCloseable {
+ /**
+ * Starts the local Ozone runtime.
+ *
+ * <p>Port and endpoint accessors return usable values after this method
+ * completes successfully.</p>
+ *
+ * @throws Exception if the local runtime cannot be started
+ */
void start() throws Exception;
+ /**
+ * Returns the host name or address shown to users for connecting to this
+ * local runtime.
+ *
+ * <p>This is a user-facing host and may differ from the bind host used by
+ * individual services.</p>
+ *
+ * @return user-facing host name or address
+ */
String getDisplayHost();
+ /**
+ * Returns the SCM client port for this local runtime.
+ *
+ * @return SCM port
+ */
int getScmPort();
+ /**
+ * Returns the OM client port for this local runtime.
+ *
+ * @return OM port
+ */
int getOmPort();
+ /**
+ * Returns the S3 Gateway HTTP port for this local runtime.
+ *
+ * @return S3 Gateway port
+ */
int getS3gPort();
+ /**
+ * Returns the full S3 Gateway endpoint shown to users.
+ *
+ * @return S3 Gateway endpoint, including scheme, host, and port
+ */
String getS3Endpoint();
+ /**
+ * Stops the local runtime and releases resources created during startup.
+ *
+ * @throws Exception if shutdown fails
+ */
@Override
void close() throws Exception;
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
index bd19b9f5dbc..6d1d4d1394f 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
@@ -17,17 +17,26 @@
package org.apache.hadoop.ozone.local;
+import java.nio.file.Path;
+import java.time.Duration;
+import java.time.format.DateTimeParseException;
import java.util.concurrent.Callable;
+import java.util.concurrent.TimeUnit;
+import org.apache.hadoop.hdds.cli.AbstractSubcommand;
import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.cli.HddsVersionProvider;
+import org.apache.hadoop.hdds.conf.TimeDurationUtil;
+import picocli.CommandLine;
import picocli.CommandLine.Command;
+import picocli.CommandLine.ITypeConverter;
+import picocli.CommandLine.Option;
/**
- * Internal CLI entry point for local single-node Ozone commands.
+ * Internal CLI entry point for local Ozone cluster commands.
*/
@Command(name = "ozone local",
hidden = true,
- description = "Internal commands for local single-node Ozone",
+ description = "Internal commands for local Ozone cluster runtime",
versionProvider = HddsVersionProvider.class,
mixinStandardHelpOptions = true,
subcommands = {
@@ -35,18 +44,235 @@
})
public class OzoneLocal extends GenericCli {
+ static final String ENV_DATA_DIR = "OZONE_LOCAL_DATA_DIR";
+ static final String ENV_FORMAT = "OZONE_LOCAL_FORMAT";
+ static final String ENV_DATANODES = "OZONE_LOCAL_DATANODES";
+ static final String ENV_HOST = "OZONE_LOCAL_HOST";
+ static final String ENV_BIND_HOST = "OZONE_LOCAL_BIND_HOST";
+ static final String ENV_SCM_PORT = "OZONE_LOCAL_SCM_PORT";
+ static final String ENV_OM_PORT = "OZONE_LOCAL_OM_PORT";
+ static final String ENV_S3G_ENABLED = "OZONE_LOCAL_S3G_ENABLED";
+ static final String ENV_S3G_PORT = "OZONE_LOCAL_S3G_PORT";
+ static final String ENV_EPHEMERAL = "OZONE_LOCAL_EPHEMERAL";
+ static final String ENV_STARTUP_TIMEOUT = "OZONE_LOCAL_STARTUP_TIMEOUT";
+ static final String ENV_S3_ACCESS_KEY = "OZONE_LOCAL_S3_ACCESS_KEY";
+ static final String ENV_S3_SECRET_KEY = "OZONE_LOCAL_S3_SECRET_KEY";
+ static final String ENV_S3_REGION = "OZONE_LOCAL_S3_REGION";
+
+ private static final String DEFAULT_DATA_DIR_VALUE = "${env:" + ENV_DATA_DIR
+ + ":-" + LocalOzoneClusterConfig.DEFAULT_DATA_DIR_VALUE + "}";
+ private static final String DEFAULT_FORMAT_VALUE = "${env:" + ENV_FORMAT
+ + ":-" + LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE_VALUE + "}";
+ private static final String DEFAULT_DATANODES_VALUE = "${env:"
+ + ENV_DATANODES + ":-" + LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE
+ + "}";
+ private static final String DEFAULT_HOST_VALUE = "${env:" + ENV_HOST
+ + ":-" + LocalOzoneClusterConfig.DEFAULT_HOST + "}";
+ private static final String DEFAULT_BIND_HOST_VALUE = "${env:"
+ + ENV_BIND_HOST + ":-" + LocalOzoneClusterConfig.DEFAULT_BIND_HOST + "}";
+ private static final String DEFAULT_SCM_PORT_VALUE = "${env:"
+ + ENV_SCM_PORT + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE
+ + "}";
+ private static final String DEFAULT_OM_PORT_VALUE = "${env:" + ENV_OM_PORT
+ + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE + "}";
+ private static final String DEFAULT_S3G_ENABLED_VALUE = "${env:"
+ + ENV_S3G_ENABLED + ":-"
+ + LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE + "}";
+ private static final String DEFAULT_S3G_PORT_VALUE = "${env:" + ENV_S3G_PORT
+ + ":-" + LocalOzoneClusterConfig.DEFAULT_PORT_VALUE + "}";
+ private static final String DEFAULT_EPHEMERAL_VALUE = "${env:"
+ + ENV_EPHEMERAL + ":-"
+ + LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE + "}";
+ private static final String DEFAULT_STARTUP_TIMEOUT_VALUE = "${env:"
+ + ENV_STARTUP_TIMEOUT + ":-"
+ + LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE + "}";
+ private static final String DEFAULT_S3_ACCESS_KEY_VALUE = "${env:"
+ + ENV_S3_ACCESS_KEY + ":-"
+ + LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY + "}";
+ private static final String DEFAULT_S3_SECRET_KEY_VALUE = "${env:"
+ + ENV_S3_SECRET_KEY + ":-"
+ + LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY + "}";
+ private static final String DEFAULT_S3_REGION_VALUE = "${env:"
+ + ENV_S3_REGION + ":-" + LocalOzoneClusterConfig.DEFAULT_S3_REGION + "}";
+
+ public OzoneLocal() {
+ super();
+ }
+
+ OzoneLocal(CommandLine.IFactory factory) {
+ super(factory);
+ }
+
public static void main(String[] args) {
new OzoneLocal().run(args);
}
@Command(name = "run",
hidden = true,
- description = "Internal placeholder for a local Ozone runtime")
- static class RunCommand implements Callable<Void> {
+ description = "Resolve configuration for local Ozone runtime startup")
+ static class RunCommand extends AbstractSubcommand implements Callable<Void>
{
+
+ @Option(names = "--data-dir",
+ defaultValue = DEFAULT_DATA_DIR_VALUE,
+ description = "Persistent data directory for the local cluster")
+ private Path dataDir;
+
+ @Option(names = "--format",
+ converter = FormatModeConverter.class,
+ defaultValue = DEFAULT_FORMAT_VALUE,
+ description = "Storage init mode: if-needed, always, never")
+ private LocalOzoneClusterConfig.FormatMode formatMode;
+
+ @Option(names = "--datanodes",
+ defaultValue = DEFAULT_DATANODES_VALUE,
+ description = "Number of datanodes to start")
+ private int datanodes;
+
+ @Option(names = "--host",
+ defaultValue = DEFAULT_HOST_VALUE,
+ description = "Advertised host to write into local service addresses")
+ private String host;
+
+ @Option(names = "--bind-host",
+ defaultValue = DEFAULT_BIND_HOST_VALUE,
+ description = "Bind host for HTTP and RPC listeners")
+ private String bindHost;
+
+ @Option(names = "--scm-port",
+ defaultValue = DEFAULT_SCM_PORT_VALUE,
+ description = "SCM client RPC port (0 means auto-allocate)")
+ private int scmPort;
+
+ @Option(names = "--om-port",
+ defaultValue = DEFAULT_OM_PORT_VALUE,
+ description = "OM RPC port (0 means auto-allocate)")
+ private int omPort;
+
+ @Option(names = "--s3g-port",
+ defaultValue = DEFAULT_S3G_PORT_VALUE,
+ description = "S3 Gateway HTTP port (0 means auto-allocate)")
+ private int s3gPort;
+
+ @Option(names = "--s3g",
+ negatable = true,
+ defaultValue = DEFAULT_S3G_ENABLED_VALUE,
+ fallbackValue = "true",
+ description = "Enable S3 Gateway")
+ private boolean s3gEnabled;
+
+ @Option(names = "--ephemeral",
+ negatable = true,
+ defaultValue = DEFAULT_EPHEMERAL_VALUE,
+ fallbackValue = "true",
+ description = "Delete the data directory on shutdown")
+ private boolean ephemeral;
+
+ @Option(names = "--startup-timeout",
+ converter = DurationConverter.class,
+ defaultValue = DEFAULT_STARTUP_TIMEOUT_VALUE,
+ description = "How long to wait for the local cluster to become ready")
+ private Duration startupTimeout;
+
+ @Option(names = "--s3-access-key",
+ defaultValue = DEFAULT_S3_ACCESS_KEY_VALUE,
+ description = "Suggested local AWS access key to print on startup")
+ private String s3AccessKey;
+
+ @Option(names = "--s3-secret-key",
+ defaultValue = DEFAULT_S3_SECRET_KEY_VALUE,
+ description = "Suggested local AWS secret key to print on startup")
+ private String s3SecretKey;
+
+ @Option(names = "--s3-region",
+ defaultValue = DEFAULT_S3_REGION_VALUE,
+ description = "Suggested local AWS region to print on startup")
+ private String s3Region;
@Override
public Void call() {
+ resolveConfig();
return null;
}
+
+ LocalOzoneClusterConfig resolveConfig() {
+ if (datanodes < 1) {
+ throw new IllegalArgumentException(
+ "Datanode count for --datanodes must be at least 1.");
+ }
+ validatePort(scmPort, "--scm-port");
+ validatePort(omPort, "--om-port");
+ validatePort(s3gPort, "--s3g-port");
+ validateStartupTimeout();
+
+ return LocalOzoneClusterConfig.builder(dataDir)
+ .setFormatMode(formatMode)
+ .setDatanodes(datanodes)
+ .setHost(host)
+ .setBindHost(bindHost)
+ .setScmPort(scmPort)
+ .setOmPort(omPort)
+ .setS3gPort(s3gPort)
+ .setS3gEnabled(s3gEnabled)
+ .setEphemeral(ephemeral)
+ .setStartupTimeout(startupTimeout)
+ .setS3AccessKey(s3AccessKey)
+ .setS3SecretKey(s3SecretKey)
+ .setS3Region(s3Region)
+ .build();
+ }
+
+ private void validateStartupTimeout() {
+ if (startupTimeout.isZero() || startupTimeout.isNegative()) {
+ throw new IllegalArgumentException(
+ "Startup timeout for --startup-timeout must be greater than
zero.");
+ }
+ }
+
+ private static void validatePort(int value, String source) {
+ if (value < 0 || value > 65_535) {
+ throw new IllegalArgumentException("Port value for " + source
+ + " must be between 0 and 65535.");
+ }
+ }
+
+ private static final class FormatModeConverter
+ implements ITypeConverter<LocalOzoneClusterConfig.FormatMode> {
+
+ @Override
+ public LocalOzoneClusterConfig.FormatMode convert(String value) {
+ try {
+ return LocalOzoneClusterConfig.FormatMode.fromString(value);
+ } catch (IllegalArgumentException ex) {
+ throw new CommandLine.TypeConversionException(
+ "Expected one of: if-needed, always, never.");
+ }
+ }
+ }
+
+ private static final class DurationConverter
+ implements ITypeConverter<Duration> {
+
+ @Override
+ public Duration convert(String value) {
+ try {
+ return Duration.parse(value.trim());
+ } catch (DateTimeParseException ignored) {
+ return parseHadoopStyleDuration(value);
+ }
+ }
+
+ private static Duration parseHadoopStyleDuration(String value) {
+ try {
+ return TimeDurationUtil.getDuration("--startup-timeout", value,
+ TimeUnit.MILLISECONDS);
+ } catch (RuntimeException ex) {
+ throw new CommandLine.TypeConversionException(durationMessage());
+ }
+ }
+
+ private static String durationMessage() {
+ return "Use ISO-8601 like PT2M or Hadoop-style values like 120s.";
+ }
+ }
}
}
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java
index ca9e55fd6ce..aa02e7fa552 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/package-info.java
@@ -16,6 +16,6 @@
*/
/**
- * Internal local single-node Ozone runtime support.
+ * Internal local Ozone cluster runtime support.
*/
package org.apache.hadoop.ozone.local;
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java
new file mode 100644
index 00000000000..7e7f8248024
--- /dev/null
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneClusterConfig.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.ozone.local;
+
+import static org.junit.jupiter.api.Assertions.assertEquals;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.nio.file.Paths;
+import java.time.Duration;
+import org.junit.jupiter.api.Test;
+
+/**
+ * Tests for {@link LocalOzoneClusterConfig}.
+ */
+class TestLocalOzoneClusterConfig {
+
+ @Test
+ void builderProvidesLocalClusterDefaults() {
+ LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder().build();
+
+ assertEquals(LocalOzoneClusterConfig.DEFAULT_DATA_DIR,
+ config.getDataDir());
+ assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED,
+ config.getFormatMode());
+ assertEquals(1, config.getDatanodes());
+ assertEquals("127.0.0.1", config.getHost());
+ assertEquals("0.0.0.0", config.getBindHost());
+ assertEquals(0, config.getScmPort());
+ assertEquals(0, config.getOmPort());
+ assertEquals(0, config.getS3gPort());
+ assertTrue(config.isS3gEnabled());
+ assertFalse(config.isEphemeral());
+ assertEquals(Duration.ofMinutes(2), config.getStartupTimeout());
+ assertEquals("admin", config.getS3AccessKey());
+ assertEquals("admin123", config.getS3SecretKey());
+ assertEquals("us-east-1", config.getS3Region());
+ }
+
+ @Test
+ void typedDefaultsMatchSharedFallbackValues() {
+ assertEquals(LocalOzoneClusterConfig.FormatMode.fromString(
+ LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE_VALUE),
+ LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE);
+ assertEquals(Integer.parseInt(
+ LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE),
+ LocalOzoneClusterConfig.DEFAULT_DATANODES);
+ assertEquals(Integer.parseInt(LocalOzoneClusterConfig.DEFAULT_PORT_VALUE),
+ LocalOzoneClusterConfig.DEFAULT_PORT);
+ assertEquals(Boolean.parseBoolean(
+ LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE),
+ LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED);
+ assertEquals(Boolean.parseBoolean(
+ LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE),
+ LocalOzoneClusterConfig.DEFAULT_EPHEMERAL);
+ assertEquals(Duration.parse(
+ LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE),
+ LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT);
+ }
+
+ @Test
+ void builderAcceptsExplicitOverrides() {
+ LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(
+ Paths.get("target", "custom-local-ozone"))
+ .setFormatMode(LocalOzoneClusterConfig.FormatMode.ALWAYS)
+ .setDatanodes(3)
+ .setHost("localhost")
+ .setBindHost("127.0.0.1")
+ .setScmPort(9860)
+ .setOmPort(9862)
+ .setS3gPort(9878)
+ .setS3gEnabled(false)
+ .setEphemeral(true)
+ .setStartupTimeout(Duration.ofSeconds(45))
+ .setS3AccessKey("dev")
+ .setS3SecretKey("secret")
+ .setS3Region("ap-south-1")
+ .build();
+
+ assertEquals(Paths.get("target", "custom-local-ozone")
+ .toAbsolutePath().normalize(), config.getDataDir());
+ assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS,
+ config.getFormatMode());
+ assertEquals(3, config.getDatanodes());
+ assertEquals("localhost", config.getHost());
+ assertEquals("127.0.0.1", config.getBindHost());
+ assertEquals(9860, config.getScmPort());
+ assertEquals(9862, config.getOmPort());
+ assertEquals(9878, config.getS3gPort());
+ assertFalse(config.isS3gEnabled());
+ assertTrue(config.isEphemeral());
+ assertEquals(Duration.ofSeconds(45), config.getStartupTimeout());
+ assertEquals("dev", config.getS3AccessKey());
+ assertEquals("secret", config.getS3SecretKey());
+ assertEquals("ap-south-1", config.getS3Region());
+ }
+
+ @Test
+ void formatModeParsesUserFacingValues() {
+ assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED,
+ LocalOzoneClusterConfig.FormatMode.fromString("if-needed"));
+ assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS,
+ LocalOzoneClusterConfig.FormatMode.fromString(" always "));
+ assertEquals(LocalOzoneClusterConfig.FormatMode.NEVER,
+ LocalOzoneClusterConfig.FormatMode.fromString("NEVER"));
+ }
+
+ @Test
+ void formatModeRejectsUnknownValues() {
+ assertThrows(IllegalArgumentException.class,
+ () -> LocalOzoneClusterConfig.FormatMode.fromString("sometimes"));
+ }
+
+ @Test
+ void formatModeRejectsNullValues() {
+ assertThrows(IllegalArgumentException.class,
+ () -> LocalOzoneClusterConfig.FormatMode.fromString(null));
+ }
+}
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
index ff44f0ee5f6..219b3ae5d54 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
@@ -21,14 +21,23 @@
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertThrows;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayOutputStream;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
+import java.lang.reflect.Field;
+import java.nio.file.Paths;
+import java.time.Duration;
import org.junit.jupiter.api.Test;
import picocli.CommandLine;
import picocli.CommandLine.Command;
+import picocli.CommandLine.IDefaultValueProvider;
+import picocli.CommandLine.Model.ArgSpec;
+import picocli.CommandLine.Model.OptionSpec;
+import picocli.CommandLine.Option;
+import picocli.CommandLine.ParameterException;
/**
* Tests for {@link OzoneLocal}.
@@ -54,14 +63,14 @@ void runCommandMetadataIsPresentAndHidden() {
}
@Test
- void genericCliRegistersRunPlaceholder() {
+ void genericCliRegistersRunCommand() {
OzoneLocal local = new OzoneLocal();
assertTrue(local.getCmd().getSubcommands().containsKey("run"));
}
@Test
- void rootHelpHidesRunPlaceholder() throws Exception {
+ void rootHelpHidesRunCommand() throws Exception {
OzoneLocal local = new OzoneLocal();
CommandLine commandLine = local.getCmd();
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -76,12 +85,13 @@ void rootHelpHidesRunPlaceholder() throws Exception {
String help = out.toString(UTF_8.name());
assertEquals(0, exitCode);
assertTrue(help.contains("Usage: ozone local"));
- assertFalse(help.contains("run"));
+ assertFalse(help.matches("(?s).*\\R\\s+run\\b.*"), help);
assertEquals("", err.toString(UTF_8.name()));
}
@Test
- void runCommandIsQuietNoOpPlaceholder() throws Exception {
+ void runCommandResolvesConfigurationQuietlyUntilRuntimeStartup()
+ throws Exception {
OzoneLocal local = new OzoneLocal();
CommandLine commandLine = local.getCmd();
ByteArrayOutputStream out = new ByteArrayOutputStream();
@@ -97,4 +107,259 @@ void runCommandIsQuietNoOpPlaceholder() throws Exception {
assertEquals("", out.toString(UTF_8.name()));
assertEquals("", err.toString(UTF_8.name()));
}
+
+ @Test
+ void runCommandOptionsUseEnvironmentDefaults() throws Exception {
+ assertEnvDefault("dataDir", OzoneLocal.ENV_DATA_DIR,
+ LocalOzoneClusterConfig.DEFAULT_DATA_DIR_VALUE);
+ assertEnvDefault("formatMode", OzoneLocal.ENV_FORMAT,
+ LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE_VALUE);
+ assertEnvDefault("datanodes", OzoneLocal.ENV_DATANODES,
+ LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE);
+ assertEnvDefault("host", OzoneLocal.ENV_HOST,
+ LocalOzoneClusterConfig.DEFAULT_HOST);
+ assertEnvDefault("bindHost", OzoneLocal.ENV_BIND_HOST,
+ LocalOzoneClusterConfig.DEFAULT_BIND_HOST);
+ assertEnvDefault("scmPort", OzoneLocal.ENV_SCM_PORT,
+ LocalOzoneClusterConfig.DEFAULT_PORT_VALUE);
+ assertEnvDefault("omPort", OzoneLocal.ENV_OM_PORT,
+ LocalOzoneClusterConfig.DEFAULT_PORT_VALUE);
+ assertEnvDefault("s3gEnabled", OzoneLocal.ENV_S3G_ENABLED,
+ LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE);
+ assertEnvDefault("s3gPort", OzoneLocal.ENV_S3G_PORT,
+ LocalOzoneClusterConfig.DEFAULT_PORT_VALUE);
+ assertEnvDefault("ephemeral", OzoneLocal.ENV_EPHEMERAL,
+ LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE);
+ assertEnvDefault("startupTimeout", OzoneLocal.ENV_STARTUP_TIMEOUT,
+ LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE);
+ assertEnvDefault("s3AccessKey", OzoneLocal.ENV_S3_ACCESS_KEY,
+ LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY);
+ assertEnvDefault("s3SecretKey", OzoneLocal.ENV_S3_SECRET_KEY,
+ LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY);
+ assertEnvDefault("s3Region", OzoneLocal.ENV_S3_REGION,
+ LocalOzoneClusterConfig.DEFAULT_S3_REGION);
+ }
+
+ @Test
+ void resolveConfigUsesPicocliDefaults() {
+ LocalOzoneClusterConfig config = resolveWithFallbackDefaults();
+
+ assertEquals(LocalOzoneClusterConfig.DEFAULT_DATA_DIR,
+ config.getDataDir());
+ assertEquals(LocalOzoneClusterConfig.FormatMode.IF_NEEDED,
+ config.getFormatMode());
+ assertEquals(1, config.getDatanodes());
+ assertEquals("127.0.0.1", config.getHost());
+ assertEquals("0.0.0.0", config.getBindHost());
+ assertEquals(0, config.getScmPort());
+ assertEquals(0, config.getOmPort());
+ assertEquals(0, config.getS3gPort());
+ assertTrue(config.isS3gEnabled());
+ assertFalse(config.isEphemeral());
+ assertEquals(Duration.ofMinutes(2), config.getStartupTimeout());
+ assertEquals("admin", config.getS3AccessKey());
+ assertEquals("admin123", config.getS3SecretKey());
+ assertEquals("us-east-1", config.getS3Region());
+ }
+
+ @Test
+ void resolveConfigUsesCliOverrides() {
+ LocalOzoneClusterConfig config = resolve(
+ "--data-dir", "target/cli-local",
+ "--format", "always",
+ "--datanodes", "3",
+ "--host", "cli-host",
+ "--bind-host", "127.0.0.1",
+ "--scm-port", "200",
+ "--om-port", "201",
+ "--s3g-port", "202",
+ "--no-s3g",
+ "--ephemeral",
+ "--startup-timeout", "45s",
+ "--s3-access-key", "cli-access",
+ "--s3-secret-key", "cli-secret",
+ "--s3-region", "cli-region");
+
+ assertEquals(Paths.get("target/cli-local").toAbsolutePath().normalize(),
+ config.getDataDir());
+ assertEquals(LocalOzoneClusterConfig.FormatMode.ALWAYS,
+ config.getFormatMode());
+ assertEquals(3, config.getDatanodes());
+ assertEquals("cli-host", config.getHost());
+ assertEquals("127.0.0.1", config.getBindHost());
+ assertEquals(200, config.getScmPort());
+ assertEquals(201, config.getOmPort());
+ assertEquals(202, config.getS3gPort());
+ assertFalse(config.isS3gEnabled());
+ assertTrue(config.isEphemeral());
+ assertEquals(Duration.ofSeconds(45), config.getStartupTimeout());
+ assertEquals("cli-access", config.getS3AccessKey());
+ assertEquals("cli-secret", config.getS3SecretKey());
+ assertEquals("cli-region", config.getS3Region());
+ }
+
+ @Test
+ void resolveConfigParsesIsoStartupTimeout() {
+ LocalOzoneClusterConfig config = resolve("--startup-timeout", "PT45S");
+
+ assertEquals(Duration.ofSeconds(45), config.getStartupTimeout());
+ }
+
+ @Test
+ void resolveConfigAllowsS3gAndEphemeralToBeNegated() {
+ LocalOzoneClusterConfig config = resolve("--s3g", "--no-ephemeral");
+
+ assertTrue(config.isS3gEnabled());
+ assertFalse(config.isEphemeral());
+ }
+
+ @Test
+ void resolveConfigRejectsInvalidFormat() {
+ assertParseError("--format", "sometimes", "--format");
+ }
+
+ @Test
+ void resolveConfigRejectsInvalidInteger() {
+ assertParseError("--datanodes", "two", "--datanodes");
+ }
+
+ @Test
+ void resolveConfigRejectsInvalidPort() {
+ assertConfigError("--scm-port", "65536", "--scm-port");
+ }
+
+ @Test
+ void resolveConfigRejectsDatanodeCountBelowOne() {
+ assertConfigError("--datanodes", "0", "--datanodes");
+ }
+
+ @Test
+ void resolveConfigRejectsInvalidDuration() {
+ assertParseError("--startup-timeout", "forever", "--startup-timeout");
+ }
+
+ @Test
+ void resolveConfigRejectsNonPositiveDuration() {
+ assertConfigError("--startup-timeout", "0s", "--startup-timeout");
+ }
+
+ @Test
+ void resolveConfigRejectsInvalidPath() {
+ assertParseError("--data-dir", "\0", "--data-dir");
+ }
+
+ @Test
+ void legacyWithoutS3gOptionIsNotAccepted() {
+ assertParseError("--without-s3g", "--without-s3g");
+ }
+
+ @Test
+ void genericCliErrorOutputIncludesOffendingConfigSource()
+ throws Exception {
+ OzoneLocal local = new OzoneLocal();
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ local.getCmd().setErr(new PrintWriter(new OutputStreamWriter(err, UTF_8),
+ true));
+
+ int exitCode = local.execute(new String[] {"run", "--datanodes", "0"});
+
+ assertEquals(-1, exitCode);
+ assertTrue(err.toString(UTF_8.name()).contains("--datanodes"));
+ }
+
+ private static LocalOzoneClusterConfig resolve(String... args) {
+ OzoneLocal.RunCommand command = new OzoneLocal.RunCommand();
+ new CommandLine(command).parseArgs(args);
+ return command.resolveConfig();
+ }
+
+ private static LocalOzoneClusterConfig resolveWithFallbackDefaults(
+ String... args) {
+ OzoneLocal.RunCommand command = new OzoneLocal.RunCommand();
+ new CommandLine(command)
+ .setDefaultValueProvider(new RunCommandFallbackDefaults())
+ .parseArgs(args);
+ return command.resolveConfig();
+ }
+
+ private static void assertConfigError(String option, String value,
+ String expectedMessage) {
+ OzoneLocal.RunCommand command = new OzoneLocal.RunCommand();
+ new CommandLine(command).parseArgs(option, value);
+
+ IllegalArgumentException error =
assertThrows(IllegalArgumentException.class,
+ command::resolveConfig);
+
+ assertTrue(error.getMessage().contains(expectedMessage),
+ error.getMessage());
+ }
+
+ private static void assertParseError(String option,
+ String expectedMessage) {
+ OzoneLocal.RunCommand command = new OzoneLocal.RunCommand();
+ ParameterException error = assertThrows(ParameterException.class,
+ () -> new CommandLine(command).parseArgs(option));
+
+ assertTrue(error.getMessage().contains(expectedMessage),
+ error.getMessage());
+ }
+
+ private static void assertParseError(String option, String value,
+ String expectedMessage) {
+ OzoneLocal.RunCommand command = new OzoneLocal.RunCommand();
+ ParameterException error = assertThrows(ParameterException.class,
+ () -> new CommandLine(command).parseArgs(option, value));
+
+ assertTrue(error.getMessage().contains(expectedMessage),
+ error.getMessage());
+ }
+
+ private static void assertEnvDefault(String fieldName,
+ String environmentVariable, String fallback) throws Exception {
+ Field field = OzoneLocal.RunCommand.class.getDeclaredField(fieldName);
+ String defaultValue = field.getAnnotation(Option.class).defaultValue();
+
+ assertEquals("${env:" + environmentVariable + ":-" + fallback + "}",
+ defaultValue);
+ }
+
+ private static final class RunCommandFallbackDefaults
+ implements IDefaultValueProvider {
+
+ @Override
+ public String defaultValue(ArgSpec argSpec) {
+ if (!(argSpec instanceof OptionSpec)) {
+ return null;
+ }
+ String option = ((OptionSpec) argSpec).longestName();
+ if ("--data-dir".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_DATA_DIR_VALUE;
+ } else if ("--format".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_FORMAT_MODE_VALUE;
+ } else if ("--datanodes".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_DATANODES_VALUE;
+ } else if ("--host".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_HOST;
+ } else if ("--bind-host".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_BIND_HOST;
+ } else if ("--scm-port".equals(option)
+ || "--om-port".equals(option)
+ || "--s3g-port".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_PORT_VALUE;
+ } else if ("--s3g".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_S3G_ENABLED_VALUE;
+ } else if ("--ephemeral".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_EPHEMERAL_VALUE;
+ } else if ("--startup-timeout".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_STARTUP_TIMEOUT_VALUE;
+ } else if ("--s3-access-key".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_S3_ACCESS_KEY;
+ } else if ("--s3-secret-key".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_S3_SECRET_KEY;
+ } else if ("--s3-region".equals(option)) {
+ return LocalOzoneClusterConfig.DEFAULT_S3_REGION;
+ }
+ return null;
+ }
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
