This is an automated email from the ASF dual-hosted git repository.
chungen0126 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 20e6a1c0f60 HDDS-15986. Report ozone local startup failures (#10934)
20e6a1c0f60 is described below
commit 20e6a1c0f6039248373efd3eb1d7f2afcf7f1535
Author: Chun-Hung Tseng <[email protected]>
AuthorDate: Tue Sep 15 21:42:32 2026 +0200
HDDS-15986. Report ozone local startup failures (#10934)
---
hadoop-ozone/tools/pom.xml | 4 +
.../hadoop/ozone/local/LocalOzoneCluster.java | 303 +++++++++++++++++----
.../org/apache/hadoop/ozone/local/OzoneLocal.java | 76 +++++-
.../hadoop/ozone/local/TestLocalOzoneCluster.java | 301 +++++++++++++++++++-
.../apache/hadoop/ozone/local/TestOzoneLocal.java | 126 ++++++++-
5 files changed, 747 insertions(+), 63 deletions(-)
diff --git a/hadoop-ozone/tools/pom.xml b/hadoop-ozone/tools/pom.xml
index 8855e408414..8429437e5ad 100644
--- a/hadoop-ozone/tools/pom.xml
+++ b/hadoop-ozone/tools/pom.xml
@@ -67,6 +67,10 @@
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-container-service</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>hdds-interface-admin</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.ozone</groupId>
<artifactId>hdds-server-framework</artifactId>
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java
index 89961e8243a..fdab4179bd8 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java
@@ -92,10 +92,14 @@
import java.util.UUID;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.TimeoutException;
+import java.util.function.Supplier;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import org.apache.hadoop.hdds.conf.TimeDurationUtil;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SafeModeRuleStatusProto;
import org.apache.hadoop.hdds.scm.proxy.SCMClientConfig;
import org.apache.hadoop.hdds.scm.server.SCMStorageConfig;
import org.apache.hadoop.hdds.scm.server.StorageContainerManager;
@@ -153,6 +157,7 @@ public final class LocalOzoneCluster implements
LocalOzoneRuntime {
private static final int LOCAL_RATIS_RPC_TIMEOUT_SECONDS = 1;
private static final long SCM_CLIENT_MAX_RETRY_TIMEOUT_MILLIS = 30_000;
private static final long READINESS_POLL_INTERVAL_MILLIS = 500;
+ private static final long READINESS_LOG_INTERVAL_NANOS =
TimeUnit.SECONDS.toNanos(5);
private static final int MAX_PORT = 65_535;
@@ -186,6 +191,11 @@ public final class LocalOzoneCluster implements
LocalOzoneRuntime {
// local ports, so an unbounded count would exhaust local ports; cap it.
static final int MAX_DATANODES = 20;
+ private static final Set<String> SHIPPED_DEFAULT_RESOURCES =
Collections.unmodifiableSet(
+ OzoneConfiguration.getConfigurationResourceFiles().stream()
+ .filter(resource -> resource.endsWith("-default.xml"))
+ .collect(Collectors.toSet()));
+
private static final String[] NO_ARGS = new String[0];
private final LocalOzoneClusterConfig config;
@@ -239,8 +249,10 @@ PreparedConfiguration prepareConfiguration() throws
IOException {
return preparedConfiguration;
}
- prepareStorageLayout();
-
+ // Every rejection of user input, validateStorageLayout() and the port
checks included, runs before
+ // prepareStorageLayout(), which deletes the data dir in format mode
ALWAYS.
+ requireSupportedDatanodeCount();
+ validateStorageLayout();
OzoneConfiguration conf = new OzoneConfiguration(seedConfiguration);
configureLocalDefaults(conf);
@@ -251,6 +263,8 @@ PreparedConfiguration prepareConfiguration() throws
IOException {
List<OzoneConfiguration> datanodeConfigurations =
configureDatanodes(conf, persistedPorts, portAllocator);
+ prepareStorageLayout();
+ createServiceDirectories();
persistedPorts.store();
preparedConfiguration = new PreparedConfiguration(conf, scmPort, omPort,
datanodeConfigurations);
@@ -314,8 +328,9 @@ public void close() throws IOException {
private void stopServices() {
try {
- // Shutdown is best-effort so one failed service cannot leak the others.
- IOUtils.closeQuietly(this::stopDatanodes, this::stopOm, this::stopScm);
+ // Logged rather than closeQuietly: stopServices() is also the start()
rollback, where a silent
+ // failure hides leaked threads and ports.
+ IOUtils.close(LOG, this::stopDatanodes, this::stopOm, this::stopScm);
} finally {
restoreSameJvmMetricsMode();
}
@@ -331,7 +346,7 @@ private void stopDatanodes() {
});
}
datanodes.clear();
- IOUtils.closeQuietly(stoppers);
+ IOUtils.close(LOG, stoppers);
}
private void stopOm() {
@@ -351,28 +366,25 @@ private void stopScm() {
}
}
- private void configureLocalDefaults(OzoneConfiguration conf) {
+ private void configureLocalDefaults(OzoneConfiguration conf) throws
IOException {
conf.set(OZONE_METADATA_DIRS, metadataDir().toString());
// SCM and OM share one configuration and JVM, so the Jetty base dir is a
// single service-neutral location; Jetty keeps per-context temp dirs.
conf.setIfUnset(OZONE_HTTP_BASEDIR, metadataDir() + SERVER_DIR);
- conf.set(OZONE_REPLICATION, ReplicationFactor.ONE.name());
- conf.set(OZONE_REPLICATION_TYPE, ReplicationType.STAND_ALONE.name());
- conf.set(OZONE_SERVER_DEFAULT_REPLICATION_KEY,
ReplicationFactor.ONE.name());
- conf.set(OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY,
+ setLocalOverrideReplication(conf, OZONE_REPLICATION,
ReplicationFactor.ONE);
+ setLocalOverride(conf, OZONE_REPLICATION_TYPE,
ReplicationType.STAND_ALONE.name());
+ setLocalOverrideReplication(conf, OZONE_SERVER_DEFAULT_REPLICATION_KEY,
ReplicationFactor.ONE);
+ setLocalOverride(conf, OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY,
ReplicationType.STAND_ALONE.name());
- conf.setBoolean(HDDS_CONTAINER_RATIS_ENABLED_KEY, false);
+ setLocalOverride(conf, HDDS_CONTAINER_RATIS_ENABLED_KEY, false);
// A single-node local cluster can heartbeat aggressively; this speeds
- // datanode registration and safe-mode exit. Use set(), not setIfUnset():
- // ozone-default.xml supplies the 30s default that would otherwise defeat
- // the override.
- conf.set(HDDS_HEARTBEAT_INTERVAL, "1s");
- conf.setBoolean(HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
- conf.setInt(HDDS_SCM_SAFEMODE_MIN_DATANODE,
- Math.max(1, config.getDatanodes()));
- conf.setTimeDuration(OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
- LOCAL_RATIS_RPC_TIMEOUT_SECONDS, TimeUnit.SECONDS);
- conf.set(HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "3s");
+ // datanode registration and safe-mode exit.
+ setLocalOverrideDuration(conf, HDDS_HEARTBEAT_INTERVAL, "1s");
+ setLocalOverride(conf, HDDS_SCM_SAFEMODE_PIPELINE_CREATION, false);
+ setLocalOverride(conf, HDDS_SCM_SAFEMODE_MIN_DATANODE,
config.getDatanodes());
+ setLocalOverrideDuration(conf, OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY,
+ LOCAL_RATIS_RPC_TIMEOUT_SECONDS + "s");
+ setLocalOverrideDuration(conf, HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
"3s");
conf.setIfUnset(OZONE_SCM_HA_RATIS_SERVER_RPC_FIRST_ELECTION_TIMEOUT,
"1s");
@@ -381,6 +393,129 @@ private void configureLocalDefaults(OzoneConfiguration
conf) {
conf.setFromObject(scmClientConfig);
}
+ /**
+ * Sets a value the local runtime requires and rejects a user-configured
conflict. Set rather than
+ * {@code setIfUnset}: ozone-default.xml would otherwise win. The typed
overloads compare through the
+ * accessor the services read the key with, so an equivalent spelling is not
a conflict.
+ */
+ private void setLocalOverride(OzoneConfiguration conf, String key, String
value)
+ throws IOException {
+ // Configuration#unset() keeps the key's source, so a null value is
unconfigured, not a conflict.
+ // Trimmed because Hadoop keeps XML whitespace and the value is rewritten
below anyway.
+ String configured = conf.get(key);
+ if (configured != null && !value.equals(configured.trim())) {
+ rejectUserConfigured(conf, key, value);
+ }
+ conf.set(key, value);
+ }
+
+ private void setLocalOverride(OzoneConfiguration conf, String key, boolean
value)
+ throws IOException {
+ // The default is the negation so a value getBoolean() cannot parse counts
as a conflict.
+ String configured = conf.get(key);
+ if (configured != null && conf.getBoolean(key, !value) != value) {
+ rejectUserConfigured(conf, key, String.valueOf(value));
+ }
+ conf.setBoolean(key, value);
+ }
+
+ private void setLocalOverride(OzoneConfiguration conf, String key, int value)
+ throws IOException {
+ String configured = conf.get(key);
+ if (configured != null && !matchesInt(conf, key, value)) {
+ rejectUserConfigured(conf, key, String.valueOf(value));
+ }
+ conf.setInt(key, value);
+ }
+
+ private void setLocalOverrideDuration(OzoneConfiguration conf, String key,
String value)
+ throws IOException {
+ long requiredMillis = TimeDurationUtil.getTimeDurationHelper(key, value,
TimeUnit.MILLISECONDS);
+ String configured = conf.get(key);
+ if (configured != null && !matchesDuration(conf, key, configured,
requiredMillis)) {
+ rejectUserConfigured(conf, key, value);
+ }
+ conf.set(key, value);
+ }
+
+ private void setLocalOverrideReplication(OzoneConfiguration conf, String key,
+ ReplicationFactor value) throws IOException {
+ String configured = conf.get(key);
+ if (configured != null && parseReplicationFactor(configured) != value) {
+ rejectUserConfigured(conf, key, value.name());
+ }
+ conf.set(key, value.name());
+ }
+
+ private static void rejectUserConfigured(OzoneConfiguration conf, String
key, String required)
+ throws IOException {
+ String source = userConfiguredSource(conf, key);
+ if (source == null) {
+ return;
+ }
+ throw new IOException("ozone local requires " + key + "=" + required
+ + ", but it is set to " + conf.get(key) + " (source: " + source
+ + "). Remove or change that setting.");
+ }
+
+ private static boolean matchesInt(OzoneConfiguration conf, String key, int
value) {
+ try {
+ return conf.getInt(key, value) == value;
+ } catch (NumberFormatException unreadable) {
+ return false;
+ }
+ }
+
+ private static boolean matchesDuration(OzoneConfiguration conf, String key,
String configured,
+ long requiredMillis) {
+ if (lacksTimeUnit(configured)) {
+ return false;
+ }
+ try {
+ return conf.getTimeDuration(key, requiredMillis, TimeUnit.MILLISECONDS)
== requiredMillis;
+ } catch (NumberFormatException unreadable) {
+ return false;
+ }
+ }
+
+ /**
+ * {@link TimeDurationUtil#getDuration} reads a bare number in the caller's
unit with only a warning,
+ * so "120" would mean 120 milliseconds. Every unit suffix it accepts ends
in a letter.
+ */
+ static boolean lacksTimeUnit(String value) {
+ String trimmed = value.trim();
+ return !trimmed.isEmpty() &&
Character.isDigit(trimmed.charAt(trimmed.length() - 1));
+ }
+
+ /** Accepts both spellings {@link
org.apache.hadoop.hdds.client.ReplicationConfig#parse} does, "1" and "ONE". */
+ private static ReplicationFactor parseReplicationFactor(String value) {
+ String trimmed = value.trim();
+ try {
+ return ReplicationFactor.valueOf(Integer.parseInt(trimmed));
+ } catch (IllegalArgumentException notNumeric) {
+ try {
+ return ReplicationFactor.valueOf(trimmed);
+ } catch (IllegalArgumentException notNamed) {
+ return null;
+ }
+ }
+ }
+
+ /**
+ * Source of the value the user chose for {@code key}, or null when its last
source is a shipped
+ * {@code *-default.xml} classpath resource. The comparison is exact:
Configuration records a classpath
+ * resource by its bare name, and {@link OzoneLocal#setConfigurationPath}
absolutizes a {@code --conf}
+ * path so a file with the same name stays a user choice.
+ */
+ private static String userConfiguredSource(OzoneConfiguration conf, String
key) {
+ String[] sources = conf.getPropertySources(key);
+ if (sources == null || sources.length == 0) {
+ return null;
+ }
+ String source = sources[sources.length - 1];
+ return SHIPPED_DEFAULT_RESOURCES.contains(source) ? null : source;
+ }
+
private int configureScm(OzoneConfiguration conf,
PersistedPortState persistedPorts, PortAllocator portAllocator)
throws IOException {
@@ -427,10 +562,9 @@ private int configureScm(OzoneConfiguration conf,
return scmClientPort;
}
- private void configureScmStorage(OzoneConfiguration conf) throws IOException
{
+ private void configureScmStorage(OzoneConfiguration conf) {
Path scmDir = config.getDataDir().resolve(SCM_DIR_NAME);
Path scmMetadataDir = scmDir.resolve(OZONE_METADATA_DIR_NAME);
- Files.createDirectories(scmDir.resolve(DATA_DIR_NAME));
conf.setIfUnset(OZONE_SCM_DB_DIRS,
scmDir.resolve(DATA_DIR_NAME).toString());
@@ -463,10 +597,9 @@ private int configureOm(OzoneConfiguration conf,
return omRpcPort;
}
- private void configureOmStorage(OzoneConfiguration conf) throws IOException {
+ private void configureOmStorage(OzoneConfiguration conf) {
Path omDir = config.getDataDir().resolve(OM_DIR_NAME);
Path omMetadataDir = omDir.resolve(OZONE_METADATA_DIR_NAME);
- Files.createDirectories(omDir.resolve(DATA_DIR_NAME));
conf.setIfUnset(OZONE_OM_DB_DIRS, omDir.resolve(DATA_DIR_NAME).toString());
conf.setIfUnset(OZONE_OM_RATIS_STORAGE_DIR,
@@ -480,13 +613,6 @@ private void configureOmStorage(OzoneConfiguration conf)
throws IOException {
private List<OzoneConfiguration> configureDatanodes(OzoneConfiguration conf,
PersistedPortState persistedPorts, PortAllocator portAllocator)
throws IOException {
- int datanodeCount = config.getDatanodes();
- if (datanodeCount > MAX_DATANODES) {
- throw new IOException("Datanode count " + datanodeCount
- + " exceeds the local maximum of " + MAX_DATANODES
- + "; each datanode reserves " + DATANODE_PORT_KEY_SUFFIXES.length
- + " local ports.");
- }
List<OzoneConfiguration> datanodeConfigurations =
new ArrayList<>(config.getDatanodes());
for (int index = 0; index < config.getDatanodes(); index++) {
@@ -534,13 +660,10 @@ private OzoneConfiguration
configureDatanode(OzoneConfiguration conf,
return dnConf;
}
- private void configureDatanodeStorage(OzoneConfiguration dnConf, int index)
- throws IOException {
+ private void configureDatanodeStorage(OzoneConfiguration dnConf, int index) {
Path datanodeDir = config.getDataDir()
.resolve(DATANODE_DIR_PREFIX + index);
Path datanodeMetadataDir = datanodeDir.resolve(OZONE_METADATA_DIR_NAME);
- Files.createDirectories(datanodeMetadataDir);
- Files.createDirectories(datanodeDir.resolve(DATA_DIR_NAME));
dnConf.set(OZONE_METADATA_DIRS, datanodeMetadataDir.toString());
// Each datanode gets its own Jetty base dir so same-JVM HTTP servers do
@@ -643,30 +766,66 @@ private void startDatanodes(List<OzoneConfiguration>
datanodeConfigurations) {
}
private void waitForClusterReadiness(Duration timeout) throws Exception {
- long deadlineNanos = System.nanoTime() + timeout.toNanos();
+ waitForReadiness(this::clusterReadinessBlocker, "Ozone cluster", timeout);
+ }
+
+ /** {@code blocker} returns why {@code subject} is not ready yet, or null
once it is. */
+ static void waitForReadiness(Supplier<String> blocker, String subject,
Duration timeout)
+ throws InterruptedException, TimeoutException {
+ long startNanos = System.nanoTime();
+ long nextLogNanos = startNanos + READINESS_LOG_INTERVAL_NANOS;
while (true) {
- if (isClusterReady()) {
+ String reason = blocker.get();
+ if (reason == null) {
return;
}
- if (System.nanoTime() >= deadlineNanos) {
- throw new TimeoutException("Timed out waiting " + timeout
- + " for the local Ozone cluster to become ready.");
+ long now = System.nanoTime();
+ // Compared as a Duration because timeout.toNanos() overflows for very
long timeouts.
+ if (Duration.ofNanos(now - startNanos).compareTo(timeout) >= 0) {
+ throw new TimeoutException("Timed out waiting " + timeout + " for the
local " + subject
+ + " to become ready: " + reason + ".");
+ }
+ if (now >= nextLogNanos) {
+ LOG.info("Waiting for the local {} to become ready: {}.", subject,
reason);
+ nextLogNanos = now + READINESS_LOG_INTERVAL_NANOS;
}
Thread.sleep(READINESS_POLL_INTERVAL_MILLIS);
}
}
- private boolean isClusterReady() {
- if (!scm.checkLeader() || !om.isLeaderReady()) {
- return false;
+ private String clusterReadinessBlocker() {
+ if (!scm.checkLeader()) {
+ return "SCM has no Ratis leader yet";
+ }
+ if (!om.isLeaderReady()) {
+ return "OM is not leader-ready yet";
}
- if (config.getDatanodes() == 0) {
- return true;
+ int registered = scm.getScmNodeManager().getAllNodeCount();
+ if (registered < config.getDatanodes()) {
+ return "only " + registered + " of " + config.getDatanodes()
+ + " datanodes have registered with SCM";
}
- // The cluster is usable once every datanode has registered with SCM and
- // SCM has left safe mode.
- return scm.getScmNodeManager().getAllNodes().size() >=
config.getDatanodes()
- && !scm.isInSafeMode();
+ // SCM refuses block allocation until it leaves safe mode
(BlockManagerImpl#allocateBlock).
+ if (scm.isInSafeMode()) {
+ return "SCM is still in safe mode ("
+ + formatSafeModeRuleBlocker(scm.getRuleStatus()) + ")";
+ }
+ return null;
+ }
+
+ static String formatSafeModeRuleBlocker(List<SafeModeRuleStatusProto> rules)
{
+ if (rules.isEmpty()) {
+ return "safe-mode rule status is not available yet";
+ }
+ String unmetRules = joinSafeModeRules(rules.stream().filter(rule ->
!rule.getValidate()));
+ return unmetRules.isEmpty()
+ ? "all rules currently report satisfied: " +
joinSafeModeRules(rules.stream())
+ : unmetRules;
+ }
+
+ private static String joinSafeModeRules(Stream<SafeModeRuleStatusProto>
rules) {
+ return rules.map(rule -> rule.getRuleName() + ": " + rule.getStatusText())
+ .collect(Collectors.joining("; "));
}
private void enableSameJvmMetricsMode() {
@@ -684,20 +843,39 @@ private void restoreSameJvmMetricsMode() {
}
}
- private void prepareStorageLayout() throws IOException {
+ private void requireSupportedDatanodeCount() throws IOException {
+ int datanodeCount = config.getDatanodes();
+ if (datanodeCount < 1) {
+ throw new IOException("Datanode count " + datanodeCount + " is below the
local minimum of 1.");
+ }
+ if (datanodeCount > MAX_DATANODES) {
+ throw new IOException("Datanode count " + datanodeCount
+ + " exceeds the local maximum of " + MAX_DATANODES
+ + "; each datanode reserves " + DATANODE_PORT_KEY_SUFFIXES.length
+ + " local ports.");
+ }
+ }
+
+ private void validateStorageLayout() throws IOException {
Path dataDir = config.getDataDir();
if (Files.exists(dataDir) && !Files.isDirectory(dataDir)) {
throw new IOException("Local Ozone data dir " + dataDir
+ " is not a directory.");
}
+ if (config.getFormatMode() == LocalOzoneClusterConfig.FormatMode.NEVER) {
+ requireExistingLayout();
+ }
+ }
+ private void prepareStorageLayout() throws IOException {
switch (config.getFormatMode()) {
case ALWAYS:
- deleteDirectory(dataDir);
+ LOG.info("Removing local Ozone data dir {} (format mode ALWAYS).",
config.getDataDir());
+ deleteDirectory(config.getDataDir());
createBaseLayout();
break;
case NEVER:
- requireExistingLayout();
+ // validateStorageLayout() already required the existing layout.
break;
case IF_NEEDED:
createBaseLayout();
@@ -708,6 +886,17 @@ private void prepareStorageLayout() throws IOException {
}
}
+ private void createServiceDirectories() throws IOException {
+ Path dataDir = config.getDataDir();
+
Files.createDirectories(dataDir.resolve(SCM_DIR_NAME).resolve(DATA_DIR_NAME));
+
Files.createDirectories(dataDir.resolve(OM_DIR_NAME).resolve(DATA_DIR_NAME));
+ for (int index = 0; index < config.getDatanodes(); index++) {
+ Path datanodeDir = dataDir.resolve(DATANODE_DIR_PREFIX + index);
+ Files.createDirectories(datanodeDir.resolve(OZONE_METADATA_DIR_NAME));
+ Files.createDirectories(datanodeDir.resolve(DATA_DIR_NAME));
+ }
+ }
+
private void createBaseLayout() throws IOException {
Files.createDirectories(config.getDataDir());
Files.createDirectories(metadataDir());
@@ -734,6 +923,10 @@ private void requireExistingLayout() throws IOException {
}
private PersistedPortState loadPersistedPortState() throws IOException {
+ if (config.getFormatMode() == LocalOzoneClusterConfig.FormatMode.ALWAYS) {
+ // Format mode ALWAYS deletes the data dir in prepareStorageLayout(), so
the persisted ports would be stale.
+ return PersistedPortState.empty(portStateFile());
+ }
PersistedPortState persistedPorts =
PersistedPortState.load(portStateFile());
if (config.getFormatMode() == LocalOzoneClusterConfig.FormatMode.NEVER) {
@@ -888,6 +1081,10 @@ private PersistedPortState(Path path) {
this.path = path;
}
+ static PersistedPortState empty(Path path) {
+ return new PersistedPortState(path);
+ }
+
static PersistedPortState load(Path path) throws IOException {
PersistedPortState state = new PersistedPortState(path);
if (!Files.exists(path)) {
diff --git
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
index 0d5e6ec1734..79b50efc729 100644
---
a/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
+++
b/hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/OzoneLocal.java
@@ -19,6 +19,7 @@
import java.io.PrintWriter;
import java.nio.file.Path;
+import java.nio.file.Paths;
import java.time.Duration;
import java.time.format.DateTimeParseException;
import java.util.concurrent.Callable;
@@ -109,10 +110,57 @@ public OzoneLocal() {
super(factory);
}
+ @Override
+ public void setConfigurationPath(String configPath) {
+ // Configuration records a classpath resource by its bare name and
LocalOzoneCluster#userConfiguredSource
+ // compares names exactly, so a scheme-less path is absolutized:
./ozone-default.xml must stay a user
+ // choice. A scheme (file:, hdfs:) cannot collide and java.nio would
mangle it into a literal path, so
+ // it passes through. fs.Path does the scheme check because it also
rejects an empty value at parse.
+ if (new org.apache.hadoop.fs.Path(configPath).toUri().getScheme() == null)
{
+ configPath =
Paths.get(configPath).toAbsolutePath().normalize().toString();
+ }
+ super.setConfigurationPath(configPath);
+ }
+
+ @Override
+ public void printError(Throwable error) {
+ if (!(error instanceof StartupFailureException)) {
+ super.printError(error);
+ return;
+ }
+
+ super.printError(error.getCause());
+ String hint = startupFailureHint(LOG.isInfoEnabled(), isVerbose());
+ if (hint != null) {
+ err().println(hint);
+ }
+ }
+
+ static String startupFailureHint(boolean infoEnabled, boolean verbose) {
+ if (infoEnabled && verbose) {
+ return null;
+ }
+ if (!infoEnabled && !verbose) {
+ return "For more detail, re-run with `ozone --loglevel INFO local run`
for service logs,"
+ + " or add --verbose for the full stack trace.";
+ }
+ if (!infoEnabled) {
+ return "For service logs, re-run with `ozone --loglevel INFO local
run`.";
+ }
+ return "For the full stack trace, add --verbose.";
+ }
+
public static void main(String[] args) {
new OzoneLocal().run(args);
}
+ private static final class StartupFailureException extends Exception {
+
+ private StartupFailureException(Throwable cause) {
+ super(cause);
+ }
+ }
+
@Command(name = "run",
description = "Start SCM, OM, and datanodes in one local process")
static class RunCommand extends AbstractSubcommand implements Callable<Void>
{
@@ -197,13 +245,22 @@ static class RunCommand extends AbstractSubcommand
implements Callable<Void> {
public Void call() throws Exception {
LocalOzoneClusterConfig config = resolveConfig();
try (LocalOzoneRuntime runtime = createRuntime(config, getOzoneConf())) {
- runtime.start();
+ start(runtime);
printSummary(runtime, config);
awaitShutdown(runtime);
}
return null;
}
+ /** Wraps startup failures so {@link OzoneLocal#printError} can print the
cause and then the hint. */
+ private void start(LocalOzoneRuntime runtime) throws Exception {
+ try {
+ runtime.start();
+ } catch (Exception ex) {
+ throw new StartupFailureException(ex);
+ }
+ }
+
LocalOzoneRuntime createRuntime(LocalOzoneClusterConfig config,
OzoneConfiguration seedConfiguration) {
return new LocalOzoneCluster(config, seedConfiguration);
}
@@ -293,8 +350,9 @@ public LocalOzoneClusterConfig.FormatMode convert(String
value) {
try {
return LocalOzoneClusterConfig.FormatMode.fromString(value);
} catch (IllegalArgumentException ex) {
- throw new CommandLine.TypeConversionException(
- "Expected one of: if-needed, always, never.");
+ // picocli's wrapper names the option but not the value, which may
come from OZONE_LOCAL_FORMAT.
+ throw new CommandLine.TypeConversionException("Invalid format mode
'" + value
+ + "'. Expected one of: if-needed, always, never.");
}
}
}
@@ -304,19 +362,25 @@ private static final class DurationConverter
@Override
public Duration convert(String value) {
+ String trimmed = value.trim();
try {
- return Duration.parse(value.trim());
+ return Duration.parse(trimmed);
} catch (DateTimeParseException ignored) {
- return parseHadoopStyleDuration(value);
+ return parseHadoopStyleDuration(trimmed);
}
}
private static Duration parseHadoopStyleDuration(String value) {
+ if (LocalOzoneCluster.lacksTimeUnit(value)) {
+ throw new CommandLine.TypeConversionException("Missing time unit in
'" + value
+ + "'. " + durationMessage());
+ }
try {
return TimeDurationUtil.getDuration("--startup-timeout", value,
TimeUnit.MILLISECONDS);
} catch (RuntimeException ex) {
- throw new CommandLine.TypeConversionException(durationMessage());
+ throw new CommandLine.TypeConversionException("Invalid duration '" +
value
+ + "'. " + durationMessage());
}
}
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java
index 1f109c6de42..c0d2e19aea5 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestLocalOzoneCluster.java
@@ -18,8 +18,10 @@
package org.apache.hadoop.ozone.local;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.hdds.HddsConfigKeys.HDDS_HEARTBEAT_INTERVAL;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_SAFEMODE_MIN_DATANODE;
import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_SAFEMODE_PIPELINE_CREATION;
+import static
org.apache.hadoop.hdds.HddsConfigKeys.HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_CONTAINER_RATIS_ENABLED_KEY;
import static org.apache.hadoop.hdds.scm.ScmConfigKeys.HDDS_DATANODE_DIR_KEY;
import static
org.apache.hadoop.hdds.scm.ScmConfigKeys.OZONE_SCM_CLIENT_ADDRESS_KEY;
@@ -29,6 +31,7 @@
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION_TYPE;
import static org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_ADDRESS_KEY;
+import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_KEY;
import static
org.apache.hadoop.ozone.om.OMConfigKeys.OZONE_SERVER_DEFAULT_REPLICATION_TYPE_KEY;
import static org.junit.jupiter.api.Assertions.assertEquals;
@@ -42,10 +45,16 @@
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.Collections;
import java.util.Properties;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.hdds.client.ReplicationFactor;
import org.apache.hadoop.hdds.client.ReplicationType;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
+import
org.apache.hadoop.hdds.protocol.proto.StorageContainerLocationProtocolProtos.SafeModeRuleStatusProto;
import org.junit.jupiter.api.Test;
import org.junit.jupiter.api.io.TempDir;
@@ -349,6 +358,255 @@ void prepareConfigurationRejectsTooManyDatanodes() throws
Exception {
+ "; each datanode reserves 8 local ports.", error.getMessage());
}
+ @Test
+ void zeroDatanodesIsRejected() throws Exception {
+ LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(
+ tempDir.resolve("local-ozone"))
+ .setDatanodes(0)
+ .build();
+
+ IOException error = assertPrepareFails(config);
+
+ assertMessageContains(error, "Datanode count 0");
+ }
+
+ /** Configuration#unset() keeps the source, so the key must read as
unconfigured, not as a conflict. */
+ @Test
+ void keyUnsetAfterBeingConfiguredIsNotRejected() throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_REPLICATION, ReplicationFactor.THREE.name(),
"test-ozone-site.xml");
+ seed.unset(OZONE_REPLICATION);
+
+ assertEquals(ReplicationFactor.ONE.name(),
prepared(seed).get(OZONE_REPLICATION));
+ }
+
+ @Test
+ void configuredMetadataDirIsReplacedByTheLocalDataDir() throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_METADATA_DIRS, "/real/cluster/metadata",
"test-ozone-site.xml");
+
+ assertEquals(metadataDir(tempDir.resolve("local-ozone")).toString(),
+ prepared(seed).get(OZONE_METADATA_DIRS));
+ }
+
+ @Test
+ void defaultsFileNamedByPathCountsAsUserConfig() {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_REPLICATION, ReplicationFactor.THREE.name(),
"/home/me/ozone-default.xml");
+
+ IOException error = assertPrepareFails(seed);
+
+ assertMessageContains(error, OZONE_REPLICATION);
+ }
+
+ @Test
+ void tooManyDatanodesIsRejectedBeforeFormatDeletesDataDir() throws Exception
{
+ Path dataDir = tempDir.resolve("local-ozone");
+ Path marker = writeMarker(dataDir, "keep me");
+ LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir)
+ .setFormatMode(LocalOzoneClusterConfig.FormatMode.ALWAYS)
+ .setDatanodes(LocalOzoneCluster.MAX_DATANODES + 1)
+ .build();
+
+ assertPrepareFails(config);
+
+ assertTrue(Files.exists(marker),
+ "format ALWAYS must not delete the data dir for a run that cannot
start");
+ }
+
+ @Test
+ void conflictingUserConfigIsRejected() {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_REPLICATION, ReplicationFactor.THREE.name(),
"test-ozone-site.xml");
+
+ IOException error = assertPrepareFails(seed);
+
+ assertMessageContains(error, OZONE_REPLICATION);
+ assertMessageContains(error, ReplicationFactor.ONE.name());
+ assertMessageContains(error, ReplicationFactor.THREE.name());
+ assertMessageContains(error, "test-ozone-site.xml");
+ }
+
+ @Test
+ void userConfigMatchingTheLocalRequirementIsAccepted() throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_REPLICATION, ReplicationFactor.ONE.name(),
"test-ozone-site.xml");
+
+ assertEquals(ReplicationFactor.ONE.name(),
prepared(seed).get(OZONE_REPLICATION));
+ }
+
+ @Test
+ void conflictingUserConfigIsRejectedBeforeFormatDeletesDataDir() throws
Exception {
+ Path dataDir = tempDir.resolve("local-ozone");
+ Path marker = writeMarker(dataDir, "keep me");
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_REPLICATION, ReplicationFactor.THREE.name(),
"test-ozone-site.xml");
+ LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir)
+ .setFormatMode(LocalOzoneClusterConfig.FormatMode.ALWAYS)
+ .build();
+
+ assertPrepareFails(config, seed);
+
+ assertTrue(Files.exists(marker),
+ "format ALWAYS must not delete the data dir for a run that cannot
start");
+ }
+
+ @Test
+ void duplicateConfiguredPortsAreRejectedBeforeFormatDeletesDataDir() throws
Exception {
+ Path dataDir = tempDir.resolve("local-ozone");
+ Path marker = writeMarker(dataDir, "keep me");
+ LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(dataDir)
+ .setFormatMode(LocalOzoneClusterConfig.FormatMode.ALWAYS)
+ .setScmPort(9860)
+ .setOmPort(9860)
+ .build();
+
+ IOException error = assertPrepareFails(config);
+
+ assertMessageContains(error, "more than once");
+ assertTrue(Files.exists(marker),
+ "format ALWAYS must not delete the data dir for a run that cannot
start");
+ }
+
+ /** Hadoop keeps XML whitespace, so padding around the required value is not
a conflict. */
+ @Test
+ void paddedValueMatchingTheLocalRequirementIsAccepted() throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_REPLICATION_TYPE, "\n STAND_ALONE\n",
"test-ozone-site.xml");
+
+ assertEquals(ReplicationType.STAND_ALONE.name(),
prepared(seed).get(OZONE_REPLICATION_TYPE));
+ }
+
+ /** Duration#toNanos throws ArithmeticException for this timeout, so the
wait must not call it. */
+ @Test
+ void readinessAcceptsTimeoutBeyondNanoTimeRange() throws Exception {
+ LocalOzoneCluster.waitForReadiness(() -> null, "Ozone cluster",
+ Duration.ofDays(999_999_999L));
+ }
+
+ @Test
+ void equivalentDurationSpellingIsAccepted() throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(HDDS_HEARTBEAT_INTERVAL, "1000ms", "test-ozone-site.xml");
+ seed.set(OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY, "1000ms",
"test-ozone-site.xml");
+ seed.set(HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT, "3000ms",
"test-ozone-site.xml");
+
+ OzoneConfiguration conf = prepared(seed);
+
+ assertEquals("1s", conf.get(HDDS_HEARTBEAT_INTERVAL));
+ assertEquals("1s", conf.get(OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY));
+ assertEquals("3s", conf.get(HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT));
+ }
+
+ @Test
+ void unitlessConfigurationDurationsAreRejected() {
+ assertUnitlessDurationRejected(HDDS_HEARTBEAT_INTERVAL, "1000");
+ assertUnitlessDurationRejected(OZONE_OM_RATIS_MINIMUM_TIMEOUT_KEY, "1");
+ assertUnitlessDurationRejected(HDDS_SCM_WAIT_TIME_AFTER_SAFE_MODE_EXIT,
"3000");
+ }
+
+ @Test
+ void equivalentBooleanSpellingIsAccepted() throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(HDDS_CONTAINER_RATIS_ENABLED_KEY, "FALSE", "test-ozone-site.xml");
+
+ assertFalse(prepared(seed).getBoolean(HDDS_CONTAINER_RATIS_ENABLED_KEY,
true));
+ }
+
+ @Test
+ void numericReplicationIsAccepted() throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_SERVER_DEFAULT_REPLICATION_KEY, "1", "test-ozone-site.xml");
+
+ assertEquals(ReplicationFactor.ONE.name(),
+ prepared(seed).get(OZONE_SERVER_DEFAULT_REPLICATION_KEY));
+ }
+
+ @Test
+ void conflictingDurationIsRejected() {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(HDDS_HEARTBEAT_INTERVAL, "30s", "test-ozone-site.xml");
+
+ IOException error = assertPrepareFails(seed);
+
+ assertMessageContains(error, HDDS_HEARTBEAT_INTERVAL);
+ assertMessageContains(error, "30s");
+ }
+
+ @Test
+ void unparseableValueIsRejected() {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(HDDS_HEARTBEAT_INTERVAL, "banana", "test-ozone-site.xml");
+
+ IOException error = assertPrepareFails(seed);
+
+ assertMessageContains(error, HDDS_HEARTBEAT_INTERVAL);
+ assertMessageContains(error, "banana");
+ }
+
+ /**
+ * Regression guard: ozone-default.xml ships hdds.heartbeat.interval=30s, so
a shipped default read as a
+ * user choice would reject every run.
+ */
+ @Test
+ void shippedDefaultIsNotTreatedAsUserConfig() throws Exception {
+ assertEquals("1s", prepared(new
OzoneConfiguration()).get(HDDS_HEARTBEAT_INTERVAL));
+ assertNotEquals("1s", new
OzoneConfiguration().get(HDDS_HEARTBEAT_INTERVAL),
+ "the shipped default must differ from the local value, or this guards
nothing");
+ }
+
+ @Test
+ void generatedShippedDefaultIsNotTreatedAsUserConfig() throws Exception {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(OZONE_REPLICATION, ReplicationFactor.THREE.name(),
"hdds-server-scm-default.xml");
+
+ assertEquals(ReplicationFactor.ONE.name(),
prepared(seed).get(OZONE_REPLICATION));
+ }
+
+ @Test
+ void safeModeRuleBlockerListsOnlyUnmetRules() {
+ String blocker = LocalOzoneCluster.formatSafeModeRuleBlocker(Arrays.asList(
+ safeModeRule("registered datanodes", true, "registered"),
+ safeModeRule("healthy containers", false, "1 of 3 reported")));
+
+ assertEquals("healthy containers: 1 of 3 reported", blocker);
+ }
+
+ @Test
+ void safeModeRuleBlockerFallsBackToSatisfiedRuleStatuses() {
+ String blocker =
LocalOzoneCluster.formatSafeModeRuleBlocker(Collections.singletonList(
+ safeModeRule("registered datanodes", true, "registered")));
+
+ assertEquals("all rules currently report satisfied: registered datanodes:
registered", blocker);
+ }
+
+ @Test
+ void safeModeRuleBlockerExplainsUnavailableRuleStatus() {
+ assertEquals("safe-mode rule status is not available yet",
+ LocalOzoneCluster.formatSafeModeRuleBlocker(Collections.emptyList()));
+ }
+
+ @Test
+ void readinessTimeoutNamesTheUnmetCondition() {
+ TimeoutException error = assertThrows(TimeoutException.class,
+ () -> LocalOzoneCluster.waitForReadiness(() -> "only 1 of 3 datanodes
have registered",
+ "Ozone cluster", Duration.ZERO));
+
+ assertMessageContains(error, "only 1 of 3 datanodes have registered");
+ assertMessageContains(error, "Ozone cluster");
+ }
+
+ @Test
+ void readinessReturnsOnceBlockerReportsReady() throws Exception {
+ AtomicInteger attempts = new AtomicInteger();
+
+ LocalOzoneCluster.waitForReadiness(
+ () -> attempts.incrementAndGet() < 2 ? "not yet" : null, "Ozone
cluster",
+ Duration.ofSeconds(30));
+
+ assertEquals(2, attempts.get());
+ }
+
@Test
void persistedPortFileContainsDatanodePorts() throws Exception {
Path dataDir = tempDir.resolve("local-ozone");
@@ -425,15 +683,46 @@ private LocalOzoneCluster
newCluster(LocalOzoneClusterConfig config) {
return new LocalOzoneCluster(config, new OzoneConfiguration());
}
+ private OzoneConfiguration prepared(OzoneConfiguration seed) throws
IOException {
+ LocalOzoneClusterConfig config = defaultConfig();
+ try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, seed)) {
+ return cluster.prepareConfiguration().getConfiguration();
+ }
+ }
+
+ private LocalOzoneClusterConfig defaultConfig() {
+ return
LocalOzoneClusterConfig.builder(tempDir.resolve("local-ozone")).build();
+ }
+
private IOException assertPrepareFails(LocalOzoneClusterConfig config) {
+ return assertPrepareFails(config, new OzoneConfiguration());
+ }
+
+ private IOException assertPrepareFails(OzoneConfiguration seed) {
+ return assertPrepareFails(defaultConfig(), seed);
+ }
+
+ private IOException assertPrepareFails(LocalOzoneClusterConfig config,
+ OzoneConfiguration seed) {
return assertThrows(IOException.class, () -> {
- try (LocalOzoneCluster cluster = newCluster(config)) {
+ try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, seed)) {
cluster.prepareConfiguration();
}
});
}
- private void assertMessageContains(IOException error, String expectedText) {
+ private void assertUnitlessDurationRejected(String key, String value) {
+ OzoneConfiguration seed = new OzoneConfiguration();
+ seed.set(key, value, "test-ozone-site.xml");
+
+ IOException error = assertPrepareFails(seed);
+
+ assertMessageContains(error, key);
+ assertMessageContains(error, value);
+ assertMessageContains(error, "test-ozone-site.xml");
+ }
+
+ private void assertMessageContains(Exception error, String expectedText) {
assertTrue(error.getMessage().contains(expectedText), error.getMessage());
}
@@ -456,6 +745,14 @@ private void assertPositivePort(Properties properties,
String key) {
assertTrue(Integer.parseInt(properties.getProperty(key)) > 0, key);
}
+ private static SafeModeRuleStatusProto safeModeRule(String name, boolean
validated, String status) {
+ return SafeModeRuleStatusProto.newBuilder()
+ .setRuleName(name)
+ .setValidate(validated)
+ .setStatusText(status)
+ .build();
+ }
+
private Path metadataDir(Path dataDir) {
return dataDir.resolve(METADATA_DIR_NAME);
}
diff --git
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
index 0f8686e91a2..c91c30d563a 100644
---
a/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
+++
b/hadoop-ozone/tools/src/test/java/org/apache/hadoop/ozone/local/TestOzoneLocal.java
@@ -18,20 +18,28 @@
package org.apache.hadoop.ozone.local;
import static java.nio.charset.StandardCharsets.UTF_8;
+import static org.apache.hadoop.ozone.OzoneConfigKeys.OZONE_REPLICATION;
import static org.junit.jupiter.api.Assertions.assertEquals;
import static org.junit.jupiter.api.Assertions.assertFalse;
import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertNull;
import static org.junit.jupiter.api.Assertions.assertThrows;
+import static org.junit.jupiter.api.Assertions.assertTimeoutPreemptively;
import static org.junit.jupiter.api.Assertions.assertTrue;
import java.io.ByteArrayOutputStream;
+import java.io.IOException;
import java.io.OutputStreamWriter;
import java.io.PrintWriter;
import java.lang.reflect.Field;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.nio.file.Paths;
import java.time.Duration;
+import org.apache.hadoop.hdds.cli.GenericCli;
import org.apache.hadoop.hdds.conf.OzoneConfiguration;
import org.junit.jupiter.api.Test;
+import org.junit.jupiter.api.io.TempDir;
import picocli.CommandLine;
import picocli.CommandLine.Command;
import picocli.CommandLine.IDefaultValueProvider;
@@ -108,6 +116,7 @@ void runCommandStartsRuntimeAndPrintsStartupSummary()
throws Exception {
assertTrue(text.contains("Local Ozone is running from"), text);
assertTrue(text.contains("SCM RPC: localhost:9860"), text);
assertTrue(text.contains("OM RPC: localhost:9862"), text);
+ assertFalse(text.contains("Datanodes:"), text);
assertTrue(text.contains("Press Ctrl+C to stop."), text);
}
@@ -123,6 +132,107 @@ void runCommandClosesRuntimeWhenStartupFails() {
assertTrue(runtime.closed);
}
+ @Test
+ void runCommandPreservesStartupFailureAsTheCause() throws Exception {
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ StubRuntime runtime = new StubRuntime("localhost", 9860, 9862);
+ runtime.failStart = true;
+ TestableRunCommand command = new TestableRunCommand(runtime);
+ CommandLine commandLine = new CommandLine(command);
+ commandLine.setErr(new PrintWriter(new OutputStreamWriter(err, UTF_8),
true));
+ commandLine.parseArgs();
+
+ Exception error = assertThrows(Exception.class, command::call);
+
+ assertTrue(error.getCause() instanceof IllegalStateException,
error.toString());
+ assertEquals("startup failed", error.getCause().getMessage());
+ assertEquals("", err.toString(UTF_8.name()));
+ assertTrue(runtime.closed);
+ }
+
+ /** Preemptive timeout: execute() otherwise blocks in awaitShutdown() until
a JVM shutdown hook fires. */
+ @Test
+ void conflictingConfigReachesStderrThroughGenericCli(@TempDir Path dataDir)
throws Exception {
+ ByteArrayOutputStream err = new ByteArrayOutputStream();
+ OzoneLocal ozoneLocal = new OzoneLocal();
+ ozoneLocal.getCmd().setErr(new PrintWriter(new OutputStreamWriter(err,
UTF_8), true));
+
+ int exitCode = assertTimeoutPreemptively(Duration.ofSeconds(30),
+ () -> ozoneLocal.execute(new String[] {"-D", OZONE_REPLICATION +
"=THREE",
+ "run", "--data-dir", dataDir.toString()}));
+
+ assertEquals(GenericCli.EXECUTION_ERROR_EXIT_CODE, exitCode);
+ String[] lines = err.toString(UTF_8.name()).trim().split("\n");
+ assertEquals(2, lines.length, err.toString(UTF_8.name()));
+ assertTrue(lines[0].contains(OZONE_REPLICATION), lines[0]);
+ assertTrue(lines[0].contains("THREE"), lines[0]);
+ assertTrue(lines[1].contains("--verbose"), lines[1]);
+ }
+
+ @Test
+ void relativeDefaultNamedConfigFileCountsAsUserConfig(@TempDir Path tempDir)
throws Exception {
+ Path customConfig = tempDir.resolve("ozone-default.xml").toAbsolutePath();
+ Files.write(customConfig, ("<configuration><property><name>" +
OZONE_REPLICATION
+ +
"</name><value>THREE</value></property></configuration>").getBytes(UTF_8));
+ Path relativeConfig =
Paths.get("").toAbsolutePath().relativize(customConfig);
+ OzoneLocal ozoneLocal = new OzoneLocal();
+
+ ozoneLocal.getCmd().parseArgs("--conf", relativeConfig.toString(), "run");
+
+ OzoneConfiguration seed = ozoneLocal.getOzoneConf();
+ assertEquals("THREE", seed.get(OZONE_REPLICATION));
+ String[] sources = seed.getPropertySources(OZONE_REPLICATION);
+ String source = sources[sources.length - 1];
+ assertEquals(customConfig.normalize().toString(), source);
+ LocalOzoneClusterConfig config = LocalOzoneClusterConfig.builder(
+ tempDir.resolve("local-ozone")).build();
+
+ IOException error = assertThrows(IOException.class, () -> {
+ try (LocalOzoneCluster cluster = new LocalOzoneCluster(config, seed)) {
+ cluster.prepareConfiguration();
+ }
+ });
+
+ assertTrue(error.getMessage().contains(source), error.getMessage());
+ }
+
+ @Test
+ void fileUriConfigPathLoadsTheNamedFile(@TempDir Path tempDir) throws
Exception {
+ Path customConfig = tempDir.resolve("my-ozone-site.xml").toAbsolutePath();
+ Files.write(customConfig, ("<configuration><property><name>" +
OZONE_REPLICATION
+ +
"</name><value>THREE</value></property></configuration>").getBytes(UTF_8));
+ OzoneLocal ozoneLocal = new OzoneLocal();
+
+ ozoneLocal.getCmd().parseArgs("--conf", customConfig.toUri().toString(),
"run");
+
+ assertEquals("THREE", ozoneLocal.getOzoneConf().get(OZONE_REPLICATION));
+ }
+
+ @Test
+ void emptyConfigPathIsRejectedAtParse() {
+ OzoneLocal ozoneLocal = new OzoneLocal();
+
+ assertThrows(CommandLine.ParameterException.class,
+ () -> ozoneLocal.getCmd().parseArgs("--conf", "", "run"));
+ }
+
+ @Test
+ void startupFailureHintOnlySuggestsMissingDetail() {
+ String noDetail = OzoneLocal.startupFailureHint(false, false);
+ assertTrue(noDetail.contains("--loglevel INFO"), noDetail);
+ assertTrue(noDetail.contains("--verbose"), noDetail);
+
+ String logsEnabled = OzoneLocal.startupFailureHint(true, false);
+ assertFalse(logsEnabled.contains("--loglevel INFO"), logsEnabled);
+ assertTrue(logsEnabled.contains("--verbose"), logsEnabled);
+
+ String verboseEnabled = OzoneLocal.startupFailureHint(false, true);
+ assertTrue(verboseEnabled.contains("--loglevel INFO"), verboseEnabled);
+ assertFalse(verboseEnabled.contains("--verbose"), verboseEnabled);
+
+ assertNull(OzoneLocal.startupFailureHint(true, true));
+ }
+
@Test
void runCommandOptionsUseEnvironmentDefaults() throws Exception {
assertEnvDefault("dataDir", OzoneLocal.ENV_DATA_DIR,
@@ -230,7 +340,7 @@ void resolveConfigAllowsS3gAndEphemeralToBeNegated() {
@Test
void resolveConfigRejectsInvalidFormat() {
- assertParseError("--format", "sometimes", "--format");
+ assertParseError("--format", "sometimes", "sometimes");
}
@Test
@@ -250,7 +360,7 @@ void resolveConfigRejectsDatanodeCountBelowOne() {
@Test
void resolveConfigRejectsInvalidDuration() {
- assertParseError("--startup-timeout", "forever", "--startup-timeout");
+ assertParseError("--startup-timeout", "forever", "Invalid duration
'forever'");
}
@Test
@@ -258,6 +368,18 @@ void resolveConfigRejectsNonPositiveDuration() {
assertConfigError("--startup-timeout", "0s", "--startup-timeout");
}
+ /** Without the unit check "120" parsed as 120 milliseconds; the quotes are
asserted because "120s" is in the hint. */
+ @Test
+ void resolveConfigRejectsDurationWithoutTimeUnit() {
+ assertParseError("--startup-timeout", "120", "Missing time unit in '120'");
+ }
+
+ @Test
+ void resolveConfigAcceptsHadoopStyleMinutes() {
+ assertEquals(Duration.ofMinutes(2), resolve("--startup-timeout", "2m")
+ .getStartupTimeout());
+ }
+
@Test
void resolveConfigRejectsInvalidPath() {
assertParseError("--data-dir", "\0", "--data-dir");
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
