henrybear327 commented on code in PR #10934:
URL: https://github.com/apache/ozone/pull/10934#discussion_r3894148521
##########
hadoop-ozone/tools/src/main/java/org/apache/hadoop/ozone/local/LocalOzoneCluster.java:
##########
@@ -643,30 +780,64 @@ private void startDatanodes(List<OzoneConfiguration>
datanodeConfigurations) {
}
private void waitForClusterReadiness(Duration timeout) throws Exception {
+ waitForReadiness(this::clusterReadinessBlocker, "Ozone cluster", timeout);
+ }
+
+ /**
+ * Polls {@code blocker} until it reports ready. {@code blocker} returns why
{@code subject} is
+ * not ready yet, so the wait can name the unmet condition instead of
reporting a bare timeout
+ * that cannot distinguish a slow start from a stuck one.
+ */
+ static void waitForReadiness(Supplier<String> blocker, String subject,
Duration timeout)
+ throws InterruptedException, TimeoutException {
long deadlineNanos = System.nanoTime() + timeout.toNanos();
+ long nextLogNanos = System.nanoTime() + READINESS_LOG_INTERVAL_NANOS;
while (true) {
- if (isClusterReady()) {
+ String reason = blocker.get();
+ if (reason == null) {
return;
}
if (System.nanoTime() >= deadlineNanos) {
- throw new TimeoutException("Timed out waiting " + timeout
- + " for the local Ozone cluster to become ready.");
+ throw new TimeoutException("Timed out waiting " + timeout + " for the
local " + subject
+ + " to become ready: " + reason + ".");
+ }
+ if (System.nanoTime() >= nextLogNanos) {
+ LOG.info("Waiting for the local {} to become ready: {}.", subject,
reason);
+ nextLogNanos = System.nanoTime() + READINESS_LOG_INTERVAL_NANOS;
}
Thread.sleep(READINESS_POLL_INTERVAL_MILLIS);
}
}
- private boolean isClusterReady() {
- if (!scm.checkLeader() || !om.isLeaderReady()) {
- return false;
+ /**
+ * Returns why the cluster is not usable yet, or null once it is ready. The
cluster is usable
+ * once SCM and OM are leader-ready, every datanode has registered with SCM,
and SCM has left
+ * safe mode.
+ */
+ private String clusterReadinessBlocker() {
+ if (!scm.checkLeader()) {
+ return "SCM has no Ratis leader yet";
+ }
+ if (!om.isLeaderReady()) {
+ return "OM is not leader-ready yet";
}
- if (config.getDatanodes() == 0) {
- return true;
+ int registered = scm.getScmNodeManager().getAllNodes().size();
+ if (registered < config.getDatanodes()) {
+ return "only " + registered + " of " + config.getDatanodes()
+ + " datanodes have registered with SCM";
}
- // The cluster is usable once every datanode has registered with SCM and
- // SCM has left safe mode.
- return scm.getScmNodeManager().getAllNodes().size() >=
config.getDatanodes()
- && !scm.isInSafeMode();
+ // Registration alone is not enough: SCM refuses block allocation until it
leaves safe mode.
+ if (scm.isInSafeMode()) {
+ return "SCM is still in safe mode (" + unmetSafeModeRules() + ")";
+ }
+ return null;
+ }
+
+ private String unmetSafeModeRules() {
+ return scm.getScmSafeModeManager().getRuleStatus().entrySet().stream()
Review Comment:
Fixed. Sorry for uploading a version that actually broke the build.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
