imbajin commented on code in PR #3221:
URL: https://github.com/apache/hugegraph/pull/3221#discussion_r4248424172
##########
hugegraph-server/hugegraph-hbase/src/main/java/org/apache/hugegraph/backend/store/hbase/HbaseStore.java:
##########
@@ -87,6 +96,117 @@ private void registerMetaHandlers() {
HbaseMetrics metrics = new HbaseMetrics(this.sessions);
return metrics.compact(this.tableNames());
});
+
+ this.registerMetaHandler(META_STORAGE_READINESS, (session, meta, args)
-> {
+ E.checkArgument(args.length == 1 && args[0] instanceof Number,
+ "Expect the timeout in ms as the only argument");
+ return this.storageReadiness(((Number) args[0]).longValue());
+ });
+ }
+
+ public static final String META_STORAGE_READINESS = "storage_readiness";
+
+ private static final ExecutorService READINESS_EXECUTOR =
Executors.newCachedThreadPool(r -> {
Review Comment:
⚠️ Blocking: no. Summary: A timed-out table check only calls cancel(true) on
a task in this unbounded cached pool; cancellation does not guarantee that a
blocking HBase Admin RPC has stopped, and the Admin stays open until the task
exits. After the result cache expires, repeated probes can accumulate workers
and Admin resources during a slow HBase outage. Please bound or single-flight
outstanding HBase checks per graph and ensure the HBase operation timeout fits
the readiness budget. Evidence: readinessOf() submits the blocking check at
line 190 and returns on timeout at lines 196–198; tableAvailable() closes the
Admin only after its calls return.
##########
hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/profile/StorageReadiness.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.api.profile;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import org.apache.hugegraph.HugeGraph;
+import org.apache.hugegraph.auth.HugeGraphAuthProxy;
+import org.apache.hugegraph.core.GraphManager;
+import org.apache.hugegraph.util.Log;
+import org.slf4j.Logger;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Whether this server can serve graph traffic, for a readiness probe.
+ * A configured graph that failed to load makes the server not ready
+ * whatever its backend. Graphs on an embedded backend are ready as soon as
+ * the REST layer answers. Graphs on a remote backend are probed through the
+ * backend's "storage_readiness" metadata: on hstore at least one known Store
+ * answers a cheap direct call (PD is only needed until the first Store list
+ * is known), on hbase every table of every store of the graph (schema,
+ * graph, system) is enabled and available. Graphs that share a backend
+ * configuration (hstore: the same PD peers) share one probe; every hbase
+ * graph is its own scope, since its table namespace is derived from the
+ * graph name. Independent configurations are probed side by side within one
+ * budget, and the result is reused for a short TTL to keep repeated probes
+ * cheap. The body never carries raw exception text, addresses or graph
+ * names, since the endpoint is unauthenticated.
+ */
+public final class StorageReadiness {
+
+ public static final String STORAGE_READINESS_META = "storage_readiness";
+ public static final String BACKEND_HSTORE = "hstore";
+ public static final String BACKEND_HBASE = "hbase";
+ /** Backends on a remote cluster, whose availability the probe checks. */
+ public static final Set<String> REMOTE_BACKENDS =
ImmutableSet.of(BACKEND_HSTORE,
+
BACKEND_HBASE);
+
+ private static final Logger LOG = Log.logger(StorageReadiness.class);
+
+ /** A probe result with the time it was taken: the TTL check and the body
come from one probe. */
+ private static final class Cached {
+
+ private final Map<String, Object> result;
+ private final long at;
+
+ private Cached(Map<String, Object> result, long at) {
+ this.result = result;
+ this.at = at;
+ }
+ }
+
+ private static final AtomicReference<Cached> LAST = new
AtomicReference<>();
+ private static final AtomicReference<CompletableFuture<Map<String,
Object>>> IN_FLIGHT =
+ new AtomicReference<>();
+ private static final AtomicInteger WAITERS = new AtomicInteger();
+ /** Probes of independent backend configurations run side by side here. */
+ private static final ExecutorService PROBES =
+ Executors.newCachedThreadPool(r -> {
+ Thread t = new Thread(r, "storage-readiness-probe");
+ t.setDaemon(true);
+ return t;
+ });
+
+ private StorageReadiness() {
+ }
+
+ /** One storage probe with a time budget in ms. */
+ public interface Probe {
+
+ Map<String, Object> probe(long timeoutMs) throws Exception;
+ }
+
+ public static Map<String, Object> check(GraphManager manager, long
timeoutMs,
+ long cacheTtlMs, int maxWaiters) {
+ // The graphs are auth proxies and the probe request carries no user,
+ // so look the graphs up and probe them as the internal admin, the way
+ // other internal paths do; the result carries no data or addresses
+ List<Map<String, Object>> holder = new ArrayList<>(1);
+ HugeGraphAuthProxy.runAsAdmin(() -> {
+ List<RemoteGraph> remotes = remoteGraphs(manager);
+ String storage = remotes.isEmpty() ? "embedded" :
remotes.stream().map(r -> r.backend).distinct()
+
.collect(Collectors.joining(","));
+ // a configured graph that did not load is not served, whatever its
+ // backend: GraphManager logs and skips it, so it is absent from
the
+ // graphs looked at above and would otherwise read as "embedded"
+ Map<String, String> failedGraphs = manager.failedGraphs();
+ if (!failedGraphs.isEmpty()) {
+ holder.add(failedBody(failedGraphs.size(), storage,
failedGraphs.values()));
+ return;
+ }
+ if (remotes.isEmpty()) {
+ Map<String, Object> body = new LinkedHashMap<>();
+ body.put("ready", true);
+ body.put("storage", "embedded");
+ body.put("reason", "no graph on a remote storage");
+ holder.add(body);
+ return;
+ }
+ holder.add(check(storage, t -> probeAll(remotes, t), timeoutMs,
cacheTtlMs, maxWaiters));
+ });
+ return holder.get(0);
+ }
+
+ /**
+ * Not ready: `failed` configured graphs could not be loaded (no names,
+ * the endpoint is public). `storage` lists the loaded graphs' backends
+ * plus the failed configs' backends ("unknown" when a config could not
+ * be read), so an all-remote failure does not read as "embedded".
+ */
+ public static Map<String, Object> failedBody(int failed, String storage,
+ java.util.Collection<String>
failedBackends) {
+ java.util.LinkedHashSet<String> backends = new
java.util.LinkedHashSet<>();
+ if (!"embedded".equals(storage)) {
+ backends.addAll(java.util.Arrays.asList(storage.split(",")));
+ }
+ for (String b : failedBackends) {
+ backends.add(b == null || b.isEmpty() ? "unknown" : b);
+ }
+ Map<String, Object> body = new LinkedHashMap<>();
+ body.put("ready", false);
+ body.put("storage", backends.isEmpty() ? "unknown" : String.join(",",
backends));
+ body.put("reason", failed + " configured graph(s) failed to load");
+ body.put("failed_graphs", failed);
+ body.put("failed_backends", failedBackends.stream()
+ .map(b -> b == null ||
b.isEmpty() ? "unknown" : b)
+
.distinct().sorted().collect(Collectors.toList()));
+ return body;
+ }
+
+ /** One graph per distinct remote backend configuration, in graph order;
public for the unit test. */
+ public static final class RemoteGraph {
+
+ final String name;
+ final String backend;
+ final Probe probe;
+
+ public RemoteGraph(String name, String backend, Probe probe) {
+ this.name = name;
+ this.backend = backend;
+ this.probe = probe;
+ }
+ }
+
+ /**
+ * Readiness covers every remote backend configuration this server
+ * uses: hstore graphs with the same PD peers share one probe, every
+ * hbase graph is probed on its own (its namespace is derived from the
+ * graph name, see {@link #configKey}); independent configurations are
+ * probed side by side within the common budget, and ready means all of
+ * them are ready.
+ */
+ static List<RemoteGraph> remoteGraphs(GraphManager manager) {
+ List<RemoteGraph> out = new ArrayList<>();
+ Set<String> seen = new HashSet<>();
+ for (String name : manager.graphs()) {
+ try {
+ HugeGraph graph = manager.graph(name);
+ if (graph == null ||
!REMOTE_BACKENDS.contains(graph.backend())) {
+ continue;
+ }
+ String key = configKey(graph.backend(), graph.configuration(),
graph.name());
Review Comment:
⚠️ Important. Blocking: yes. Summary: graph.name() is only the store name,
but the HBase provider derives its namespace from the graph-space-qualified
identity. Two HBase graphs with the same name in different graphspaces and
identical connection settings therefore collide in this key, so only one
graph's tables are checked and readiness can be green while the other is
unavailable. Please key by the full graph-space-qualified backend identity and
cover this case. Evidence: StandardHugeGraph.name(), BackendProviderFactory
composes graphSpace, slash, and store, and HbaseStoreProvider.namespace().
##########
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStorageProbe.java:
##########
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.backend.store.hstore;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.hugegraph.pd.client.PDClient;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.store.grpc.state.HgStoreStateGrpc;
+import org.apache.hugegraph.store.grpc.state.SubStateReq;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
+import org.slf4j.Logger;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.StatusRuntimeException;
+
+/**
+ * Storage-aware readiness of this server, from this server's point of view:
+ * at least one Store answers a direct, local, read-only gRPC call
+ * (HgStoreState.getScanState, which reads the node's own scan-pool stats and
+ * never touches raft). The Store list comes from PD, refreshed in the
+ * background (single-flight) while the last known list is used right away, so
+ * PD only matters until the first list is known: a PD that is slow, restarting
+ * or down afterwards does not change the readiness of a server whose Stores
+ * still answer. Every known Store is pinged in parallel and the first answer
+ * wins; every wait is bounded by one shared time budget. The result carries
+ * no addresses and no raw exception text, since it is served without
+ * authentication; the full messages go to the log.
+ * <p>
+ * Scope: ready means this server knows a Store list and reaches at least one
+ * Store over gRPC. A Store whose status RPC answers while its raft or
+ * partition path is broken is not detected. A read through the graph path
+ * was measured as the gate and rejected: a failed read invalidates the
+ * partition cache the queries share (with PD down the data plane then fails
+ * too), and one moving partition stalls every read through that client past
+ * the budget, so a rolling Store restart took every server out of the
+ * Service while the traffic was fine.
+ */
+public final class HstoreStorageProbe {
+
+ public static final String META_STORAGE_READINESS = "storage_readiness";
+
+ private static final Logger LOG = Log.logger(HstoreStorageProbe.class);
+
+ private static final ExecutorService EXECUTOR =
Executors.newCachedThreadPool(
Review Comment:
⚠️ Blocking: no. Summary: This cached pool is unbounded, and the probe
submits one blocking ping task for every known Store before waiting for the
first answer. A large PD Store list can create and retain one worker per Store
from a single readiness request; the API waiter cap does not limit this
fan-out. Please bound ping concurrency or use asynchronous gRPC calls while
preserving the shared deadline. Evidence: probe() submits all Stores to this
executor at lines 288–294.
##########
hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/core/GraphManager.java:
##########
@@ -1267,7 +1269,9 @@ public void loadGraphsFromLocal(Map<String, String>
graphConfs) {
HugeFactory.checkGraphName(name, "rest-server.properties");
try {
this.loadGraph(name, graphConfPath);
+ this.failedGraphs.remove(name);
} catch (Throwable e) {
+ this.failedGraphs.put(name, backendOf(graphConfPath));
Review Comment:
‼️ Critical. Blocking: yes. Summary: This records failures only in
loadGraphsFromLocal; PD metadata graphs use loadGraph(Map), whose catch logs
and continues when start_ignore_single_graph_error is enabled without updating
failedGraphs. If the only PD-configured remote graph fails to load,
remoteGraphs() sees none and readiness returns ready=true with
storage=embedded. Please track failed PD-loaded graphs too and clear them after
a successful load. Evidence: the loadGraph(Map) catch at lines 1357–1362 and
StorageReadiness.check() empty-remote branch.
##########
hugegraph-server/hugegraph-hstore/src/main/java/org/apache/hugegraph/backend/store/hstore/HstoreStorageProbe.java:
##########
@@ -0,0 +1,388 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.backend.store.hstore;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicReference;
+
+import org.apache.commons.lang3.concurrent.BasicThreadFactory;
+import org.apache.hugegraph.pd.client.PDClient;
+import org.apache.hugegraph.pd.common.PDException;
+import org.apache.hugegraph.pd.grpc.Metapb;
+import org.apache.hugegraph.store.grpc.state.HgStoreStateGrpc;
+import org.apache.hugegraph.store.grpc.state.SubStateReq;
+import org.apache.hugegraph.util.E;
+import org.apache.hugegraph.util.Log;
+import org.slf4j.Logger;
+
+import io.grpc.ManagedChannel;
+import io.grpc.ManagedChannelBuilder;
+import io.grpc.StatusRuntimeException;
+
+/**
+ * Storage-aware readiness of this server, from this server's point of view:
+ * at least one Store answers a direct, local, read-only gRPC call
+ * (HgStoreState.getScanState, which reads the node's own scan-pool stats and
+ * never touches raft). The Store list comes from PD, refreshed in the
+ * background (single-flight) while the last known list is used right away, so
+ * PD only matters until the first list is known: a PD that is slow, restarting
+ * or down afterwards does not change the readiness of a server whose Stores
+ * still answer. Every known Store is pinged in parallel and the first answer
+ * wins; every wait is bounded by one shared time budget. The result carries
+ * no addresses and no raw exception text, since it is served without
+ * authentication; the full messages go to the log.
+ * <p>
+ * Scope: ready means this server knows a Store list and reaches at least one
+ * Store over gRPC. A Store whose status RPC answers while its raft or
+ * partition path is broken is not detected. A read through the graph path
+ * was measured as the gate and rejected: a failed read invalidates the
+ * partition cache the queries share (with PD down the data plane then fails
+ * too), and one moving partition stalls every read through that client past
+ * the budget, so a rolling Store restart took every server out of the
+ * Service while the traffic was fine.
+ */
+public final class HstoreStorageProbe {
+
+ public static final String META_STORAGE_READINESS = "storage_readiness";
+
+ private static final Logger LOG = Log.logger(HstoreStorageProbe.class);
+
+ private static final ExecutorService EXECUTOR =
Executors.newCachedThreadPool(
+ new
BasicThreadFactory.Builder().namingPattern("storage-readiness-%d")
+ .daemon(true).build());
+
+ private static final KnownStores KNOWN = new KnownStores();
+ private static final Map<String, ManagedChannel> CHANNELS = new
ConcurrentHashMap<>();
+ private static final Set<String> STALE_CHANNELS =
ConcurrentHashMap.newKeySet();
+
+ private HstoreStorageProbe() {
+ }
+
+ /** The active stores as PD sees them. */
+ public interface StoreLister {
+
+ List<Metapb.Store> activeStores() throws Exception;
+ }
+
+ /** One cheap call to one store; returning (any value) means it answered.
*/
+ public interface StorePinger {
+
+ void ping(Metapb.Store store, long timeoutMs) throws Exception;
+ }
+
+ /** The last store list PD answered with, shared by consecutive probes. */
+ public static final class KnownStores {
+
+ private volatile List<Metapb.Store> stores = Collections.emptyList();
+ private volatile long at;
+ private volatile Boolean pdOk;
+ private volatile long pdAt;
+ private final AtomicReference<CompletableFuture<List<Metapb.Store>>>
inFlight =
+ new AtomicReference<>();
+
+ public List<Metapb.Store> stores() {
+ return this.stores;
+ }
+
+ public long ageMs() {
+ return this.at == 0L ? -1L : System.currentTimeMillis() - this.at;
+ }
+
+ /** Outcome of the last finished PD refresh, null before the first
one. */
+ public Boolean pdOk() {
+ return this.pdOk;
+ }
+
+ public long pdAgeMs() {
+ return this.pdAt == 0L ? -1L : System.currentTimeMillis() -
this.pdAt;
+ }
+
+ public void update(List<Metapb.Store> stores) {
+ this.pdOk = true;
+ this.pdAt = System.currentTimeMillis();
+ if (stores != null && !stores.isEmpty()) {
Review Comment:
⚠️ Important. Blocking: yes. Summary: A successful empty PD response sets
pdOk=true but this condition keeps the previous nonempty Store list. probe()
then uses that stale list without waiting for the refresh and can return ready
when an old Store still answers, even though PD reports zero active Stores.
Please replace the cached list on a successful empty response and preserve
stale entries only when refresh fails or times out. Evidence:
KnownStores.update() and probe() branches at lines 258–279 and 316–320.
##########
hugegraph-server/hugegraph-api/src/main/java/org/apache/hugegraph/api/profile/StorageReadiness.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to You under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the
+ * License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hugegraph.api.profile;
+
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.stream.Collectors;
+
+import org.apache.hugegraph.HugeGraph;
+import org.apache.hugegraph.auth.HugeGraphAuthProxy;
+import org.apache.hugegraph.core.GraphManager;
+import org.apache.hugegraph.util.Log;
+import org.slf4j.Logger;
+
+import com.google.common.collect.ImmutableSet;
+
+/**
+ * Whether this server can serve graph traffic, for a readiness probe.
+ * A configured graph that failed to load makes the server not ready
+ * whatever its backend. Graphs on an embedded backend are ready as soon as
+ * the REST layer answers. Graphs on a remote backend are probed through the
+ * backend's "storage_readiness" metadata: on hstore at least one known Store
+ * answers a cheap direct call (PD is only needed until the first Store list
+ * is known), on hbase every table of every store of the graph (schema,
+ * graph, system) is enabled and available. Graphs that share a backend
+ * configuration (hstore: the same PD peers) share one probe; every hbase
+ * graph is its own scope, since its table namespace is derived from the
+ * graph name. Independent configurations are probed side by side within one
+ * budget, and the result is reused for a short TTL to keep repeated probes
+ * cheap. The body never carries raw exception text, addresses or graph
+ * names, since the endpoint is unauthenticated.
+ */
+public final class StorageReadiness {
+
+ public static final String STORAGE_READINESS_META = "storage_readiness";
+ public static final String BACKEND_HSTORE = "hstore";
+ public static final String BACKEND_HBASE = "hbase";
+ /** Backends on a remote cluster, whose availability the probe checks. */
+ public static final Set<String> REMOTE_BACKENDS =
ImmutableSet.of(BACKEND_HSTORE,
+
BACKEND_HBASE);
+
+ private static final Logger LOG = Log.logger(StorageReadiness.class);
+
+ /** A probe result with the time it was taken: the TTL check and the body
come from one probe. */
+ private static final class Cached {
+
+ private final Map<String, Object> result;
+ private final long at;
+
+ private Cached(Map<String, Object> result, long at) {
+ this.result = result;
+ this.at = at;
+ }
+ }
+
+ private static final AtomicReference<Cached> LAST = new
AtomicReference<>();
+ private static final AtomicReference<CompletableFuture<Map<String,
Object>>> IN_FLIGHT =
+ new AtomicReference<>();
+ private static final AtomicInteger WAITERS = new AtomicInteger();
+ /** Probes of independent backend configurations run side by side here. */
+ private static final ExecutorService PROBES =
+ Executors.newCachedThreadPool(r -> {
+ Thread t = new Thread(r, "storage-readiness-probe");
+ t.setDaemon(true);
+ return t;
+ });
+
+ private StorageReadiness() {
+ }
+
+ /** One storage probe with a time budget in ms. */
+ public interface Probe {
+
+ Map<String, Object> probe(long timeoutMs) throws Exception;
+ }
+
+ public static Map<String, Object> check(GraphManager manager, long
timeoutMs,
+ long cacheTtlMs, int maxWaiters) {
+ // The graphs are auth proxies and the probe request carries no user,
+ // so look the graphs up and probe them as the internal admin, the way
+ // other internal paths do; the result carries no data or addresses
+ List<Map<String, Object>> holder = new ArrayList<>(1);
+ HugeGraphAuthProxy.runAsAdmin(() -> {
+ List<RemoteGraph> remotes = remoteGraphs(manager);
+ String storage = remotes.isEmpty() ? "embedded" :
remotes.stream().map(r -> r.backend).distinct()
+
.collect(Collectors.joining(","));
+ // a configured graph that did not load is not served, whatever its
+ // backend: GraphManager logs and skips it, so it is absent from
the
+ // graphs looked at above and would otherwise read as "embedded"
+ Map<String, String> failedGraphs = manager.failedGraphs();
+ if (!failedGraphs.isEmpty()) {
+ holder.add(failedBody(failedGraphs.size(), storage,
failedGraphs.values()));
+ return;
+ }
+ if (remotes.isEmpty()) {
+ Map<String, Object> body = new LinkedHashMap<>();
Review Comment:
⚠️ Blocking: no. Summary: This embedded response and failedBody() omit
cached, while the new README contract says every JSON body carries whether the
answer was cached. Please include cached=false on these non-cached response
paths. Evidence: hugegraph-server/README.md readiness response contract; the
remote-probe path sets this field in probeOnce() and notReady().
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
