bitflicker64 commented on code in PR #3130:
URL: https://github.com/apache/hugegraph/pull/3130#discussion_r3752896742
##########
hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/AbstractGrpcClient.java:
##########
@@ -169,6 +193,224 @@ protected AbstractStub setStubOption(AbstractStub value) {
config.getGrpcMaxOutboundMessageSize());
}
+ private static boolean usesChannels(HgPair<ManagedChannel, ?>[] pairs,
+ ManagedChannel[] channels) {
+ if (pairs == null || pairs.length != channels.length) {
+ return false;
+ }
+ for (int i = 0; i < pairs.length; i++) {
+ HgPair<ManagedChannel, ?> pair = pairs[i];
+ if (pair == null || pair.getKey() != channels[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void refreshChannelsIfAddressChanged(String target) {
+ if (!this.shouldRefreshChannels(target)) {
+ return;
+ }
+
+ ReentrantLock refreshLock = refreshLocks.computeIfAbsent(target,
+ key -> new
ReentrantLock());
+ if (!refreshLock.tryLock()) {
+ return;
+ }
+
+ try {
+ if (!this.shouldRefreshChannels(target)) {
+ return;
+ }
+
+ String resolvedTarget = this.resolveTarget(target);
+ this.postponeNextRefresh(target);
+ if (resolvedTarget.isEmpty()) {
+ return;
+ }
+
+ ManagedChannel[] staleChannels = channels.get(target);
+ String previousTarget = resolvedTargets.get(target);
+ if (previousTarget == null && staleChannels == null) {
+ resolvedTargets.put(target, resolvedTarget);
+ return;
+ }
+ if (resolvedTarget.equals(previousTarget)) {
+ return;
+ }
+ if (staleChannels == null) {
+ resolvedTargets.put(target, resolvedTarget);
+ return;
+ }
+
+ ManagedChannel[] replacementChannels;
+ try {
+ replacementChannels = this.createChannels(target);
+ } catch (RuntimeException ignored) {
+ return;
+ }
+
+ boolean replaced = false;
+ synchronized (channels) {
+ if (channels.get(target) == staleChannels) {
+ channels.put(target, replacementChannels);
+ resolvedTargets.put(target, resolvedTarget);
+ replaced = true;
+ }
+ }
+
+ if (replaced) {
+ this.retireChannels(staleChannels);
+ } else {
+ this.retireChannels(replacementChannels);
+ }
+ } finally {
+ refreshLock.unlock();
+ }
+ }
+
+ private boolean shouldRefreshChannels(String target) {
+ AtomicLong nextResolution = nextResolutions.computeIfAbsent(target,
+ key -> new
AtomicLong());
+ return System.nanoTime() - nextResolution.get() >= 0L;
+ }
+
+ private void postponeNextRefresh(String target) {
+ long interval = Math.max(0L, this.channelRefreshIntervalNanos());
+ nextResolutions.computeIfAbsent(target, key -> new AtomicLong())
+ .set(System.nanoTime() + interval);
+ }
+
+ protected long channelRefreshIntervalNanos() {
+ return DEFAULT_CHANNEL_REFRESH_INTERVAL_NANOS;
+ }
+
+ protected long channelDrainTimeoutNanos() {
+ return TimeUnit.SECONDS.toNanos(config.getGrpcTimeoutSeconds());
+ }
+
+ private ManagedChannel[] createChannels(String target) {
+ ManagedChannel[] value = new ManagedChannel[concurrency];
+ CountDownLatch latch = new CountDownLatch(concurrency);
+ AtomicReference<RuntimeException> failure = new AtomicReference<>();
+ for (int i = 0; i < concurrency; i++) {
+ int fi = i;
+ executor.execute(() -> {
+ try {
+ value[fi] = createChannel(target);
+ } catch (Exception e) {
+ failure.compareAndSet(null, new RuntimeException(e));
+ } finally {
+ latch.countDown();
+ }
+ });
+ }
+
+ InterruptedException interruption = null;
+ while (latch.getCount() > 0L) {
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ interruption = e;
+ }
+ }
+
+ if (failure.get() != null || interruption != null) {
+ forceTerminateChannels(value);
+ }
+ if (interruption != null) {
+ Thread.currentThread().interrupt();
+ throw new RuntimeException(interruption);
+ }
+ if (failure.get() != null) {
+ throw failure.get();
+ }
+ return value;
+ }
+
+ private void retireChannels(ManagedChannel[] retiredChannels) {
+ Arrays.stream(retiredChannels)
+ .filter(channel -> channel != null && !channel.isShutdown())
+ .forEach(ManagedChannel::shutdown);
+
+ long timeout = Math.max(0L, this.channelDrainTimeoutNanos());
+ CHANNEL_CLEANUP_EXECUTOR.schedule(
+ () -> forceTerminateChannels(retiredChannels), timeout,
+ TimeUnit.NANOSECONDS);
+ }
+
+ private void forceTerminateChannels(ManagedChannel[] retiredChannels) {
+ for (ManagedChannel channel : retiredChannels) {
+ if (channel != null && !channel.isTerminated()) {
+ channel.shutdownNow();
+ }
+ }
+ }
+
+ private static String targetHost(String target) {
+ if (target == null || target.isEmpty()) {
+ return "";
+ }
+
+ String endpoint = target;
+ if (target.startsWith("dns://")) {
+ endpoint = target.substring("dns://".length());
+ while (endpoint.startsWith("/")) {
+ endpoint = endpoint.substring(1);
+ }
+ int pathStart = endpoint.indexOf('/');
+ if (pathStart >= 0) {
+ endpoint = endpoint.substring(pathStart + 1);
+ }
+ } else if (target.contains("://")) {
+ return "";
+ }
+
+ return endpointHost(endpoint);
+ }
+
+ private static String endpointHost(String endpoint) {
+ if (endpoint == null || endpoint.isEmpty()) {
+ return "";
+ }
+
+ if (endpoint.charAt(0) == '[') {
+ int hostEnd = endpoint.indexOf(']');
+ if (hostEnd <= 1) {
+ return "";
+ }
+ return endpoint.substring(1, hostEnd);
+ }
+
+ int lastColon = endpoint.lastIndexOf(':');
+ if (lastColon < 0) {
+ return endpoint;
+ }
+ if (endpoint.indexOf(':') != lastColon) {
+ return endpoint;
+ }
+ return endpoint.substring(0, lastColon);
+ }
+
+ protected InetAddress[] resolveHost(String host) throws
UnknownHostException {
+ return InetAddress.getAllByName(host);
+ }
+
+ protected String resolveTarget(String target) {
+ String host = targetHost(target);
+ if (host.isEmpty()) {
+ return "";
+ }
+ try {
+ return Arrays.stream(this.resolveHost(host))
Review Comment:
Revalidated on final head `5afea3b2`. The finding was valid and the refresh
boundary is now explicit: `triggerChannelRefresh()` submits DNS resolution and
replacement coordination to the prestarted `channel-maintenance-*` executor;
replacement channel construction runs on the existing `common-*` executor;
graceful retirement runs on maintenance; forced retirement runs on the
separately prestarted `channel-retirement-*` scheduler so blocked DNS cannot
starve the drain deadline.
`testRefreshSucceedsWhenCallerIsDeniedSocketAndThreadAccess()` denies both
socket and thread access on a `gremlin-server-exec-*` caller, proves all four
stages stay off it, and still returns a stub. Focused failure tests also prove
resolution, refresh submission, channel submission/creation, and
retirement-scheduling failures keep the published healthy pool live and allow
retry. Java 11 passes 24/24.
##########
hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/AbstractGrpcClient.java:
##########
@@ -169,6 +193,224 @@ protected AbstractStub setStubOption(AbstractStub value) {
config.getGrpcMaxOutboundMessageSize());
}
+ private static boolean usesChannels(HgPair<ManagedChannel, ?>[] pairs,
+ ManagedChannel[] channels) {
+ if (pairs == null || pairs.length != channels.length) {
+ return false;
+ }
+ for (int i = 0; i < pairs.length; i++) {
+ HgPair<ManagedChannel, ?> pair = pairs[i];
+ if (pair == null || pair.getKey() != channels[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ private void refreshChannelsIfAddressChanged(String target) {
+ if (!this.shouldRefreshChannels(target)) {
+ return;
+ }
+
+ ReentrantLock refreshLock = refreshLocks.computeIfAbsent(target,
+ key -> new
ReentrantLock());
+ if (!refreshLock.tryLock()) {
+ return;
+ }
+
+ try {
+ if (!this.shouldRefreshChannels(target)) {
+ return;
+ }
+
+ String resolvedTarget = this.resolveTarget(target);
+ this.postponeNextRefresh(target);
+ if (resolvedTarget.isEmpty()) {
+ return;
+ }
+
+ ManagedChannel[] staleChannels = channels.get(target);
+ String previousTarget = resolvedTargets.get(target);
+ if (previousTarget == null && staleChannels == null) {
+ resolvedTargets.put(target, resolvedTarget);
+ return;
+ }
+ if (resolvedTarget.equals(previousTarget)) {
+ return;
+ }
+ if (staleChannels == null) {
+ resolvedTargets.put(target, resolvedTarget);
+ return;
+ }
+
+ ManagedChannel[] replacementChannels;
+ try {
+ replacementChannels = this.createChannels(target);
+ } catch (RuntimeException ignored) {
+ return;
+ }
+
+ boolean replaced = false;
+ synchronized (channels) {
+ if (channels.get(target) == staleChannels) {
+ channels.put(target, replacementChannels);
+ resolvedTargets.put(target, resolvedTarget);
+ replaced = true;
+ }
+ }
+
+ if (replaced) {
+ this.retireChannels(staleChannels);
Review Comment:
Revalidated on final head `5afea3b2`. The reported raw-channel path is gone:
`QueryV2Client#getQueryServiceStub(String)` delegates directly to
`getAsyncStub(target)`. The shared `acquireStub()` path constructs the
configured stub, then rechecks the published channel-pool identity before
caching or returning it and retries on a concurrent replacement.
`testQueryV2StubFollowsPublishedPoolAcrossRefresh()` blocks inside
`setStubOption()` at that real post-cache construction boundary, replaces the
pool, and proves the returned and cached QueryV2 stubs are current, live, and
still spread across every channel. The production line is covered in the
aggregate JaCoCo XML (`line 47: mi=0, ci=5`). `getChannels(String)` remains
public to preserve the existing API, but QueryV2 no longer consumes it.
##########
hugegraph-store/hg-store-client/src/main/java/org/apache/hugegraph/store/client/grpc/AbstractGrpcClient.java:
##########
@@ -169,6 +201,289 @@ protected AbstractStub setStubOption(AbstractStub value) {
config.getGrpcMaxOutboundMessageSize());
}
+ private static boolean usesChannels(HgPair<ManagedChannel, ?>[] pairs,
+ ManagedChannel[] channels) {
+ if (pairs == null || pairs.length != channels.length) {
+ return false;
+ }
+ for (int i = 0; i < pairs.length; i++) {
+ HgPair<ManagedChannel, ?> pair = pairs[i];
+ if (pair == null || pair.getKey() != channels[i]) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ /**
+ * Submits a refresh for the target unless one is already in flight or the
refresh interval
+ * has not elapsed. Returns the in-flight refresh, or null when none is
running.
+ */
+ private CompletableFuture<Void> triggerChannelRefresh(String target) {
+ CompletableFuture<Void> inFlight = refreshTasks.get(target);
+ if (inFlight != null) {
+ return inFlight;
+ }
+ if (!this.shouldRefreshChannels(target)) {
+ return null;
+ }
+
+ CompletableFuture<Void> refresh = new CompletableFuture<>();
+ CompletableFuture<Void> running = refreshTasks.putIfAbsent(target,
refresh);
+ if (running != null) {
+ return running;
+ }
+
+ // Throttle before submitting, so that a failing resolver cannot be
retried in a loop.
+ this.postponeNextRefresh(target);
+ try {
+ this.submitChannelRefresh(() -> {
+ try {
+ this.refreshChannelsIfAddressChanged(target);
+ } catch (Throwable e) {
+ // The executor discards what a task throws, so report it
here.
+ log.warn("Failed to refresh channels of target {}",
target, e);
+ } finally {
+ this.completeRefresh(target, refresh);
+ }
+ });
+ } catch (Throwable e) {
+ // Includes a thread creation denied on this thread; never leave
the entry behind.
+ log.warn("Failed to submit a channel refresh for target {}",
target, e);
+ this.completeRefresh(target, refresh);
+ }
+ return refresh;
+ }
+
+ private void completeRefresh(String target, CompletableFuture<Void>
refresh) {
+ /*
+ * Throttle from completion as well as from submission: a resolver
that is slow rather
+ * than failing can outlast its own interval, which would let every
later call queue
+ * another lookup behind it.
+ */
+ this.postponeNextRefresh(target);
+ refreshTasks.remove(target, refresh);
+ refresh.complete(null);
+ }
+
+ private void submitChannelRefresh(Runnable task) {
+ CHANNEL_MAINTENANCE_EXECUTOR.execute(task);
+ }
+
+ private void awaitInitialResolution(CompletableFuture<Void> refresh) {
+ if (refresh == null) {
+ return;
+ }
+ try {
+ refresh.get(Math.max(0L, this.initialResolutionTimeoutNanos()),
+ TimeUnit.NANOSECONDS);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ } catch (Exception ignored) {
+ // A slow or failing resolver must not delay the first pool any
further.
+ }
+ }
+
+ /**
+ * Runs on a maintenance thread, never on a request thread. At most one
runs per target at a
+ * time — that comes from the refreshTasks entry, not from the size of the
executor. Replaces
+ * the target's pool when its resolved address set has changed, publishing
the replacement
+ * before retiring the previous pool.
+ */
+ private void refreshChannelsIfAddressChanged(String target) {
+ String resolvedTarget = this.resolveTarget(target);
+ if (resolvedTarget.isEmpty()) {
+ return;
+ }
+
+ ManagedChannel[] staleChannels = channels.get(target);
+ String previousTarget = resolvedTargets.get(target);
+ if (resolvedTarget.equals(previousTarget)) {
+ return;
+ }
+ if (staleChannels == null) {
+ /*
+ * Nothing to replace yet. Recording the address here is what lets
the common path
+ * build its first pool already knowing the address, instead of
rebuilding it.
+ */
+ resolvedTargets.put(target, resolvedTarget);
+ return;
+ }
+
+ ManagedChannel[] replacementChannels;
+ try {
+ replacementChannels = this.createChannels(target);
+ } catch (RuntimeException e) {
+ // Keep serving from the last healthy pool.
+ log.warn("Failed to create replacement channels of target {}, " +
+ "keeping the current pool", target, e);
+ return;
+ }
+
+ boolean replaced = false;
+ synchronized (channels) {
+ if (channels.get(target) == staleChannels) {
+ channels.put(target, replacementChannels);
+ resolvedTargets.put(target, resolvedTarget);
+ replaced = true;
+ }
+ }
+ if (replaced) {
+ log.info("Replaced the channel pool of target {}, address changed
from {} to {}",
+ target, previousTarget, resolvedTarget);
+ }
+
+ this.retireChannels(replaced ? staleChannels : replacementChannels);
+ }
+
+ private boolean shouldRefreshChannels(String target) {
+ AtomicLong nextResolution = nextResolutions.computeIfAbsent(target,
+ key -> new
AtomicLong());
+ return System.nanoTime() - nextResolution.get() >= 0L;
Review Comment:
Fixed and revalidated on final head `5afea3b2`. The finding was valid:
`nextResolutions` now stores `AtomicReference<Long>`, where `null` is the
explicit never-scheduled state and every `long` clock value, including zero and
negatives, is a real deadline. `shouldRefreshChannels()` uses `deadline == null
|| nanoTime() - deadline >= 0L`; signed subtraction is the standard
overflow-safe monotonic deadline test for this bounded interval, including
wraparound. `nanoTime()` is injectable.
`testInitialRefreshRunsWhenNanoTimeIsNegative()` starts at `-1L` and proves the
first resolution runs, while `testRefreshDeadlineSurvivesNanoTimeWraparound()`
crosses `Long.MAX_VALUE` to `Long.MIN_VALUE` and proves the deadline neither
fires early nor gets lost. Final Java 11 result: 24/24, zero
failures/errors/skips.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
