Copilot commented on code in PR #3477:
URL: https://github.com/apache/hugegraph/pull/3477#discussion_r4563916549


##########
hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hugegraph.MultiClusterTest;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hugegraph.SimpleClusterTest.BaseSimpleTest.RestClient;
+import org.apache.hugegraph.ct.env.AbstractEnv;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.core.Response;
+
+/**
+ * Schema changes made through one Server must reach the schema caches of the
+ * other Servers (apache/hugegraph#3235). Every test changes the schema through
+ * one Server and then asks the other two, which have the old schema cached.
+ * <p>
+ * Not covered here, because the code does not provide it: two Servers changing
+ * the same schema element at the same moment, and a write that is in flight on
+ * another Server at the instant an index is created.
+ */
+public class MultiClusterSchemaCacheTest extends BaseMultiClusterTest {
+
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+    private static final String PKS = URL_PREFIX + "/schema/propertykeys";
+    private static final String VLS = URL_PREFIX + "/schema/vertexlabels";
+    private static final String ILS = URL_PREFIX + "/schema/indexlabels";
+    private static final String VERTICES = URL_PREFIX + "/graph/vertices";
+    private static final String TASKS = URL_PREFIX + "/tasks";
+    private static final long VISIBLE_SECONDS = 30L;
+
+    @BeforeClass
+    public static void waitForGraphOnEveryServer() throws Exception {
+        waitFor("the graph to open on every Server", 120L, () -> {
+            for (RestClient client : clients) {
+                if (status(client.get(URL_PREFIX)) != 200) {
+                    return false;
+                }
+            }
+            return true;
+        });
+    }
+
+    @Test
+    public void testUpdateReachesEveryServer() throws Exception {
+        String pk = "sc_upd_note";
+        createPropertyKey(0, pk, "TEXT");
+        for (RestClient client : clients) {
+            json(client.get(PKS + "/" + pk));
+        }
+
+        ok(put(clients.get(0), PKS + "/" + pk, "append",
+               "{\"name\": \"" + pk + "\", \"user_data\": {\"marker\": 
\"set\"}}"));
+
+        waitFor("the appended user_data on every Server", VISIBLE_SECONDS, () 
-> {
+            for (RestClient client : clients) {
+                if (!json(client.get(PKS + "/" + 
pk)).path("user_data").has("marker")) {
+                    return false;
+                }
+            }
+            return true;
+        });
+    }
+
+    @Test
+    public void testAppendThroughAnotherServerKeepsTheFirstAppend() throws 
Exception {
+        String a = "sc_app_a";
+        String p1 = "sc_app_p1";
+        String p2 = "sc_app_p2";
+        String vl = "sc_app_person";
+        for (String pk : new String[]{a, p1, p2}) {
+            createPropertyKey(0, pk, "TEXT");
+        }
+        createVertexLabel(0, vl, a);
+        for (RestClient client : clients) {
+            json(client.get(VLS + "/" + vl));
+        }
+
+        ok(put(clients.get(0), VLS + "/" + vl, "append", appendBody(vl, p1)));
+        waitFor(p1 + " on every Server", VISIBLE_SECONDS,
+                () -> everyServerHasProperties(vl, a, p1));
+
+        // Server 1 writes the whole label back with its own change added
+        ok(put(clients.get(1), VLS + "/" + vl, "append", appendBody(vl, p2)));
+        waitFor(p1 + " and " + p2 + " on every Server", VISIBLE_SECONDS,
+                () -> everyServerHasProperties(vl, a, p1, p2));
+    }
+
+    @Test
+    public void testIndexCreatedThroughOneServerWorksOnTheOthers() throws 
Exception {
+        int indexes = 4;
+        String name = "sc_idx_name";
+        String vl = "sc_idx_person";
+        List<String> props = new ArrayList<>();
+        createPropertyKey(0, name, "TEXT");
+        for (int k = 0; k < indexes; k++) {
+            props.add("sc_idx_c" + k);
+            createPropertyKey(0, props.get(k), "INT");
+        }
+        createVertexLabel(0, vl, name, props.toArray(new String[0]));
+        for (int i = 0; i < 20; i++) {
+            StringBuilder body = new StringBuilder("{\"label\": \"" + vl +
+                                                   "\", \"properties\": {\"" + 
name +
+                                                   "\": \"v" + i + "\"");
+            for (String prop : props) {
+                body.append(", \"").append(prop).append("\": ").append(i);
+            }
+            ok(clients.get(0).post(VERTICES, body.append("}}").toString()));
+        }
+
+        for (int k = 0; k < indexes; k++) {
+            String prop = props.get(k);
+            String il = "sc_idx_by_c" + k;
+            for (RestClient client : clients) {
+                json(client.get(VLS + "/" + vl));
+                json(client.get(ILS));
+            }
+
+            // The other Servers serve reads of this label while the index is 
created
+            AtomicBoolean stop = new AtomicBoolean(false);
+            ExecutorService readers = Executors.newFixedThreadPool(4);
+            List<Future<?>> reads = new ArrayList<>();
+            for (int reader = 0; reader < 4; reader++) {
+                RestClient client = clients.get(1 + reader % 2);
+                reads.add(readers.submit(() -> {
+                    while (!stop.get()) {
+                        Response r = client.target().path(VERTICES)
+                                           .queryParam("label", vl)
+                                           .queryParam("limit", 
3).request().get();
+                        r.readEntity(String.class);
+                    }
+                }));
+            }
+            try {
+                JsonNode created = json(clients.get(0).post(ILS,
+                        "{\"name\": \"" + il + "\", \"base_type\": 
\"VERTEX_LABEL\", " +
+                        "\"base_value\": \"" + vl + "\", \"index_type\": 
\"SECONDARY\", " +
+                        "\"fields\": [\"" + prop + "\"]}"));
+                String task = created.path("task_id").asText();
+                waitFor("the rebuild task of " + il, 120L, () ->
+                        "success".equals(json(clients.get(0).get(TASKS + "/" + 
task))
+                                                 
.path("task_status").asText()));
+            } finally {
+                stop.set(true);
+                readers.shutdown();
+                Assert.assertTrue(readers.awaitTermination(60L, 
TimeUnit.SECONDS));
+            }
+            // A reader that stopped on an error left the index created with 
fewer reads
+            for (Future<?> read : reads) {
+                read.get();
+            }
+
+            waitFor(il + " created and listed in its label on every Server",
+                    VISIBLE_SECONDS, () -> {
+                        for (RestClient client : clients) {
+                            JsonNode index = json(client.get(ILS + "/" + il));
+                            if 
(!"CREATED".equals(index.path("status").asText())) {
+                                return false;
+                            }
+                            JsonNode label = json(client.get(VLS + "/" + vl));
+                            if (!contains(label.path("index_labels"), il)) {
+                                return false;
+                            }
+                        }
+                        return true;
+                    });
+
+            // A vertex written through any Server is found through the index 
on any Server
+            for (int writer = 0; writer < clients.size(); writer++) {
+                ok(clients.get(writer).post(VERTICES,
+                        "{\"label\": \"" + vl + "\", \"properties\": {\"" + 
name +
+                        "\": \"w" + k + "_" + writer + "\", \"" + prop + "\": 
" +
+                        value(k, writer) + "}}"));
+            }
+            int trial = k;
+            waitFor("vertices written through every Server in index " + il,
+                    VISIBLE_SECONDS, () -> {
+                        for (RestClient reader : clients) {
+                            for (int writer = 0; writer < clients.size(); 
writer++) {
+                                Response r = reader.target().path(VERTICES)
+                                                   .queryParam("label", vl)
+                                                   .queryParam("properties", 
"{condition}")
+                                                   
.resolveTemplate("condition",
+                                                                    "{\"" + 
prop + "\": " +
+                                                                    
value(trial, writer) + "}")
+                                                   .request().get();
+                                if (r.getStatus() != 200) {
+                                    r.readEntity(String.class);
+                                    return false;
+                                }
+                                if (json(r).path("vertices").size() != 1) {
+                                    return false;
+                                }
+                            }
+                        }
+                        return true;
+                    });
+        }
+    }
+
+    @Test
+    public void testRemoveReachesEveryServerWithSyncDeletion() throws 
Exception {
+        /*
+         * task.sync_deletion used to switch the clear event of a removal off.
+         * This checks the removal end to end, but it is not the regression
+         * test for that: the Servers also publish clear events while they set
+         * up a new graph, which can clear the caches here on their own. The
+         * unit test of CachedSchemaTransactionV2 pins the publish itself.
+         */
+        String graph = "schemacachesync";
+        String prefix = "graphspaces/DEFAULT/graphs/" + graph;
+        String pks = prefix + "/schema/propertykeys";
+        if (status(clients.get(0).get(prefix)) != 200) {
+            ok(clients.get(0).post(prefix,
+                                   "{\"backend\": \"hstore\", \"serializer\": 
\"binary\", " +
+                                   "\"store\": \"" + graph + "\", " +
+                                   "\"task.sync_deletion\": \"true\"}"));
+        }
+        waitFor("the second graph on every Server", 120L, () -> {
+            for (RestClient client : clients) {
+                if (status(client.get(prefix)) != 200) {
+                    return false;
+                }
+            }
+            return true;
+        });
+
+        String pk = "sc_rm_note";
+        ok(clients.get(0).post(pks, propertyKeyBody(pk, "TEXT")));
+        waitFor(pk + " listed on every Server", VISIBLE_SECONDS,
+                () -> everyServerLists(pks, pk, true));
+
+        Response removed = clients.get(0).target().path(pks + "/" + 
pk).request().delete();
+        ok(removed);
+        waitFor(pk + " gone from every Server", VISIBLE_SECONDS,
+                () -> everyServerLists(pks, pk, false));
+    }
+
+    @Test
+    public void testChangeDuringPDRestartReachesEveryServer() throws Exception 
{
+        AbstractEnv cluster = (AbstractEnv) env;
+        int pdNodes = cluster.getPDRestAddrs().size();
+        // One of the PD nodes is the leader: restarting it restarts every 
watch
+        for (int node = 0; node < pdNodes; node++) {
+            String pk = "sc_gap_" + node;
+            for (RestClient client : clients) {
+                json(client.get(PKS));
+            }
+
+            cluster.restartPDNode(node);
+            // One change as early as PD takes it, while the watches may still 
be down
+            waitFor("an add right after restarting PD node " + node, 180L, () 
-> {
+                Response r = clients.get(0).post(PKS, propertyKeyBody(pk, 
"TEXT"));
+                String content = r.readEntity(String.class);
+                return r.getStatus() / 100 == 2 ||
+                       (r.getStatus() == 400 && content.contains("exist"));
+            });
+            cluster.awaitPDNode(node);
+
+            waitFor(pk + " listed on every Server after restarting PD node " + 
node,
+                    90L, () -> everyServerLists(PKS, pk, true));
+        }
+    }
+
+    // ------------------------------------------------------------ helpers
+
+    private static int value(int trial, int writer) {
+        return 100000 + 10 * trial + writer;
+    }
+
+    private static void createPropertyKey(int server, String name, String 
dataType) {
+        ok(clients.get(server).post(PKS, propertyKeyBody(name, dataType)));
+    }
+
+    private static String propertyKeyBody(String name, String dataType) {
+        return "{\"name\": \"" + name + "\", \"data_type\": \"" + dataType +
+               "\", \"cardinality\": \"SINGLE\"}";
+    }
+
+    private static void createVertexLabel(int server, String name, String 
first,
+                                          String... others) {
+        StringBuilder props = new StringBuilder("\"" + first + "\"");
+        for (String other : others) {
+            props.append(", \"").append(other).append("\"");
+        }
+        ok(clients.get(server).post(VLS,
+                                    "{\"name\": \"" + name + "\", 
\"id_strategy\": " +
+                                    "\"AUTOMATIC\", \"properties\": [" + props 
+
+                                    "], \"nullable_keys\": [" + props + "]}"));
+    }
+
+    private static String appendBody(String label, String property) {
+        return "{\"name\": \"" + label + "\", \"properties\": [\"" + property +
+               "\"], \"nullable_keys\": [\"" + property + "\"]}";
+    }
+
+    private static boolean everyServerHasProperties(String label, String... 
properties)
+            throws Exception {
+        for (RestClient client : clients) {
+            JsonNode found = json(client.get(VLS + "/" + 
label)).path("properties");
+            for (String property : properties) {
+                if (!contains(found, property)) {
+                    return false;
+                }
+            }
+        }
+        return true;
+    }
+
+    private static boolean everyServerLists(String path, String name, boolean 
listed)
+            throws Exception {
+        for (RestClient client : clients) {
+            Response r = client.get(path);
+            if (r.getStatus() != 200) {
+                r.readEntity(String.class);
+                return false;
+            }
+            boolean found = false;
+            for (JsonNode pk : json(r).path("propertykeys")) {
+                found |= name.equals(pk.path("name").asText());
+            }
+            if (found != listed) {
+                return false;
+            }
+        }
+        return true;
+    }
+
+    private static boolean contains(JsonNode array, String text) {
+        for (JsonNode item : array) {
+            if (text.equals(item.asText())) {
+                return true;
+            }
+        }
+        return false;
+    }
+
+    private static Response put(RestClient client, String path, String action, 
String body) {
+        return client.target().path(path).queryParam("action", action)
+                     .request().put(Entity.json(body));
+    }
+
+    private static int status(Response response) {
+        response.readEntity(String.class);
+        return response.getStatus();
+    }
+
+    private static void ok(Response response) {
+        String content = response.readEntity(String.class);
+        Assert.assertEquals("Response with status " + response.getStatus() +
+                            " and content " + content, 2, response.getStatus() 
/ 100);
+    }
+
+    private static JsonNode json(Response response) throws Exception {
+        String content = response.readEntity(String.class);
+        Assert.assertEquals("Response with status " + response.getStatus() +
+                            " and content " + content, 2, response.getStatus() 
/ 100);
+        return MAPPER.readTree(content);

Review Comment:
   `Response` objects are not closed in the high-frequency reader loop (and 
also not closed in helper methods). In JAX-RS this can leak connections/buffers 
and make the multi-cluster suite flaky under load. Use try-with-resources (or 
explicitly `close()`) around each `Response` to ensure the underlying 
connection is released promptly.



##########
hugegraph-cluster-test/hugegraph-clustertest-test/src/main/java/org/apache/hugegraph/MultiClusterTest/MultiClusterSchemaCacheTest.java:
##########
@@ -0,0 +1,414 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hugegraph.MultiClusterTest;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.hugegraph.SimpleClusterTest.BaseSimpleTest.RestClient;
+import org.apache.hugegraph.ct.env.AbstractEnv;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+
+import jakarta.ws.rs.client.Entity;
+import jakarta.ws.rs.core.Response;
+
+/**
+ * Schema changes made through one Server must reach the schema caches of the
+ * other Servers (apache/hugegraph#3235). Every test changes the schema through
+ * one Server and then asks the other two, which have the old schema cached.
+ * <p>
+ * Not covered here, because the code does not provide it: two Servers changing
+ * the same schema element at the same moment, and a write that is in flight on
+ * another Server at the instant an index is created.
+ */
+public class MultiClusterSchemaCacheTest extends BaseMultiClusterTest {
+
+    private static final ObjectMapper MAPPER = new ObjectMapper();
+    private static final String PKS = URL_PREFIX + "/schema/propertykeys";
+    private static final String VLS = URL_PREFIX + "/schema/vertexlabels";
+    private static final String ILS = URL_PREFIX + "/schema/indexlabels";
+    private static final String VERTICES = URL_PREFIX + "/graph/vertices";
+    private static final String TASKS = URL_PREFIX + "/tasks";
+    private static final long VISIBLE_SECONDS = 30L;
+
+    @BeforeClass
+    public static void waitForGraphOnEveryServer() throws Exception {
+        waitFor("the graph to open on every Server", 120L, () -> {
+            for (RestClient client : clients) {
+                if (status(client.get(URL_PREFIX)) != 200) {
+                    return false;
+                }
+            }
+            return true;
+        });
+    }
+
+    @Test
+    public void testUpdateReachesEveryServer() throws Exception {
+        String pk = "sc_upd_note";
+        createPropertyKey(0, pk, "TEXT");
+        for (RestClient client : clients) {
+            json(client.get(PKS + "/" + pk));
+        }
+
+        ok(put(clients.get(0), PKS + "/" + pk, "append",
+               "{\"name\": \"" + pk + "\", \"user_data\": {\"marker\": 
\"set\"}}"));
+
+        waitFor("the appended user_data on every Server", VISIBLE_SECONDS, () 
-> {
+            for (RestClient client : clients) {
+                if (!json(client.get(PKS + "/" + 
pk)).path("user_data").has("marker")) {
+                    return false;
+                }
+            }
+            return true;
+        });
+    }
+
+    @Test
+    public void testAppendThroughAnotherServerKeepsTheFirstAppend() throws 
Exception {
+        String a = "sc_app_a";
+        String p1 = "sc_app_p1";
+        String p2 = "sc_app_p2";
+        String vl = "sc_app_person";
+        for (String pk : new String[]{a, p1, p2}) {
+            createPropertyKey(0, pk, "TEXT");
+        }
+        createVertexLabel(0, vl, a);
+        for (RestClient client : clients) {
+            json(client.get(VLS + "/" + vl));
+        }
+
+        ok(put(clients.get(0), VLS + "/" + vl, "append", appendBody(vl, p1)));
+        waitFor(p1 + " on every Server", VISIBLE_SECONDS,
+                () -> everyServerHasProperties(vl, a, p1));
+
+        // Server 1 writes the whole label back with its own change added
+        ok(put(clients.get(1), VLS + "/" + vl, "append", appendBody(vl, p2)));
+        waitFor(p1 + " and " + p2 + " on every Server", VISIBLE_SECONDS,
+                () -> everyServerHasProperties(vl, a, p1, p2));
+    }
+
+    @Test
+    public void testIndexCreatedThroughOneServerWorksOnTheOthers() throws 
Exception {
+        int indexes = 4;
+        String name = "sc_idx_name";
+        String vl = "sc_idx_person";
+        List<String> props = new ArrayList<>();
+        createPropertyKey(0, name, "TEXT");
+        for (int k = 0; k < indexes; k++) {
+            props.add("sc_idx_c" + k);
+            createPropertyKey(0, props.get(k), "INT");
+        }
+        createVertexLabel(0, vl, name, props.toArray(new String[0]));
+        for (int i = 0; i < 20; i++) {
+            StringBuilder body = new StringBuilder("{\"label\": \"" + vl +
+                                                   "\", \"properties\": {\"" + 
name +
+                                                   "\": \"v" + i + "\"");
+            for (String prop : props) {
+                body.append(", \"").append(prop).append("\": ").append(i);
+            }
+            ok(clients.get(0).post(VERTICES, body.append("}}").toString()));
+        }
+
+        for (int k = 0; k < indexes; k++) {
+            String prop = props.get(k);
+            String il = "sc_idx_by_c" + k;
+            for (RestClient client : clients) {
+                json(client.get(VLS + "/" + vl));
+                json(client.get(ILS));
+            }
+
+            // The other Servers serve reads of this label while the index is 
created
+            AtomicBoolean stop = new AtomicBoolean(false);
+            ExecutorService readers = Executors.newFixedThreadPool(4);
+            List<Future<?>> reads = new ArrayList<>();
+            for (int reader = 0; reader < 4; reader++) {
+                RestClient client = clients.get(1 + reader % 2);
+                reads.add(readers.submit(() -> {
+                    while (!stop.get()) {
+                        Response r = client.target().path(VERTICES)
+                                           .queryParam("label", vl)
+                                           .queryParam("limit", 
3).request().get();
+                        r.readEntity(String.class);
+                    }
+                }));

Review Comment:
   `Response` objects are not closed in the high-frequency reader loop (and 
also not closed in helper methods). In JAX-RS this can leak connections/buffers 
and make the multi-cluster suite flaky under load. Use try-with-resources (or 
explicitly `close()`) around each `Response` to ensure the underlying 
connection is released promptly.



##########
hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java:
##########
@@ -77,6 +77,8 @@ public class CachedSchemaTransactionTest extends BaseUnitTest 
{
 
     @Before
     public void setup() {
+        // A V2 graph opened without a PD leaves a clear event to send again
+        CachedSchemaTransactionV2Test.resetPendingClearEvents();
         this.graph = HugeFactory.open(FakeObjects.newConfig());

Review Comment:
   Referencing `CachedSchemaTransactionV2Test.resetPendingClearEvents()` from 
another test class tightly couples unrelated tests and makes refactors brittle 
(renames/moves of the V2 test break other suites). Consider moving this helper 
into a dedicated test utility class (e.g., `SchemaCacheTestSupport`) or 
exposing a small package-private reset hook in production code specifically for 
tests, then call that from both test classes.



##########
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/meta/PdMetaDriver.java:
##########
@@ -116,6 +116,17 @@ public <T> void listen(String key, Consumer<T> consumer) {
         }
     }
 
+    @Override
+    public <T> void listen(String key, Consumer<T> consumer,
+                           Runnable onStarted) {
+        try {
+            this.client.listen(key, (Consumer<WatchResponse>) consumer,
+                               throwable -> { }, onStarted);

Review Comment:
   The new watch-start callback makes recovery behavior more explicit, but the 
error consumer is still a no-op here. This can hide permanent/non-retriable 
watch failures (or repeated reconnect loops) from logs/metrics, making 
production diagnosis difficult. Consider logging the throwable (at least 
WARN/ERROR with the key) or forwarding it to a shared error handler so watch 
failures are observable.



##########
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java:
##########
@@ -500,13 +740,160 @@ private void loadAllSchema() {
     @Override
     public void clear() {
         // Clear schema info firstly
-        super.clear();
-        this.clearCache(false);
-        this.notifySchemaCacheClear();
+        try {
+            super.clear();
+        } finally {
+            // A clear that threw may have removed part of the schema
+            try {
+                this.clearCache(false);
+            } finally {
+                this.publishClearEvent();
+            }
+        }
+    }
+
+    /**
+     * Clear events of this Server that could not be published.
+     * <p>
+     * A clear event is sent after its schema write is committed, so a failed
+     * publish must not fail the write. The graph is kept here instead and the
+     * event is sent again: after a delay that doubles up to a limit, and at
+     * once when the clear-event watch becomes live. One event says "drop your
+     * caches of this graph", so one successful publish covers every earlier
+     * failure for that graph.
+     * <p>
+     * The set lives in memory. If the Server stops before a retry succeeds,
+     * the publish it does on the next graph open covers it. If the Server
+     * never returns, the other Servers keep what they cached until the next
+     * event for that graph or their next watch restart.
+     */
+    static final class PendingClearEvents {
+
+        private static final long RETRY_MIN_DELAY = 1000L;
+        private static final long RETRY_MAX_DELAY = 30000L;
+
+        private final ConcurrentMap<GraphRef, Long> pending =
+                new ConcurrentHashMap<>();
+        private final AtomicLong failures = new AtomicLong();
+
+        // Guarded by this
+        private ScheduledExecutorService executor;
+        private ScheduledFuture<?> retry;
+        private long retryDelay = RETRY_MIN_DELAY;
+        // The number of the newest scheduled retry
+        private long retryEpoch;
+
+        void publish(String graphSpace, String graph) {
+            try {
+                clearEventPublisher.accept(graphSpace, graph);
+            } catch (RuntimeException e) {
+                LOG.warn("Failed to publish the schema cache clear event of " +
+                         "graph '{}/{}', it will be sent again",
+                         graphSpace, graph, e);
+                /*
+                 * The number tells a retry whether the graph failed again
+                 * after the publish the retry is about to count as done.
+                 */
+                this.pending.put(new GraphRef(graphSpace, graph),
+                                 this.failures.incrementAndGet());
+                this.scheduleRetry(false);
+            }
+        }
+
+        void retryNow() {
+            this.scheduleRetry(true);
+        }
+
+        int size() {
+            return this.pending.size();
+        }
+
+        private synchronized void scheduleRetry(boolean now) {
+            if (this.pending.isEmpty()) {
+                return;
+            }
+            if (this.retry != null && !this.retry.isDone()) {
+                if (!now) {
+                    return;
+                }
+                this.retry.cancel(false);
+            }
+            if (this.executor == null) {
+                this.executor = Executors.newSingleThreadScheduledExecutor(r 
-> {
+                    Thread thread = new Thread(r, "schema-cache-clear-retry");
+                    thread.setDaemon(true);
+                    return thread;
+                });
+            }

Review Comment:
   `PendingClearEvents` keeps a daemon `ScheduledExecutorService` alive 
indefinitely once created, even after `pending` becomes empty. While not 
functionally incorrect, it increases background thread footprint over the 
server lifetime after a single transient publish failure. Consider shutting 
down the executor when `pending` becomes empty (and recreating it on-demand), 
or reusing a shared scheduler if one exists in the codebase.



##########
hugegraph-cluster-test/hugegraph-clustertest-minicluster/src/main/java/org/apache/hugegraph/ct/node/AbstractNodeWrapper.java:
##########
@@ -154,8 +157,10 @@ public boolean isStarted() {
             return false;
         }
 
-        try (Scanner sc = new Scanner(Paths.get(getLogPath()),
-                                      StandardCharsets.UTF_8.name())) {
+        try (InputStream log = Files.newInputStream(Paths.get(getLogPath()))) {
+            // Skip what an earlier run of this node wrote before a restart
+            log.skipNBytes(this.startupLogOffset);
+            Scanner sc = new Scanner(log, StandardCharsets.UTF_8.name());
             while (sc.hasNextLine()) {
                 String line = sc.nextLine();
                 if (line.contains(startLine)) return true;

Review Comment:
   `Scanner` isn’t closed here. Even though the underlying `InputStream` is 
closed by the try-with-resources, closing the `Scanner` explicitly (e.g., by 
including it in try-with-resources) is safer and avoids retaining buffers 
longer than necessary.



##########
hugegraph-server/hugegraph-test/src/main/java/org/apache/hugegraph/unit/cache/CachedSchemaTransactionTest.java:
##########
@@ -96,6 +98,7 @@ public void teardown() throws Exception {
                 this.graph.close();
                 this.graph = null;
             }
+            CachedSchemaTransactionV2Test.resetPendingClearEvents();

Review Comment:
   Referencing `CachedSchemaTransactionV2Test.resetPendingClearEvents()` from 
another test class tightly couples unrelated tests and makes refactors brittle 
(renames/moves of the V2 test break other suites). Consider moving this helper 
into a dedicated test utility class (e.g., `SchemaCacheTestSupport`) or 
exposing a small package-private reset hook in production code specifically for 
tests, then call that from both test classes.



##########
hugegraph-server/hugegraph-core/src/main/java/org/apache/hugegraph/backend/cache/CachedSchemaTransactionV2.java:
##########
@@ -500,13 +740,160 @@ private void loadAllSchema() {
     @Override
     public void clear() {
         // Clear schema info firstly
-        super.clear();
-        this.clearCache(false);
-        this.notifySchemaCacheClear();
+        try {
+            super.clear();
+        } finally {
+            // A clear that threw may have removed part of the schema
+            try {
+                this.clearCache(false);
+            } finally {
+                this.publishClearEvent();
+            }
+        }
+    }
+
+    /**
+     * Clear events of this Server that could not be published.
+     * <p>
+     * A clear event is sent after its schema write is committed, so a failed
+     * publish must not fail the write. The graph is kept here instead and the
+     * event is sent again: after a delay that doubles up to a limit, and at
+     * once when the clear-event watch becomes live. One event says "drop your
+     * caches of this graph", so one successful publish covers every earlier
+     * failure for that graph.
+     * <p>
+     * The set lives in memory. If the Server stops before a retry succeeds,
+     * the publish it does on the next graph open covers it. If the Server
+     * never returns, the other Servers keep what they cached until the next
+     * event for that graph or their next watch restart.
+     */
+    static final class PendingClearEvents {
+
+        private static final long RETRY_MIN_DELAY = 1000L;
+        private static final long RETRY_MAX_DELAY = 30000L;
+
+        private final ConcurrentMap<GraphRef, Long> pending =
+                new ConcurrentHashMap<>();
+        private final AtomicLong failures = new AtomicLong();
+
+        // Guarded by this
+        private ScheduledExecutorService executor;
+        private ScheduledFuture<?> retry;
+        private long retryDelay = RETRY_MIN_DELAY;
+        // The number of the newest scheduled retry
+        private long retryEpoch;
+
+        void publish(String graphSpace, String graph) {
+            try {
+                clearEventPublisher.accept(graphSpace, graph);
+            } catch (RuntimeException e) {
+                LOG.warn("Failed to publish the schema cache clear event of " +
+                         "graph '{}/{}', it will be sent again",
+                         graphSpace, graph, e);
+                /*
+                 * The number tells a retry whether the graph failed again
+                 * after the publish the retry is about to count as done.
+                 */
+                this.pending.put(new GraphRef(graphSpace, graph),
+                                 this.failures.incrementAndGet());
+                this.scheduleRetry(false);
+            }
+        }
+
+        void retryNow() {
+            this.scheduleRetry(true);
+        }
+
+        int size() {
+            return this.pending.size();
+        }
+
+        private synchronized void scheduleRetry(boolean now) {
+            if (this.pending.isEmpty()) {
+                return;
+            }
+            if (this.retry != null && !this.retry.isDone()) {
+                if (!now) {
+                    return;
+                }
+                this.retry.cancel(false);
+            }
+            if (this.executor == null) {
+                this.executor = Executors.newSingleThreadScheduledExecutor(r 
-> {
+                    Thread thread = new Thread(r, "schema-cache-clear-retry");
+                    thread.setDaemon(true);
+                    return thread;
+                });
+            }
+            long delay = now ? 0L : this.retryDelay;
+            long epoch = ++this.retryEpoch;
+            this.retry = this.executor.schedule(() -> this.retryPending(epoch),
+                                                delay, TimeUnit.MILLISECONDS);
+        }
+
+        private void retryPending(long epoch) {
+            List<Map.Entry<GraphRef, Long>> entries =
+                    new ArrayList<>(this.pending.entrySet());
+            for (Map.Entry<GraphRef, Long> entry : entries) {
+                GraphRef graph = entry.getKey();
+                try {
+                    clearEventPublisher.accept(graph.graphSpace(),
+                                               graph.graph());
+                    // Keep it if the graph failed again in the meantime
+                    this.pending.remove(graph, entry.getValue());
+                } catch (RuntimeException e) {
+                    LOG.warn("Failed again to publish the schema cache " +
+                             "clear event of graph '{}/{}': {}",
+                             graph.graphSpace(), graph.graph(), e.toString());
+                }
+            }
+            synchronized (this) {
+                if (epoch != this.retryEpoch) {
+                    /*
+                     * A watch start scheduled a newer retry while this one
+                     * was running: that one goes on, or there would be two.
+                     */
+                    return;
+                }
+                this.retry = null;
+                if (this.pending.isEmpty()) {
+                    this.retryDelay = RETRY_MIN_DELAY;
+                    return;
+                }

Review Comment:
   `PendingClearEvents` keeps a daemon `ScheduledExecutorService` alive 
indefinitely once created, even after `pending` becomes empty. While not 
functionally incorrect, it increases background thread footprint over the 
server lifetime after a single transient publish failure. Consider shutting 
down the executor when `pending` becomes empty (and recreating it on-demand), 
or reusing a shared scheduler if one exists in the codebase.



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]


---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]

Reply via email to