allthingssecurity commented on code in PR #30137:
URL: https://github.com/apache/camel/pull/30137#discussion_r4585468443


##########
components/camel-zookeeper-master/src/test/java/org/apache/camel/component/zookeepermaster/dst/FakeZooKeeper.java:
##########
@@ -0,0 +1,954 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.camel.component.zookeepermaster.dst;
+
+import java.lang.reflect.InvocationHandler;
+import java.lang.reflect.Method;
+import java.lang.reflect.ParameterizedType;
+import java.lang.reflect.Proxy;
+import java.lang.reflect.Type;
+import java.lang.reflect.TypeVariable;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedHashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.Executor;
+
+import org.apache.camel.component.zookeepermaster.CamelNodeState;
+import org.apache.camel.test.dst.Dst;
+import org.apache.camel.test.dst.DstExecutor;
+import org.apache.curator.CuratorZookeeperClient;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.listen.Listenable;
+import org.apache.curator.framework.state.ConnectionState;
+import org.apache.curator.framework.state.ConnectionStateListener;
+import org.apache.curator.retry.RetryOneTime;
+import org.apache.curator.utils.EnsurePath;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.data.Stat;
+import tools.jackson.databind.DeserializationFeature;
+import tools.jackson.databind.ObjectMapper;
+import tools.jackson.databind.json.JsonMapper;
+
+/**
+ * A fake ZooKeeper ensemble, the {@link CuratorFramework} of this node on it, 
and the other nodes of the group (peers)
+ * as sessions of their own. Written from the ZooKeeper Programmer's Guide 
(3.9) and the Curator documentation (5.x);
+ * the rules it follows are given (in the words of the documents where 
possible, not verbatim) where they are
+ * implemented and marked:
+ * <ul>
+ * <li>[Z] ZooKeeper Programmer's Guide, "ZooKeeper Sessions", "ZooKeeper 
Watches", "Ephemeral Nodes", "Sequence
+ * Nodes";</li>
+ * <li>[R] ZooKeeper Recipes, "Leader Election";</li>
+ * <li>[C] Curator, "Error Handling" (connection states), the Javadoc of 
{@code ConnectionState}, {@code RetryLoop} and
+ * {@code DeleteBuilder.guaranteed()};</li>
+ * <li>[A] an assumption, where the documents leave the behaviour open; the 
seed picks between the allowed choices.</li>
+ * </ul>
+ * <p>
+ * Modelled: persistent, ephemeral and ephemeral sequential znodes; one-time 
child and data watches, held while the
+ * client is disconnected and delivered after it reconnected to the same 
session; a session that expires when the client
+ * cannot reach the ensemble for longer than the session timeout, with its 
ephemeral znodes; the connection states
+ * SUSPENDED, LOST and RECONNECTED, delivered on a connection state thread; 
watch events on an event thread; the wait of
+ * an operation for the connection (up to the connection timeout) and its 
foreground retry after a connection loss (the
+ * retry policy of the client); a create that the ensemble applied but whose 
answer was lost (a retry then makes a
+ * second znode); guaranteed deletes, done again in the background after a 
reconnect.
+ * <p>
+ * Not modelled: ACLs, containers and TTL znodes, background (asynchronous) 
operations, transactions, the state events
+ * of type None that ZooKeeper gives every watcher (the watchers of the group 
ignore them), read-only mode, the order of
+ * the children in a getChildren answer (sorted here).
+ * <p>
+ * The client notices a connection loss after a seeded delay below two thirds 
of the session timeout (as the ZooKeeper
+ * client does with its read timeout), or at once when one of its calls fails 
on it (a reset connection).
+ * <p>
+ * Timing (the one-stack rule of the harness): writes take effect at the start 
of their latency and reads at the end;
+ * watch and connection state events are tasks of their own threads and never 
block.
+ */
+final class FakeZooKeeper {
+
+    private static final ObjectMapper MAPPER = 
JsonMapper.builderWithJackson2Defaults()
+            
.disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).build();
+
+    private final Dst dst;
+    final List<String> violations = new ArrayList<>();
+    boolean faults = true;
+    /** The negotiated session timeout. */
+    long sessionTimeout = 6000;
+    /** Foreground retries of an operation that failed with a connection loss, 
and the sleep between two. */
+    int retries = 1;
+    long retrySleep = 1000;
+    /** [C] CuratorFrameworkFactory: the connection timeout defaults to 15 
seconds. */
+    long connectionTimeout = 15_000;
+    double slowRate;
+    double responseLostRate;
+
+    private final DstExecutor eventThread;
+    private final DstExecutor stateThread;
+    private final Map<String, Znode> tree = new TreeMap<>();
+    private long zxid;
+    private long nextSession = 1;
+
+    // the session of this node's client, and what the client believes about 
its connection
+    private long session;
+    private boolean network = true;
+    private boolean clientConnected = true;
+    private boolean sessionExpired;
+    private boolean lostPosted;
+    private int outage;
+    private final List<ConnectionStateListener> listeners = new ArrayList<>();
+    private final Map<String, Set<Watcher>> childWatches = new HashMap<>();
+    private final Map<String, Set<Watcher>> dataWatches = new HashMap<>();
+    private final List<Runnable> held = new ArrayList<>();
+    private final Set<String> failedDeletes = new LinkedHashSet<>();
+    private boolean closed;
+    private final CuratorFramework client;
+    private final FakeZookeeperClient zookeeperClient;
+
+    // the other nodes of the group: container -> session
+    private final Map<String, Long> peers = new TreeMap<>();
+
+    private static final class Znode {
+        byte[] data;
+        int version;
+        int cversion;
+        long owner;
+        long czxid;
+        long mzxid;
+        long ctime;
+        long mtime;
+    }
+
+    FakeZooKeeper(Dst dst) {
+        this.dst = dst;
+        this.eventThread = dst.executor("zookeeper-event-thread", 1);
+        this.stateThread = dst.executor("curator-connection-state", 1);
+        this.session = nextSession++;
+        tree.put("/", new Znode());
+        this.client = (CuratorFramework) 
Proxy.newProxyInstance(CuratorFramework.class.getClassLoader(),
+                new Class<?>[] { CuratorFramework.class }, this::invokeClient);
+        this.zookeeperClient = new FakeZookeeperClient();
+    }
+
+    /** The CuratorFramework of this node, as an application shares it (bound 
as the curator bean). */
+    CuratorFramework client() {
+        return client;
+    }
+
+    boolean isClosed() {
+        return closed;
+    }
+
+    int listenerCount() {
+        return listeners.size();
+    }
+
+    boolean isNetworkUp() {
+        return network;
+    }
+
+    boolean isClientConnected() {
+        return clientConnected;
+    }
+
+    // 
------------------------------------------------------------------------------------------------------------
+    // the operator: connection losses and peers
+
+    /**
+     * The client loses its connection to the ensemble for {@code millis}. [Z] 
The client will see a disconnected event
+     * when it is disconnected from the ensemble; [Z] Expirations happens when 
the cluster does not hear from the client
+     * within the specified session timeout period; [C] SUSPENDED: There has 
been a loss of connection; LOST: The
+     * connection is confirmed to be lost ... Curator will set the LOST state 
when it believes that the ZooKeeper
+     * session has expired (after the negotiated session timeout without a 
reconnect, Curator injects the expiry itself,
+     * so the old session is not resumed after that).
+     */
+    void disconnect(long millis) {
+        if (!network) {
+            return;
+        }
+        network = false;
+        int k = ++outage;
+        dst.trace("zk: connection lost for " + millis + " ms (session " + 
session + ")");
+        // [A] the client notices the loss within two thirds of the session 
timeout (the read timeout of the client), or
+        // at once when one of its calls fails on it
+        long detect = dst.between(10, Math.max(10, sessionTimeout * 2 / 3));
+        stateThread.schedule("detect connection loss", detect, () -> 
notice(k));
+        // the ensemble expires the session when it does not hear from the 
client within the session timeout
+        eventThread.schedule("session expiry on the ensemble", sessionTimeout, 
() -> {
+            if (outage == k && !network) {
+                expireLocalSession();
+            }
+        });
+        dst.executor("network", 1).schedule("connection back", millis, () -> 
reconnect(k));
+    }
+
+    /**
+     * The client notices the connection loss: [Z] a ConnectionLoss comes with 
the Disconnected event of the client ([C]
+     * SUSPENDED).
+     */
+    private void notice(int k) {
+        if (outage == k && !network && clientConnected) {
+            clientConnected = false;
+            post(ConnectionState.SUSPENDED);
+            stateThread.schedule("session timeout of the client", 
sessionTimeout, () -> {
+                if (outage == k && !network && !lostPosted) {
+                    lostPosted = true;
+                    post(ConnectionState.LOST);
+                }
+            });
+        }
+    }
+
+    private void reconnect(int k) {
+        if (outage != k || network) {
+            return;
+        }
+        network = true;
+        if (sessionExpired || lostPosted) {
+            // [Z] the client will be told that the session has expired; [C] a 
new session is made, RECONNECTED
+            if (!sessionExpired) {
+                expireLocalSession();
+            }
+            if (!lostPosted) {
+                clientConnected = false;
+                post(ConnectionState.LOST);
+            }
+            session = nextSession++;
+            sessionExpired = false;
+            lostPosted = false;
+            held.clear();
+            failedDeletes.clear();
+            clientConnected = true;
+            dst.trace("zk: reconnected with a new session " + session);
+            post(ConnectionState.RECONNECTED);
+            return;
+        }
+        dst.trace("zk: reconnected to session " + session);
+        if (!clientConnected) {
+            clientConnected = true;
+            post(ConnectionState.RECONNECTED);
+        }
+        // [Z] When a client reconnects, any previously registered watches 
will be reregistered and triggered if
+        // needed"
+        List<Runnable> events = new ArrayList<>(held);
+        held.clear();
+        events.forEach(Runnable::run);
+        // [C] guaranteed(): Curator will record failed node deletions and 
attempt to delete them in the background
+        // until successful"
+        for (String path : new ArrayList<>(failedDeletes)) {
+            retryDelete(path);
+        }
+    }
+
+    private void retryDelete(String path) {
+        long s = session;
+        eventThread.schedule("guaranteed delete of " + path, latency(), () -> {
+            if (network && session == s && failedDeletes.remove(path) && 
tree.containsKey(path)) {
+                dst.trace("zk: guaranteed delete of " + path + " done in the 
background");
+                remove(path);
+            }
+        });
+    }
+
+    private void expireLocalSession() {
+        long expired = session;
+        sessionExpired = true;
+        dst.trace("zk: session " + expired + " expired");
+        // [Z] the watches of the session are lost; ephemeral nodes ... are 
removed when the session ends
+        childWatches.clear();
+        dataWatches.clear();
+        held.clear();
+        removeEphemeralsOf(expired);
+    }
+
+    private void post(ConnectionState state) {
+        dst.trace("zk: connection state " + state);
+        stateThread.execute(() -> {
+            // [C] ConnectionStateManager calls the listeners registered when 
it dispatches the state
+            List<ConnectionStateListener> snapshot = new 
ArrayList<>(listeners);
+            dst.trace("zk: connection state " + state + " to " + 
snapshot.size() + " listeners");
+            for (ConnectionStateListener l : snapshot) {
+                try {
+                    l.stateChanged(client, state);
+                } catch (RuntimeException e) {
+                    // Curator logs it
+                    dst.trace("zk: listener failed: " + e);
+                }
+            }
+        });
+    }
+
+    /** A peer joins the group: its session makes an ephemeral sequential 
znode, as ZooKeeperGroup does. */
+    void peerJoins(String groupPath, String container) {
+        if (peers.containsKey(container)) {
+            return;
+        }
+        long s = nextSession++;
+        peers.put(container, s);
+        ensureParents(groupPath + "/0");
+        CamelNodeState state = new CamelNodeState("dst", container);
+        String path = createNode(groupPath + "/0", encode(state), 
CreateMode.EPHEMERAL_SEQUENTIAL, s);
+        dst.trace("zk: peer " + container + " joins as " + path);
+    }
+
+    /** A peer leaves the group: it closes its session, so its znode is 
removed at once. */
+    void peerLeaves(String container) {
+        Long s = peers.remove(container);
+        if (s != null) {
+            dst.trace("zk: peer " + container + " leaves");
+            removeEphemeralsOf(s);
+        }
+    }
+
+    /** A peer dies: the ensemble removes its znode once its session timed 
out. */
+    void peerDies(String container) {
+        Long s = peers.remove(container);
+        if (s != null) {
+            dst.trace("zk: peer " + container + " dies");
+            deadPeerSessions.add(s);
+            eventThread.schedule("session expiry of " + container, 
sessionTimeout, () -> removeEphemeralsOf(s));
+        }
+    }
+
+    /** The guaranteed deletes that the client still has to do again in the 
background. */
+    Set<String> pendingDeletes() {
+        return failedDeletes;
+    }
+
+    Set<String> peers() {
+        return peers.keySet();
+    }
+
+    // 
------------------------------------------------------------------------------------------------------------
+    // what the members of the group see
+
+    /** The znodes under the group path that the sessions of this node own 
(the current or an expired one). */
+    List<String> localZnodes(String groupPath) {
+        List<String> answer = new ArrayList<>();
+        for (Map.Entry<String, Znode> e : tree.entrySet()) {
+            if (isChild(groupPath, e.getKey()) && e.getValue().owner != 0 && 
!peers.containsValue(e.getValue().owner)
+                    && !isDeadPeer(e.getValue().owner)) {
+                answer.add(e.getKey());
+            }
+        }
+        return answer;
+    }
+
+    private final Set<Long> deadPeerSessions = new HashSet<>();
+
+    private boolean isDeadPeer(long owner) {
+        return deadPeerSessions.contains(owner);
+    }
+
+    /**
+     * The znode of the master, as every member of the group computes it from 
the znodes: one znode per container (the
+     * one with the greatest path), and of those the one with the lowest 
sequence number ([R] the client with the
+     * smallest sequence number is the leader). Null when the group has no 
members.
+     */
+    String master(String groupPath) {
+        Map<String, String> latest = new HashMap<>();
+        for (Map.Entry<String, Znode> e : tree.entrySet()) {
+            if (isChild(groupPath, e.getKey())) {
+                String container = container(e.getValue().data);
+                String cur = latest.get(container);
+                if (cur == null || cur.compareTo(e.getKey()) < 0) {
+                    latest.put(container, e.getKey());
+                }
+            }
+        }
+        return latest.values().stream().sorted().findFirst().orElse(null);
+    }
+
+    /** The container of the peer that owns the znode, or null when it is not 
a peer's. */
+    String peerOf(String path) {
+        Znode z = path != null ? tree.get(path) : null;
+        if (z == null) {
+            return null;
+        }
+        for (Map.Entry<String, Long> e : peers.entrySet()) {
+            if (e.getValue() == z.owner) {
+                return e.getKey();
+            }
+        }
+        return isDeadPeer(z.owner) ? "dead peer" : null;
+    }
+
+    private static boolean isChild(String parent, String path) {
+        return path.startsWith(parent + "/") && path.indexOf('/', 
parent.length() + 1) < 0;
+    }
+
+    private static String container(byte[] data) {
+        try {
+            return MAPPER.readValue(data, CamelNodeState.class).getContainer();
+        } catch (RuntimeException e) {
+            return "undecodable";
+        }
+    }
+
+    private static byte[] encode(CamelNodeState state) {
+        return 
MAPPER.writeValueAsString(state).getBytes(StandardCharsets.UTF_8);
+    }
+
+    // 
------------------------------------------------------------------------------------------------------------
+    // the ensemble
+
+    private long latency() {
+        if (faults && dst.chance(slowRate)) {
+            return dst.between(200, 2000);
+        }
+        return dst.between(1, 20);
+    }
+
+    private void ensureParents(String path) {
+        int i = path.indexOf('/', 1);
+        while (i > 0) {
+            String p = path.substring(0, i);
+            if (!tree.containsKey(p)) {
+                createNode(p, new byte[0], CreateMode.PERSISTENT, 0);
+            }
+            i = path.indexOf('/', i + 1);
+        }
+    }
+
+    private static String parentOf(String path) {
+        int i = path.lastIndexOf('/');
+        return i <= 0 ? "/" : path.substring(0, i);
+    }
+
+    private String createNode(String path, byte[] data, CreateMode mode, long 
owner) {
+        Znode parent = tree.get(parentOf(path));
+        String name = path;
+        if (mode.isSequential()) {
+            // [Z] a monotonically increasing counter will be appended to the 
name ... formatted as %010d, the
+            // counter of the parent
+            name = path + String.format("%010d", parent.cversion);

Review Comment:
   Correction to my reply above: it was wrong, and your nit was right. In 
ZooKeeper 3.9 a delete does not change the cversion of the parent (checked with 
javap on 3.9.7: the delete path of `PrepRequestProcessor` only decrements 
`childCount` and sets `pzxid`, `DataTree.deleteNode` only sets `pzxid`; only a 
create calls `setCversion`). Fixed in ef1be44bb54a: `remove()` no longer bumps 
`cversion`, so the fake matches the ensemble and the comment here holds as 
written.
   
   _Claude Code on behalf of allthingssecurity_
   



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to