Repository: ignite Updated Branches: refs/heads/ignite-zk 98a171c68 -> 5793ffbe2
zk Project: http://git-wip-us.apache.org/repos/asf/ignite/repo Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b389f38c Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b389f38c Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b389f38c Branch: refs/heads/ignite-zk Commit: b389f38cbc59f41dc1c95854684059f15b225b8c Parents: 98a171c Author: sboikov <[email protected]> Authored: Fri Nov 17 09:33:30 2017 +0300 Committer: sboikov <[email protected]> Committed: Fri Nov 17 11:14:05 2017 +0300 ---------------------------------------------------------------------- .../managers/discovery/ConsistentIdNode.java | 33 +++ .../discovery/GridDiscoveryManager.java | 4 +- .../managers/discovery/JoiningNodesAware.java | 2 +- .../tcp/internal/TcpDiscoveryNode.java | 3 +- .../discovery/zk/ZookeeperDiscoverySpi2.java | 64 +++-- .../zk/internal/ZKJoiningNodeData.java | 61 +++++ .../discovery/zk/internal/ZkDiscoveryImpl.java | 116 --------- .../discovery/zk/internal/ZookeeperClient.java | 145 +++++++++-- .../zk/internal/ZookeeperClusterNode.java | 3 + .../zk/internal/ZookeeperDiscoveryImpl.java | 238 +++++++++++++++++++ .../zk/internal/ZookeeperClientTest.java | 127 ++++++++++ 11 files changed, 636 insertions(+), 160 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdNode.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdNode.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdNode.java new file mode 100644 index 0000000..fc806ff --- /dev/null +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdNode.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.internal.managers.discovery; + +import java.io.Serializable; +import org.apache.ignite.cluster.ClusterNode; + +/** + * TODO ZK + */ +public interface ConsistentIdNode extends ClusterNode { + /** + * Sets consistent globally unique node ID which survives node restarts. + * + * @param consistentId Consistent globally unique node ID. + */ + public void setConsistentId(Serializable consistentId); +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java index a6737dc..5262e32 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java @@ -554,8 +554,8 @@ public class GridDiscoveryManager extends GridManagerAdapter<DiscoverySpi> { for (IgniteInClosure<ClusterNode> lsnr : locNodeInitLsnrs) lsnr.apply(locNode); - if (locNode instanceof TcpDiscoveryNode) { - final TcpDiscoveryNode node = (TcpDiscoveryNode)locNode; + if (locNode instanceof ConsistentIdNode) { + final ConsistentIdNode node = (ConsistentIdNode)locNode; if (consistentId != null) node.setConsistentId(consistentId); http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java index 85128e4..7cd73f4 100644 --- a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java +++ b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java @@ -20,7 +20,7 @@ package org.apache.ignite.internal.managers.discovery; import java.util.UUID; /** - * + * TODO ZK */ public interface JoiningNodesAware { public boolean knownNode(UUID nodeId); http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java ---------------------------------------------------------------------- diff --git a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java index 38c2a1b..e5ec655 100644 --- a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java +++ b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java @@ -36,6 +36,7 @@ import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.configuration.IgniteConfiguration; import org.apache.ignite.internal.ClusterMetricsSnapshot; import org.apache.ignite.internal.IgniteNodeAttributes; +import org.apache.ignite.internal.managers.discovery.ConsistentIdNode; import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter; import org.apache.ignite.internal.util.tostring.GridToStringExclude; import org.apache.ignite.internal.util.tostring.GridToStringInclude; @@ -58,7 +59,7 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTE * <strong>This class is not intended for public use</strong> and has been made * <tt>public</tt> due to certain limitations of Java technology. */ -public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements ClusterNode, +public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements ConsistentIdNode, Comparable<TcpDiscoveryNode>, Externalizable { /** */ private static final long serialVersionUID = 0L; http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java index 99810d0..0ba55c9 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java @@ -20,22 +20,17 @@ package org.apache.ignite.spi.discovery.zk; import java.io.Serializable; import java.util.ArrayList; import java.util.Collection; -import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.TreeMap; import java.util.UUID; -import java.util.concurrent.CountDownLatch; -import java.util.concurrent.TimeUnit; import org.apache.ignite.IgniteException; import org.apache.ignite.IgniteLogger; import org.apache.ignite.cluster.ClusterNode; import org.apache.ignite.internal.managers.discovery.JoiningNodesAware; import org.apache.ignite.internal.util.tostring.GridToStringInclude; import org.apache.ignite.internal.util.typedef.internal.S; -import org.apache.ignite.internal.util.typedef.internal.U; import org.apache.ignite.lang.IgniteProductVersion; -import org.apache.ignite.marshaller.jdk.JdkMarshaller; import org.apache.ignite.resources.LoggerResource; import org.apache.ignite.spi.IgniteSpiAdapter; import org.apache.ignite.spi.IgniteSpiException; @@ -49,11 +44,8 @@ import org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport; import org.apache.ignite.spi.discovery.DiscoverySpiListener; import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator; import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport; -import org.apache.ignite.spi.discovery.zk.internal.ZkDiscoveryImpl; -import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClient; +import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl; import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClusterNode; -import org.apache.zookeeper.AsyncCallback; -import org.apache.zookeeper.data.Stat; import org.jetbrains.annotations.Nullable; /** @@ -64,7 +56,16 @@ import org.jetbrains.annotations.Nullable; @DiscoverySpiHistorySupport(true) public class ZookeeperDiscoverySpi2 extends IgniteSpiAdapter implements DiscoverySpi, JoiningNodesAware { /** */ - private String connectString; + private String zkConnectionString; + + /** */ + private int sesTimeout = 5000; + + /** */ + private String basePath = "/apacheIgnite"; + + /** */ + private String clusterName = "default"; /** */ private DiscoverySpiListener lsnr; @@ -76,10 +77,7 @@ public class ZookeeperDiscoverySpi2 extends IgniteSpiAdapter implements Discover private DiscoveryMetricsProvider metricsProvider; /** */ - private int sesTimeout = 5000; - - /** */ - private ZkDiscoveryImpl impl; + private ZookeeperDiscoveryImpl impl; /** */ private ZookeeperClusterNode locNode; @@ -97,6 +95,22 @@ public class ZookeeperDiscoverySpi2 extends IgniteSpiAdapter implements Discover @LoggerResource private IgniteLogger log; + public String getBasePath() { + return basePath; + } + + public void setBasePath(String basePath) { + this.basePath = basePath; + } + + public String getClusterName() { + return clusterName; + } + + public void setClusterName(String clusterName) { + this.clusterName = clusterName; + } + public int getSessionTimeout() { return sesTimeout; } @@ -105,12 +119,12 @@ public class ZookeeperDiscoverySpi2 extends IgniteSpiAdapter implements Discover this.sesTimeout = sesTimeout; } - public String getConnectString() { - return connectString; + public String getZkConnectionString() { + return zkConnectionString; } - public void setConnectString(String connectString) { - this.connectString = connectString; + public void setZkConnectionString(String zkConnectionString) { + this.zkConnectionString = zkConnectionString; } /** */ @@ -318,14 +332,24 @@ public class ZookeeperDiscoverySpi2 extends IgniteSpiAdapter implements Discover @Override public void spiStart(@Nullable String igniteInstanceName) throws IgniteSpiException { initLocalNode(); + log.info("Start Zookeeper discovery [zkConnectionString=" + zkConnectionString + + ", sesTimeout=" + sesTimeout + + ", basePath=" + basePath + + ", clusterName=" + clusterName + ']'); + DiscoveryDataBag discoDataBag = new DiscoveryDataBag(locNode.id()); exchange.collect(discoDataBag); - impl = new ZkDiscoveryImpl(log, lsnr); + impl = new ZookeeperDiscoveryImpl(log, + basePath, + clusterName, + locNode, + lsnr, + exchange); try { - impl.joinTopology(igniteInstanceName, connectString, sesTimeout); + impl.joinTopology(igniteInstanceName, zkConnectionString, sesTimeout); } catch (InterruptedException e) { Thread.currentThread().interrupt(); http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZKJoiningNodeData.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZKJoiningNodeData.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZKJoiningNodeData.java new file mode 100644 index 0000000..278d45e --- /dev/null +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZKJoiningNodeData.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.zk.internal; + +import java.io.Serializable; +import java.util.Map; + +/** + * + */ +class ZKJoiningNodeData implements Serializable { + /** */ + private static final long serialVersionUID = 0L; + + /** */ + private final ZookeeperClusterNode node; + + /** */ + private final Map<Integer, Serializable> joiningNodeData; + + /** + * @param node Node. + * @param joiningNodeData Discovery data. + */ + ZKJoiningNodeData(ZookeeperClusterNode node, Map<Integer, Serializable> joiningNodeData) { + assert node != null && node.id() != null : node; + assert joiningNodeData != null; + + this.node = node; + this.joiningNodeData = joiningNodeData; + } + + /** + * @return Node. + */ + ZookeeperClusterNode node() { + return node; + } + + /** + * @return Discovery data. + */ + Map<Integer, Serializable> joiningNodeData() { + return joiningNodeData; + } +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryImpl.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryImpl.java deleted file mode 100644 index 4c7a7bf..0000000 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryImpl.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.ignite.spi.discovery.zk.internal; - -import java.util.List; -import java.util.concurrent.CountDownLatch; -import org.apache.ignite.IgniteCheckedException; -import org.apache.ignite.IgniteException; -import org.apache.ignite.IgniteLogger; -import org.apache.ignite.internal.util.future.GridFutureAdapter; -import org.apache.ignite.internal.util.typedef.internal.U; -import org.apache.ignite.lang.IgniteRunnable; -import org.apache.ignite.marshaller.jdk.JdkMarshaller; -import org.apache.ignite.spi.IgniteSpiException; -import org.apache.ignite.spi.discovery.DiscoverySpiListener; -import org.apache.zookeeper.AsyncCallback; -import org.apache.zookeeper.CreateMode; -import org.apache.zookeeper.data.Stat; - -/** - * - */ -public class ZkDiscoveryImpl { - /** */ - private static final String IGNITE_PATH = "/apache-ignite"; - - /** */ - private final JdkMarshaller marsh = new JdkMarshaller(); - - /** */ - private final IgniteLogger log; - - /** */ - private final DiscoverySpiListener lsnr; - - /** */ - private ZookeeperClient zkClient; - - /** */ - private final GridFutureAdapter<Void> joinFut = new GridFutureAdapter<>(); - - public ZkDiscoveryImpl(IgniteLogger log, DiscoverySpiListener lsnr) { - this.log = log.getLogger(getClass()); - this.lsnr = lsnr; - } - - public void joinTopology(String igniteInstanceName, String connectString, int sesTimeout) - throws InterruptedException { - try { - zkClient = new ZookeeperClient(igniteInstanceName, - log, - connectString, - sesTimeout, - new ConnectionLossListener()); - } - catch (Exception e) { - throw new IgniteSpiException("Failed to create Zookeeper client", e); - } - - try { - zkClient.createIfNeeded(IGNITE_PATH, null, CreateMode.PERSISTENT); - } - catch (ZookeeperClientFailedException e) { - throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e); - } - } - - /** - * - */ - public void stop() { - if (zkClient != null) - zkClient.close(); - } - - private <T> T unmarshal(byte[] data) throws IgniteCheckedException { - return marsh.unmarshal(data, null); - } - - private byte[] marshal(Object obj) throws IgniteCheckedException { - return marsh.marshal(obj); - } - - /** - * - */ - private class ConnectionLossListener implements IgniteRunnable { - @Override public void run() { - - } - } - - /** - * - */ - private class ZKChildrenUpdateCallback implements AsyncCallback.Children2Callback { - @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { - - } - } -} http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java index 8f3f074..37b153d 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java @@ -17,6 +17,7 @@ package org.apache.ignite.spi.discovery.zk.internal; +import java.util.ArrayDeque; import java.util.List; import java.util.Timer; import java.util.TimerTask; @@ -31,6 +32,7 @@ import org.apache.zookeeper.Watcher; import org.apache.zookeeper.ZooDefs; import org.apache.zookeeper.ZooKeeper; import org.apache.zookeeper.data.ACL; +import org.apache.zookeeper.data.Stat; /** * @@ -58,7 +60,7 @@ public class ZookeeperClient implements Watcher { private long connLossTimeout; /** */ - private long connectStartTime; + private volatile long connStartTime; /** */ private final Object stateMux = new Object(); @@ -69,6 +71,9 @@ public class ZookeeperClient implements Watcher { /** */ private final Timer connTimer; + /** */ + private final ArrayDeque<ZkAsyncOperation> retryQ = new ArrayDeque<>(); + ZookeeperClient(IgniteLogger log, String connectString, int sesTimeout, IgniteRunnable connLostC) throws Exception { this(null, log, connectString, sesTimeout, connLostC); } @@ -86,7 +91,7 @@ public class ZookeeperClient implements Watcher { connLossTimeout = sesTimeout; - connectStartTime = System.currentTimeMillis(); + connStartTime = System.currentTimeMillis(); String threadName = Thread.currentThread().getName(); @@ -108,7 +113,7 @@ public class ZookeeperClient implements Watcher { /** {@inheritDoc} */ @Override public void process(WatchedEvent evt) { if (evt.getType() == Event.EventType.None) { - boolean connLost = false; + ConnectionState newState; synchronized (stateMux) { if (state == ConnectionState.Lost) { @@ -117,8 +122,6 @@ public class ZookeeperClient implements Watcher { return; } - ConnectionState newState; - Event.KeeperState zkState = evt.getState(); switch (zkState) { @@ -149,24 +152,28 @@ public class ZookeeperClient implements Watcher { state = newState; if (newState == ConnectionState.Disconnected) { - connectStartTime = System.currentTimeMillis(); + connStartTime = System.currentTimeMillis(); scheduleConnectionCheck(); } else if (newState == ConnectionState.Connected) stateMux.notifyAll(); else { - assert state == ConnectionState.Lost; + assert state == ConnectionState.Lost : state; closeClient(); - - connLost = true; } } + else + return; } - if (connLost) + if (newState == ConnectionState.Lost) notifyConnectionLost(); + else if (newState == ConnectionState.Connected) { + for (ZkAsyncOperation op : retryQ) + op.execute(); + } } } @@ -178,12 +185,33 @@ public class ZookeeperClient implements Watcher { connLostC.run(); } + /** + * @param path + * @return + * @throws ZookeeperClientFailedException + * @throws InterruptedException + */ + public boolean exists(String path) throws ZookeeperClientFailedException, InterruptedException { + for (;;) { + long connStartTime = this.connStartTime; + + try { + return zk.exists(path, false) != null; + } + catch (Exception e) { + onZookeeperError(connStartTime, e); + } + } + } + public void createIfNeeded(String path, byte[] data, CreateMode createMode) throws ZookeeperClientFailedException, InterruptedException { if (data == null) data = EMPTY_BYTES; for (;;) { + long connStartTime = this.connStartTime; + try { zk.create(path, data, ZK_ACL, createMode); @@ -195,13 +223,15 @@ public class ZookeeperClient implements Watcher { break; } catch (Exception e) { - onZookeeperError(e); + onZookeeperError(connStartTime, e); } } } public void getChildrenAsync(String path, boolean watch, AsyncCallback.Children2Callback cb, Object ctx) { - zk.getChildren(path, watch, cb, ctx); + GetChildrenOperation op = new GetChildrenOperation(path, watch, cb, ctx); + + zk.getChildren(path, watch, new ChildreCallbackWrapper(op), ctx); } /** @@ -214,7 +244,7 @@ public class ZookeeperClient implements Watcher { /** * @param e Error. */ - private void onZookeeperError(Exception e) throws ZookeeperClientFailedException, InterruptedException { + private void onZookeeperError(long prevConnStartTime, Exception e) throws ZookeeperClientFailedException, InterruptedException { ZookeeperClientFailedException err = null; synchronized (stateMux) { @@ -229,22 +259,24 @@ public class ZookeeperClient implements Watcher { throw new ZookeeperClientFailedException(e); } - if (e instanceof KeeperException.ConnectionLossException) { + boolean retry = (e instanceof KeeperException) && needRetry(((KeeperException)e).code().intValue()); + + if (retry) { long remainingTime; - if (state == ConnectionState.Connected) { + if (state == ConnectionState.Connected && connStartTime == prevConnStartTime) { state = ConnectionState.Disconnected; - connectStartTime = System.currentTimeMillis(); + connStartTime = System.currentTimeMillis(); remainingTime = connLossTimeout; } else { - assert connectStartTime != 0; + assert connStartTime != 0; assert state == ConnectionState.Disconnected; - remainingTime = connLossTimeout - (System.currentTimeMillis() - connectStartTime); + remainingTime = connLossTimeout - (System.currentTimeMillis() - connStartTime); if (remainingTime <= 0) { state = ConnectionState.Lost; @@ -286,6 +318,51 @@ public class ZookeeperClient implements Watcher { } /** + * @param code Zookeeper error code. + * @return {@code True} if can retry operation. + */ + private boolean needRetry(int code) { + // TODO ZL: other codes. + return code == KeeperException.Code.CONNECTIONLOSS.intValue(); + } + + /** + * + */ + interface ZkAsyncOperation { + void execute(); + } + + /** + * + */ + class GetChildrenOperation implements ZkAsyncOperation { + /** */ + private final String path; + + /** */ + private final boolean watch; + + /** */ + private final AsyncCallback.Children2Callback cb; + + /** */ + private final Object ctx; + + public GetChildrenOperation(String path, boolean watch, AsyncCallback.Children2Callback cb, Object ctx) { + this.path = path; + this.watch = watch; + this.cb = cb; + this.ctx = ctx; + } + + /** {@inheritDoc} */ + @Override public void execute() { + getChildrenAsync(path, watch, cb, ctx); + } + } + + /** * */ private void closeClient() { @@ -305,7 +382,35 @@ public class ZookeeperClient implements Watcher { private void scheduleConnectionCheck() { assert state == ConnectionState.Disconnected : state; - connTimer.schedule(new ConnectionTimeoutTask(connectStartTime), connLossTimeout); + connTimer.schedule(new ConnectionTimeoutTask(connStartTime), connLossTimeout); + } + + /** + * + */ + class ChildreCallbackWrapper implements AsyncCallback.Children2Callback { + /** */ + private final GetChildrenOperation op; + + /** + * @param op Operation. + */ + private ChildreCallbackWrapper(GetChildrenOperation op) { + this.op = op; + } + + /** {@inheritDoc} */ + @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { + if (needRetry(rc)) { + U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']'); + + retryQ.add(op); + } + else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) + U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']'); + else + op.cb.processResult(rc, path, ctx, children, stat); + } } /** @@ -328,7 +433,7 @@ public class ZookeeperClient implements Watcher { synchronized (stateMux) { if (state == ConnectionState.Disconnected && - ZookeeperClient.this.connectStartTime == connectStartTime) { + ZookeeperClient.this.connStartTime == connectStartTime) { state = ConnectionState.Lost; http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java index 7f0e6b7..3f8ee27 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java @@ -41,6 +41,9 @@ import static org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTE */ public class ZookeeperClusterNode implements ClusterNode, Serializable { /** */ + private static final long serialVersionUID = 0L; + + /** */ private UUID id; /** */ http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java new file mode 100644 index 0000000..aae2d06 --- /dev/null +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java @@ -0,0 +1,238 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.ignite.spi.discovery.zk.internal; + +import java.util.List; +import org.apache.curator.utils.PathUtils; +import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteLogger; +import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; +import org.apache.ignite.internal.util.future.GridFutureAdapter; +import org.apache.ignite.internal.util.typedef.F; +import org.apache.ignite.internal.util.typedef.internal.U; +import org.apache.ignite.lang.IgniteRunnable; +import org.apache.ignite.marshaller.jdk.JdkMarshaller; +import org.apache.ignite.spi.IgniteSpiException; +import org.apache.ignite.spi.discovery.DiscoveryDataBag; +import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange; +import org.apache.ignite.spi.discovery.DiscoverySpiListener; +import org.apache.zookeeper.AsyncCallback; +import org.apache.zookeeper.data.Stat; + +import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL; +import static org.apache.zookeeper.CreateMode.PERSISTENT; + +/** + * TODO ZK: check if compression makes sense. + */ +public class ZookeeperDiscoveryImpl { + /** */ + private static final String JOIN_DATA_DIR = "joinData"; + + /** */ + private static final String ALIVE_NODES_DIR = "alive"; + + /** */ + private final String basePath; + + /** */ + private final String clusterName; + + /** */ + private final JdkMarshaller marsh = new JdkMarshaller(); + + /** */ + private final IgniteLogger log; + + /** */ + private final ZookeeperClusterNode locNode; + + /** */ + private final DiscoverySpiListener lsnr; + + /** */ + private final DiscoverySpiDataExchange exchange; + + /** */ + private ZookeeperClient zkClient; + + /** */ + private final GridFutureAdapter<Void> joinFut = new GridFutureAdapter<>(); + + public ZookeeperDiscoveryImpl(IgniteLogger log, + String basePath, + String clusterName, + ZookeeperClusterNode locNode, + DiscoverySpiListener lsnr, + DiscoverySpiDataExchange exchange) { + assert locNode.id() != null : locNode; + + if (F.isEmpty(clusterName)) + throw new IllegalArgumentException("Cluster name is empty."); + + PathUtils.validatePath(basePath); + + this.log = log.getLogger(getClass()); + this.basePath = basePath; + this.clusterName = clusterName; + this.locNode = locNode; + this.lsnr = lsnr; + this.exchange = exchange; + } + + /** + * @param path Relative path. + * @return Full path. + */ + private String zkPath(String path) { + return basePath + "/" + clusterName + "/" + path; + } + + /** + * @param igniteInstanceName + * @param connectString + * @param sesTimeout + * @throws InterruptedException If interrupted. + */ + public void joinTopology(String igniteInstanceName, String connectString, int sesTimeout) + throws InterruptedException + { + DiscoveryDataBag discoDataBag = new DiscoveryDataBag(locNode.id()); + + exchange.collect(discoDataBag); + + ZKJoiningNodeData joinData = new ZKJoiningNodeData(locNode, discoDataBag.joiningNodeData()); + + byte[] joinDataBytes; + + try { + joinDataBytes = marshal(joinData); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to marshal joining node data", e); + } + + try { + zkClient = new ZookeeperClient(igniteInstanceName, + log, + connectString, + sesTimeout, + new ConnectionLossListener()); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to create Zookeeper client", e); + } + + try { + initZkNodes(); + + // TODO ZK: handle max size + zkClient.createIfNeeded(zkPath(JOIN_DATA_DIR) + "/" + locNode.id() + "-", joinDataBytes, EPHEMERAL_SEQUENTIAL); + + zkClient.createIfNeeded(zkPath(ALIVE_NODES_DIR) + "/" + locNode.id() + "-", null, EPHEMERAL_SEQUENTIAL); + + zkClient.getChildrenAsync(zkPath(ALIVE_NODES_DIR), false, new AsyncCallback.Children2Callback() { + @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { + assert rc == 0 : rc; + + + } + }, null); + + for (;;) { + try { + joinFut.get(10_000); + + break; + } + catch (IgniteFutureTimeoutCheckedException e) { + U.warn(log, "Waiting for local join event [nodeId=" + locNode.id() + ", name=" + igniteInstanceName + ']'); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to join cluster", e); + } + } + } + catch (ZookeeperClientFailedException e) { + throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e); + } + } + + private void initZkNodes() throws ZookeeperClientFailedException, InterruptedException { + // TODO ZK: use multi. + String joinDir = zkPath(JOIN_DATA_DIR); + + if (zkClient.exists(joinDir)) + return; // Assume all others dirs are created. + + zkClient.createIfNeeded(basePath, null, PERSISTENT); + + zkClient.createIfNeeded(zkPath(ALIVE_NODES_DIR), null, PERSISTENT); + + zkClient.createIfNeeded(joinDir, null, PERSISTENT); + } + + /** + * + */ + public void stop() { + if (zkClient != null) + zkClient.close(); + } + + /** + * @param bytes Bytes. + * @return Unmarshalled object. + * @throws IgniteCheckedException If failed. + */ + private <T> T unmarshal(byte[] bytes) throws IgniteCheckedException { + assert bytes != null && bytes.length > 0; + + return marsh.unmarshal(bytes, null); + } + + /** + * @param obj Object. + * @return Bytes. + * @throws IgniteCheckedException If failed. + */ + private byte[] marshal(Object obj) throws IgniteCheckedException { + assert obj != null; + + return marsh.marshal(obj); + } + + /** + * + */ + private class ConnectionLossListener implements IgniteRunnable { + @Override public void run() { + + } + } + + /** + * + */ + private class ZKChildrenUpdateCallback implements AsyncCallback.Children2Callback { + @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { + + } + } + +} http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java index 3a2b733..0d4c86b 100644 --- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java +++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java @@ -19,7 +19,9 @@ package org.apache.ignite.spi.discovery.zk.internal; import java.util.List; import java.util.concurrent.Callable; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ThreadLocalRandom; +import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import org.apache.curator.test.TestingCluster; import org.apache.ignite.internal.IgniteInternalFuture; @@ -115,6 +117,43 @@ public class ZookeeperClientTest extends GridCommonAbstractTest { /** * @throws Exception If failed. */ + public void testConnectionLoss4() throws Exception { + startZK(1); + + CallbackFuture cb = new CallbackFuture(); + + final ZookeeperClient client = new ZookeeperClient(log, zkCluster.getConnectString(), 3000, cb); + + client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT); + + final CountDownLatch l = new CountDownLatch(1); + + client.getChildrenAsync("/apacheIgnite1", false, new AsyncCallback.Children2Callback() { + @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { + closeZK(); + + try { + client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT); + } + catch (ZookeeperClientFailedException e) { + info("Expected error: " + e); + + l.countDown(); + } + catch (Exception e) { + fail("Unexpected error: " + e); + } + } + }, null); + + assertTrue(l.await(10, TimeUnit.SECONDS)); + + cb.get(); + } + + /** + * @throws Exception If failed. + */ public void testReconnect1() throws Exception { startZK(1); @@ -146,6 +185,94 @@ public class ZookeeperClientTest extends GridCommonAbstractTest { /** * @throws Exception If failed. */ + public void testReconnect1_Callback() throws Exception { + startZK(1); + + ZookeeperClient client = new ZookeeperClient(log, zkCluster.getConnectString(), 30_000, null); + + client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT); + + zkCluster.getServers().get(0).stop(); + + final CountDownLatch l = new CountDownLatch(1); + + client.getChildrenAsync("/apacheIgnite1", false, new AsyncCallback.Children2Callback() { + @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { + info("Callback: " + rc); + + if (rc == 0) + l.countDown(); + } + }, null); + + IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() { + @Override public Void call() throws Exception { + U.sleep(2000); + + info("Restart zookeeper server"); + + zkCluster.getServers().get(0).restart(); + + info("Zookeeper server restarted"); + + return null; + } + }, "start-zk"); + + assertTrue(l.await(10, TimeUnit.SECONDS)); + + fut.get(); + } + + /** + * @throws Exception If failed. + */ + public void testReconnect1_InCallback() throws Exception { + startZK(1); + + final ZookeeperClient client = new ZookeeperClient(log, zkCluster.getConnectString(), 30_000, null); + + client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT); + + final CountDownLatch l = new CountDownLatch(1); + + client.getChildrenAsync("/apacheIgnite1", false, new AsyncCallback.Children2Callback() { + @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { + try { + zkCluster.getServers().get(0).stop(); + + IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() { + @Override public Void call() throws Exception { + U.sleep(2000); + + info("Restart zookeeper server"); + + zkCluster.getServers().get(0).restart(); + + info("Zookeeper server restarted"); + + return null; + } + }, "start-zk"); + + client.createIfNeeded("/apacheIgnite2", null, CreateMode.PERSISTENT); + + l.countDown(); + + fut.get(); + } + catch (Exception e) { + fail("Unexpected error: " + e); + } + } + }, null); + + assertTrue(l.await(10, TimeUnit.SECONDS)); + } + + /** + * @throws Exception If failed. + */ public void testReconnect2() throws Exception { startZK(1);
