Repository: ignite
Updated Branches:
  refs/heads/ignite-zk 98a171c68 -> 5793ffbe2


zk


Project: http://git-wip-us.apache.org/repos/asf/ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/ignite/commit/b389f38c
Tree: http://git-wip-us.apache.org/repos/asf/ignite/tree/b389f38c
Diff: http://git-wip-us.apache.org/repos/asf/ignite/diff/b389f38c

Branch: refs/heads/ignite-zk
Commit: b389f38cbc59f41dc1c95854684059f15b225b8c
Parents: 98a171c
Author: sboikov <[email protected]>
Authored: Fri Nov 17 09:33:30 2017 +0300
Committer: sboikov <[email protected]>
Committed: Fri Nov 17 11:14:05 2017 +0300

----------------------------------------------------------------------
 .../managers/discovery/ConsistentIdNode.java    |  33 +++
 .../discovery/GridDiscoveryManager.java         |   4 +-
 .../managers/discovery/JoiningNodesAware.java   |   2 +-
 .../tcp/internal/TcpDiscoveryNode.java          |   3 +-
 .../discovery/zk/ZookeeperDiscoverySpi2.java    |  64 +++--
 .../zk/internal/ZKJoiningNodeData.java          |  61 +++++
 .../discovery/zk/internal/ZkDiscoveryImpl.java  | 116 ---------
 .../discovery/zk/internal/ZookeeperClient.java  | 145 +++++++++--
 .../zk/internal/ZookeeperClusterNode.java       |   3 +
 .../zk/internal/ZookeeperDiscoveryImpl.java     | 238 +++++++++++++++++++
 .../zk/internal/ZookeeperClientTest.java        | 127 ++++++++++
 11 files changed, 636 insertions(+), 160 deletions(-)
----------------------------------------------------------------------


http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdNode.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdNode.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdNode.java
new file mode 100644
index 0000000..fc806ff
--- /dev/null
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/ConsistentIdNode.java
@@ -0,0 +1,33 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.internal.managers.discovery;
+
+import java.io.Serializable;
+import org.apache.ignite.cluster.ClusterNode;
+
+/**
+ * TODO ZK
+ */
+public interface ConsistentIdNode extends ClusterNode {
+    /**
+     * Sets consistent globally unique node ID which survives node restarts.
+     *
+     * @param consistentId Consistent globally unique node ID.
+     */
+    public void setConsistentId(Serializable consistentId);
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
index a6737dc..5262e32 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/GridDiscoveryManager.java
@@ -554,8 +554,8 @@ public class GridDiscoveryManager extends 
GridManagerAdapter<DiscoverySpi> {
                 for (IgniteInClosure<ClusterNode> lsnr : locNodeInitLsnrs)
                     lsnr.apply(locNode);
 
-                if (locNode instanceof TcpDiscoveryNode) {
-                    final TcpDiscoveryNode node = (TcpDiscoveryNode)locNode;
+                if (locNode instanceof ConsistentIdNode) {
+                    final ConsistentIdNode node = (ConsistentIdNode)locNode;
 
                     if (consistentId != null)
                         node.setConsistentId(consistentId);

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java
 
b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java
index 85128e4..7cd73f4 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/internal/managers/discovery/JoiningNodesAware.java
@@ -20,7 +20,7 @@ package org.apache.ignite.internal.managers.discovery;
 import java.util.UUID;
 
 /**
- *
+ * TODO ZK
  */
 public interface JoiningNodesAware {
     public boolean knownNode(UUID nodeId);

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
----------------------------------------------------------------------
diff --git 
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
 
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
index 38c2a1b..e5ec655 100644
--- 
a/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
+++ 
b/modules/core/src/main/java/org/apache/ignite/spi/discovery/tcp/internal/TcpDiscoveryNode.java
@@ -36,6 +36,7 @@ import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.configuration.IgniteConfiguration;
 import org.apache.ignite.internal.ClusterMetricsSnapshot;
 import org.apache.ignite.internal.IgniteNodeAttributes;
+import org.apache.ignite.internal.managers.discovery.ConsistentIdNode;
 import org.apache.ignite.internal.util.lang.GridMetadataAwareAdapter;
 import org.apache.ignite.internal.util.tostring.GridToStringExclude;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
@@ -58,7 +59,7 @@ import static 
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTE
  * <strong>This class is not intended for public use</strong> and has been made
  * <tt>public</tt> due to certain limitations of Java technology.
  */
-public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements 
ClusterNode,
+public class TcpDiscoveryNode extends GridMetadataAwareAdapter implements 
ConsistentIdNode,
     Comparable<TcpDiscoveryNode>, Externalizable {
     /** */
     private static final long serialVersionUID = 0L;

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java
----------------------------------------------------------------------
diff --git 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java
 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java
index 99810d0..0ba55c9 100644
--- 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java
+++ 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpi2.java
@@ -20,22 +20,17 @@ package org.apache.ignite.spi.discovery.zk;
 import java.io.Serializable;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
 import java.util.TreeMap;
 import java.util.UUID;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
 import org.apache.ignite.IgniteException;
 import org.apache.ignite.IgniteLogger;
 import org.apache.ignite.cluster.ClusterNode;
 import org.apache.ignite.internal.managers.discovery.JoiningNodesAware;
 import org.apache.ignite.internal.util.tostring.GridToStringInclude;
 import org.apache.ignite.internal.util.typedef.internal.S;
-import org.apache.ignite.internal.util.typedef.internal.U;
 import org.apache.ignite.lang.IgniteProductVersion;
-import org.apache.ignite.marshaller.jdk.JdkMarshaller;
 import org.apache.ignite.resources.LoggerResource;
 import org.apache.ignite.spi.IgniteSpiAdapter;
 import org.apache.ignite.spi.IgniteSpiException;
@@ -49,11 +44,8 @@ import 
org.apache.ignite.spi.discovery.DiscoverySpiHistorySupport;
 import org.apache.ignite.spi.discovery.DiscoverySpiListener;
 import org.apache.ignite.spi.discovery.DiscoverySpiNodeAuthenticator;
 import org.apache.ignite.spi.discovery.DiscoverySpiOrderSupport;
-import org.apache.ignite.spi.discovery.zk.internal.ZkDiscoveryImpl;
-import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClient;
+import org.apache.ignite.spi.discovery.zk.internal.ZookeeperDiscoveryImpl;
 import org.apache.ignite.spi.discovery.zk.internal.ZookeeperClusterNode;
-import org.apache.zookeeper.AsyncCallback;
-import org.apache.zookeeper.data.Stat;
 import org.jetbrains.annotations.Nullable;
 
 /**
@@ -64,7 +56,16 @@ import org.jetbrains.annotations.Nullable;
 @DiscoverySpiHistorySupport(true)
 public class ZookeeperDiscoverySpi2 extends IgniteSpiAdapter implements 
DiscoverySpi, JoiningNodesAware {
     /** */
-    private String connectString;
+    private String zkConnectionString;
+
+    /** */
+    private int sesTimeout = 5000;
+
+    /** */
+    private String basePath = "/apacheIgnite";
+
+    /** */
+    private String clusterName = "default";
 
     /** */
     private DiscoverySpiListener lsnr;
@@ -76,10 +77,7 @@ public class ZookeeperDiscoverySpi2 extends IgniteSpiAdapter 
implements Discover
     private DiscoveryMetricsProvider metricsProvider;
 
     /** */
-    private int sesTimeout = 5000;
-
-    /** */
-    private ZkDiscoveryImpl impl;
+    private ZookeeperDiscoveryImpl impl;
 
     /** */
     private ZookeeperClusterNode locNode;
@@ -97,6 +95,22 @@ public class ZookeeperDiscoverySpi2 extends IgniteSpiAdapter 
implements Discover
     @LoggerResource
     private IgniteLogger log;
 
+    public String getBasePath() {
+        return basePath;
+    }
+
+    public void setBasePath(String basePath) {
+        this.basePath = basePath;
+    }
+
+    public String getClusterName() {
+        return clusterName;
+    }
+
+    public void setClusterName(String clusterName) {
+        this.clusterName = clusterName;
+    }
+
     public int getSessionTimeout() {
         return sesTimeout;
     }
@@ -105,12 +119,12 @@ public class ZookeeperDiscoverySpi2 extends 
IgniteSpiAdapter implements Discover
         this.sesTimeout = sesTimeout;
     }
 
-    public String getConnectString() {
-        return connectString;
+    public String getZkConnectionString() {
+        return zkConnectionString;
     }
 
-    public void setConnectString(String connectString) {
-        this.connectString = connectString;
+    public void setZkConnectionString(String zkConnectionString) {
+        this.zkConnectionString = zkConnectionString;
     }
 
     /** */
@@ -318,14 +332,24 @@ public class ZookeeperDiscoverySpi2 extends 
IgniteSpiAdapter implements Discover
     @Override public void spiStart(@Nullable String igniteInstanceName) throws 
IgniteSpiException {
         initLocalNode();
 
+        log.info("Start Zookeeper discovery [zkConnectionString=" + 
zkConnectionString +
+            ", sesTimeout=" + sesTimeout +
+            ", basePath=" + basePath +
+            ", clusterName=" + clusterName + ']');
+
         DiscoveryDataBag discoDataBag = new DiscoveryDataBag(locNode.id());
 
         exchange.collect(discoDataBag);
 
-        impl = new ZkDiscoveryImpl(log, lsnr);
+        impl = new ZookeeperDiscoveryImpl(log,
+            basePath,
+            clusterName,
+            locNode,
+            lsnr,
+            exchange);
 
         try {
-            impl.joinTopology(igniteInstanceName, connectString, sesTimeout);
+            impl.joinTopology(igniteInstanceName, zkConnectionString, 
sesTimeout);
         }
         catch (InterruptedException e) {
             Thread.currentThread().interrupt();

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZKJoiningNodeData.java
----------------------------------------------------------------------
diff --git 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZKJoiningNodeData.java
 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZKJoiningNodeData.java
new file mode 100644
index 0000000..278d45e
--- /dev/null
+++ 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZKJoiningNodeData.java
@@ -0,0 +1,61 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.io.Serializable;
+import java.util.Map;
+
+/**
+ *
+ */
+class ZKJoiningNodeData implements Serializable {
+    /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
+    private final ZookeeperClusterNode node;
+
+    /** */
+    private final Map<Integer, Serializable> joiningNodeData;
+
+    /**
+     * @param node Node.
+     * @param joiningNodeData Discovery data.
+     */
+    ZKJoiningNodeData(ZookeeperClusterNode node, Map<Integer, Serializable> 
joiningNodeData) {
+        assert node != null && node.id() != null : node;
+        assert joiningNodeData != null;
+
+        this.node = node;
+        this.joiningNodeData = joiningNodeData;
+    }
+
+    /**
+     * @return Node.
+     */
+    ZookeeperClusterNode node() {
+        return node;
+    }
+
+    /**
+     * @return Discovery data.
+     */
+    Map<Integer, Serializable> joiningNodeData() {
+        return joiningNodeData;
+    }
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryImpl.java
----------------------------------------------------------------------
diff --git 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryImpl.java
 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryImpl.java
deleted file mode 100644
index 4c7a7bf..0000000
--- 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZkDiscoveryImpl.java
+++ /dev/null
@@ -1,116 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements.  See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License.  You may obtain a copy of the License at
- *
- *      http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.ignite.spi.discovery.zk.internal;
-
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import org.apache.ignite.IgniteCheckedException;
-import org.apache.ignite.IgniteException;
-import org.apache.ignite.IgniteLogger;
-import org.apache.ignite.internal.util.future.GridFutureAdapter;
-import org.apache.ignite.internal.util.typedef.internal.U;
-import org.apache.ignite.lang.IgniteRunnable;
-import org.apache.ignite.marshaller.jdk.JdkMarshaller;
-import org.apache.ignite.spi.IgniteSpiException;
-import org.apache.ignite.spi.discovery.DiscoverySpiListener;
-import org.apache.zookeeper.AsyncCallback;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.data.Stat;
-
-/**
- *
- */
-public class ZkDiscoveryImpl {
-    /** */
-    private static final String IGNITE_PATH = "/apache-ignite";
-
-    /** */
-    private final JdkMarshaller marsh = new JdkMarshaller();
-
-    /** */
-    private final IgniteLogger log;
-
-    /** */
-    private final DiscoverySpiListener lsnr;
-
-    /** */
-    private ZookeeperClient zkClient;
-
-    /** */
-    private final GridFutureAdapter<Void> joinFut = new GridFutureAdapter<>();
-
-    public ZkDiscoveryImpl(IgniteLogger log, DiscoverySpiListener lsnr) {
-        this.log = log.getLogger(getClass());
-        this.lsnr = lsnr;
-    }
-
-    public void joinTopology(String igniteInstanceName, String connectString, 
int sesTimeout)
-        throws InterruptedException {
-        try {
-            zkClient = new ZookeeperClient(igniteInstanceName,
-                log,
-                connectString,
-                sesTimeout,
-                new ConnectionLossListener());
-        }
-        catch (Exception e) {
-            throw new IgniteSpiException("Failed to create Zookeeper client", 
e);
-        }
-
-        try {
-            zkClient.createIfNeeded(IGNITE_PATH, null, CreateMode.PERSISTENT);
-        }
-        catch (ZookeeperClientFailedException e) {
-            throw new IgniteSpiException("Failed to initialize Zookeeper 
nodes", e);
-        }
-    }
-
-    /**
-     *
-     */
-    public void stop() {
-        if (zkClient != null)
-            zkClient.close();
-    }
-
-    private <T> T unmarshal(byte[] data) throws IgniteCheckedException {
-        return marsh.unmarshal(data, null);
-    }
-
-    private byte[] marshal(Object obj) throws IgniteCheckedException {
-        return marsh.marshal(obj);
-    }
-
-    /**
-     *
-     */
-    private class ConnectionLossListener implements IgniteRunnable {
-        @Override public void run() {
-
-        }
-    }
-
-    /**
-     *
-     */
-    private class ZKChildrenUpdateCallback implements 
AsyncCallback.Children2Callback {
-        @Override public void processResult(int rc, String path, Object ctx, 
List<String> children, Stat stat) {
-
-        }
-    }
-}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
----------------------------------------------------------------------
diff --git 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
index 8f3f074..37b153d 100644
--- 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
+++ 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java
@@ -17,6 +17,7 @@
 
 package org.apache.ignite.spi.discovery.zk.internal;
 
+import java.util.ArrayDeque;
 import java.util.List;
 import java.util.Timer;
 import java.util.TimerTask;
@@ -31,6 +32,7 @@ import org.apache.zookeeper.Watcher;
 import org.apache.zookeeper.ZooDefs;
 import org.apache.zookeeper.ZooKeeper;
 import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
 
 /**
  *
@@ -58,7 +60,7 @@ public class ZookeeperClient implements Watcher {
     private long connLossTimeout;
 
     /** */
-    private long connectStartTime;
+    private volatile long connStartTime;
 
     /** */
     private final Object stateMux = new Object();
@@ -69,6 +71,9 @@ public class ZookeeperClient implements Watcher {
     /** */
     private final Timer connTimer;
 
+    /** */
+    private final ArrayDeque<ZkAsyncOperation> retryQ = new ArrayDeque<>();
+
     ZookeeperClient(IgniteLogger log, String connectString, int sesTimeout, 
IgniteRunnable connLostC) throws Exception {
         this(null, log, connectString, sesTimeout, connLostC);
     }
@@ -86,7 +91,7 @@ public class ZookeeperClient implements Watcher {
 
         connLossTimeout = sesTimeout;
 
-        connectStartTime = System.currentTimeMillis();
+        connStartTime = System.currentTimeMillis();
 
         String threadName = Thread.currentThread().getName();
 
@@ -108,7 +113,7 @@ public class ZookeeperClient implements Watcher {
     /** {@inheritDoc} */
     @Override public void process(WatchedEvent evt) {
         if (evt.getType() == Event.EventType.None) {
-            boolean connLost = false;
+            ConnectionState newState;
 
             synchronized (stateMux) {
                 if (state == ConnectionState.Lost) {
@@ -117,8 +122,6 @@ public class ZookeeperClient implements Watcher {
                     return;
                 }
 
-                ConnectionState newState;
-
                 Event.KeeperState zkState = evt.getState();
 
                 switch (zkState) {
@@ -149,24 +152,28 @@ public class ZookeeperClient implements Watcher {
                     state = newState;
 
                     if (newState == ConnectionState.Disconnected) {
-                        connectStartTime = System.currentTimeMillis();
+                        connStartTime = System.currentTimeMillis();
 
                         scheduleConnectionCheck();
                     }
                     else if (newState == ConnectionState.Connected)
                         stateMux.notifyAll();
                     else {
-                        assert state == ConnectionState.Lost;
+                        assert state == ConnectionState.Lost : state;
 
                         closeClient();
-
-                        connLost = true;
                     }
                 }
+                else
+                    return;
             }
 
-            if (connLost)
+            if (newState == ConnectionState.Lost)
                 notifyConnectionLost();
+            else if (newState == ConnectionState.Connected) {
+                for (ZkAsyncOperation op : retryQ)
+                    op.execute();
+            }
         }
     }
 
@@ -178,12 +185,33 @@ public class ZookeeperClient implements Watcher {
             connLostC.run();
     }
 
+    /**
+     * @param path
+     * @return
+     * @throws ZookeeperClientFailedException
+     * @throws InterruptedException
+     */
+    public boolean exists(String path) throws ZookeeperClientFailedException, 
InterruptedException {
+        for (;;) {
+            long connStartTime = this.connStartTime;
+
+            try {
+                return zk.exists(path, false) != null;
+            }
+            catch (Exception e) {
+                onZookeeperError(connStartTime, e);
+            }
+        }
+    }
+
     public void createIfNeeded(String path, byte[] data, CreateMode createMode)
         throws ZookeeperClientFailedException, InterruptedException {
         if (data == null)
             data = EMPTY_BYTES;
 
         for (;;) {
+            long connStartTime = this.connStartTime;
+
             try {
                 zk.create(path, data, ZK_ACL, createMode);
 
@@ -195,13 +223,15 @@ public class ZookeeperClient implements Watcher {
                 break;
             }
             catch (Exception e) {
-                onZookeeperError(e);
+                onZookeeperError(connStartTime, e);
             }
         }
     }
 
     public void getChildrenAsync(String path, boolean watch, 
AsyncCallback.Children2Callback cb, Object ctx) {
-        zk.getChildren(path, watch, cb, ctx);
+        GetChildrenOperation op = new GetChildrenOperation(path, watch, cb, 
ctx);
+
+        zk.getChildren(path, watch, new ChildreCallbackWrapper(op), ctx);
     }
 
     /**
@@ -214,7 +244,7 @@ public class ZookeeperClient implements Watcher {
     /**
      * @param e Error.
      */
-    private void onZookeeperError(Exception e) throws 
ZookeeperClientFailedException, InterruptedException {
+    private void onZookeeperError(long prevConnStartTime, Exception e) throws 
ZookeeperClientFailedException, InterruptedException {
         ZookeeperClientFailedException err = null;
 
         synchronized (stateMux) {
@@ -229,22 +259,24 @@ public class ZookeeperClient implements Watcher {
                 throw new ZookeeperClientFailedException(e);
             }
 
-            if (e instanceof KeeperException.ConnectionLossException) {
+            boolean retry = (e instanceof KeeperException) && 
needRetry(((KeeperException)e).code().intValue());
+
+            if (retry) {
                 long remainingTime;
 
-                if (state == ConnectionState.Connected) {
+                if (state == ConnectionState.Connected && connStartTime == 
prevConnStartTime) {
                     state = ConnectionState.Disconnected;
 
-                    connectStartTime = System.currentTimeMillis();
+                    connStartTime = System.currentTimeMillis();
 
                     remainingTime = connLossTimeout;
                 }
                 else {
-                    assert connectStartTime != 0;
+                    assert connStartTime != 0;
 
                     assert state == ConnectionState.Disconnected;
 
-                    remainingTime = connLossTimeout - 
(System.currentTimeMillis() - connectStartTime);
+                    remainingTime = connLossTimeout - 
(System.currentTimeMillis() - connStartTime);
 
                     if (remainingTime <= 0) {
                         state = ConnectionState.Lost;
@@ -286,6 +318,51 @@ public class ZookeeperClient implements Watcher {
     }
 
     /**
+     * @param code Zookeeper error code.
+     * @return {@code True} if can retry operation.
+     */
+    private boolean needRetry(int code) {
+        // TODO ZL: other codes.
+        return code == KeeperException.Code.CONNECTIONLOSS.intValue();
+    }
+
+    /**
+     *
+     */
+    interface ZkAsyncOperation {
+        void execute();
+    }
+
+    /**
+     *
+     */
+    class GetChildrenOperation implements ZkAsyncOperation {
+        /** */
+        private final String path;
+
+        /** */
+        private final boolean watch;
+
+        /** */
+        private final AsyncCallback.Children2Callback cb;
+
+        /** */
+        private final Object ctx;
+
+        public GetChildrenOperation(String path, boolean watch, 
AsyncCallback.Children2Callback cb, Object ctx) {
+            this.path = path;
+            this.watch = watch;
+            this.cb = cb;
+            this.ctx = ctx;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void execute() {
+            getChildrenAsync(path, watch, cb, ctx);
+        }
+    }
+
+    /**
      *
      */
     private void closeClient() {
@@ -305,7 +382,35 @@ public class ZookeeperClient implements Watcher {
     private void scheduleConnectionCheck() {
         assert state == ConnectionState.Disconnected : state;
 
-        connTimer.schedule(new ConnectionTimeoutTask(connectStartTime), 
connLossTimeout);
+        connTimer.schedule(new ConnectionTimeoutTask(connStartTime), 
connLossTimeout);
+    }
+
+    /**
+     *
+     */
+    class ChildreCallbackWrapper implements AsyncCallback.Children2Callback {
+        /** */
+        private final GetChildrenOperation op;
+
+        /**
+         * @param op Operation.
+         */
+        private ChildreCallbackWrapper(GetChildrenOperation op) {
+            this.op = op;
+        }
+
+        /** {@inheritDoc} */
+        @Override public void processResult(int rc, String path, Object ctx, 
List<String> children, Stat stat) {
+            if (needRetry(rc)) {
+                U.warn(log, "Failed to execute async operation, connection 
lost. Will retry after connection restore [path=" + path + ']');
+
+                retryQ.add(op);
+            }
+            else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue())
+                U.warn(log, "Failed to execute async operation, connection 
lost [path=" + path + ']');
+            else
+                op.cb.processResult(rc, path, ctx, children, stat);
+        }
     }
 
     /**
@@ -328,7 +433,7 @@ public class ZookeeperClient implements Watcher {
 
             synchronized (stateMux) {
                 if (state == ConnectionState.Disconnected &&
-                    ZookeeperClient.this.connectStartTime == connectStartTime) 
{
+                    ZookeeperClient.this.connStartTime == connectStartTime) {
 
                     state = ConnectionState.Lost;
 

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
----------------------------------------------------------------------
diff --git 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
index 7f0e6b7..3f8ee27 100644
--- 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
+++ 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java
@@ -41,6 +41,9 @@ import static 
org.apache.ignite.internal.IgniteNodeAttributes.ATTR_NODE_CONSISTE
  */
 public class ZookeeperClusterNode implements ClusterNode, Serializable {
     /** */
+    private static final long serialVersionUID = 0L;
+
+    /** */
     private UUID id;
 
     /** */

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
----------------------------------------------------------------------
diff --git 
a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
new file mode 100644
index 0000000..aae2d06
--- /dev/null
+++ 
b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java
@@ -0,0 +1,238 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements.  See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License.  You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.ignite.spi.discovery.zk.internal;
+
+import java.util.List;
+import org.apache.curator.utils.PathUtils;
+import org.apache.ignite.IgniteCheckedException;
+import org.apache.ignite.IgniteLogger;
+import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException;
+import org.apache.ignite.internal.util.future.GridFutureAdapter;
+import org.apache.ignite.internal.util.typedef.F;
+import org.apache.ignite.internal.util.typedef.internal.U;
+import org.apache.ignite.lang.IgniteRunnable;
+import org.apache.ignite.marshaller.jdk.JdkMarshaller;
+import org.apache.ignite.spi.IgniteSpiException;
+import org.apache.ignite.spi.discovery.DiscoveryDataBag;
+import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange;
+import org.apache.ignite.spi.discovery.DiscoverySpiListener;
+import org.apache.zookeeper.AsyncCallback;
+import org.apache.zookeeper.data.Stat;
+
+import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL;
+import static org.apache.zookeeper.CreateMode.PERSISTENT;
+
+/**
+ * TODO ZK: check if compression makes sense.
+ */
+public class ZookeeperDiscoveryImpl {
+    /** */
+    private static final String JOIN_DATA_DIR = "joinData";
+
+    /** */
+    private static final String ALIVE_NODES_DIR = "alive";
+
+    /** */
+    private final String basePath;
+
+    /** */
+    private final String clusterName;
+
+    /** */
+    private final JdkMarshaller marsh = new JdkMarshaller();
+
+    /** */
+    private final IgniteLogger log;
+
+    /** */
+    private final ZookeeperClusterNode locNode;
+
+    /** */
+    private final DiscoverySpiListener lsnr;
+
+    /** */
+    private final DiscoverySpiDataExchange exchange;
+
+    /** */
+    private ZookeeperClient zkClient;
+
+    /** */
+    private final GridFutureAdapter<Void> joinFut = new GridFutureAdapter<>();
+
+    public ZookeeperDiscoveryImpl(IgniteLogger log,
+        String basePath,
+        String clusterName,
+        ZookeeperClusterNode locNode,
+        DiscoverySpiListener lsnr,
+        DiscoverySpiDataExchange exchange) {
+        assert locNode.id() != null : locNode;
+
+        if (F.isEmpty(clusterName))
+            throw new IllegalArgumentException("Cluster name is empty.");
+
+        PathUtils.validatePath(basePath);
+
+        this.log = log.getLogger(getClass());
+        this.basePath = basePath;
+        this.clusterName = clusterName;
+        this.locNode = locNode;
+        this.lsnr = lsnr;
+        this.exchange = exchange;
+    }
+
+    /**
+     * @param path Relative path.
+     * @return Full path.
+     */
+    private String zkPath(String path) {
+        return basePath + "/" + clusterName + "/" + path;
+    }
+
+    /**
+     * @param igniteInstanceName
+     * @param connectString
+     * @param sesTimeout
+     * @throws InterruptedException If interrupted.
+     */
+    public void joinTopology(String igniteInstanceName, String connectString, 
int sesTimeout)
+        throws InterruptedException
+    {
+        DiscoveryDataBag discoDataBag = new DiscoveryDataBag(locNode.id());
+
+        exchange.collect(discoDataBag);
+
+        ZKJoiningNodeData joinData = new ZKJoiningNodeData(locNode, 
discoDataBag.joiningNodeData());
+
+        byte[] joinDataBytes;
+
+        try {
+            joinDataBytes = marshal(joinData);
+        }
+        catch (Exception e) {
+            throw new IgniteSpiException("Failed to marshal joining node 
data", e);
+        }
+
+        try {
+            zkClient = new ZookeeperClient(igniteInstanceName,
+                log,
+                connectString,
+                sesTimeout,
+                new ConnectionLossListener());
+        }
+        catch (Exception e) {
+            throw new IgniteSpiException("Failed to create Zookeeper client", 
e);
+        }
+
+        try {
+            initZkNodes();
+
+            // TODO ZK: handle max size
+            zkClient.createIfNeeded(zkPath(JOIN_DATA_DIR) + "/" + locNode.id() 
+ "-", joinDataBytes, EPHEMERAL_SEQUENTIAL);
+
+            zkClient.createIfNeeded(zkPath(ALIVE_NODES_DIR) + "/" + 
locNode.id() + "-", null, EPHEMERAL_SEQUENTIAL);
+
+            zkClient.getChildrenAsync(zkPath(ALIVE_NODES_DIR), false, new 
AsyncCallback.Children2Callback() {
+                @Override public void processResult(int rc, String path, 
Object ctx, List<String> children, Stat stat) {
+                    assert rc == 0 : rc;
+
+
+                }
+            }, null);
+
+            for (;;) {
+                try {
+                    joinFut.get(10_000);
+
+                    break;
+                }
+                catch (IgniteFutureTimeoutCheckedException e) {
+                    U.warn(log, "Waiting for local join event [nodeId=" + 
locNode.id() + ", name=" + igniteInstanceName + ']');
+                }
+                catch (Exception e) {
+                    throw new IgniteSpiException("Failed to join cluster", e);
+                }
+            }
+        }
+        catch (ZookeeperClientFailedException e) {
+            throw new IgniteSpiException("Failed to initialize Zookeeper 
nodes", e);
+        }
+    }
+
+    private void initZkNodes() throws ZookeeperClientFailedException, 
InterruptedException {
+        // TODO ZK: use multi.
+        String joinDir = zkPath(JOIN_DATA_DIR);
+
+        if (zkClient.exists(joinDir))
+            return; // Assume all others dirs are created.
+
+        zkClient.createIfNeeded(basePath, null, PERSISTENT);
+
+        zkClient.createIfNeeded(zkPath(ALIVE_NODES_DIR), null, PERSISTENT);
+
+        zkClient.createIfNeeded(joinDir, null, PERSISTENT);
+    }
+
+    /**
+     *
+     */
+    public void stop() {
+        if (zkClient != null)
+            zkClient.close();
+    }
+
+    /**
+     * @param bytes Bytes.
+     * @return Unmarshalled object.
+     * @throws IgniteCheckedException If failed.
+     */
+    private <T> T unmarshal(byte[] bytes) throws IgniteCheckedException {
+        assert bytes != null && bytes.length > 0;
+
+        return marsh.unmarshal(bytes, null);
+    }
+
+    /**
+     * @param obj Object.
+     * @return Bytes.
+     * @throws IgniteCheckedException If failed.
+     */
+    private byte[] marshal(Object obj) throws IgniteCheckedException {
+        assert obj != null;
+
+        return marsh.marshal(obj);
+    }
+
+    /**
+     *
+     */
+    private class ConnectionLossListener implements IgniteRunnable {
+        @Override public void run() {
+
+        }
+    }
+
+    /**
+     *
+     */
+    private class ZKChildrenUpdateCallback implements 
AsyncCallback.Children2Callback {
+        @Override public void processResult(int rc, String path, Object ctx, 
List<String> children, Stat stat) {
+
+        }
+    }
+
+}

http://git-wip-us.apache.org/repos/asf/ignite/blob/b389f38c/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
----------------------------------------------------------------------
diff --git 
a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
 
b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
index 3a2b733..0d4c86b 100644
--- 
a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
+++ 
b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClientTest.java
@@ -19,7 +19,9 @@ package org.apache.ignite.spi.discovery.zk.internal;
 
 import java.util.List;
 import java.util.concurrent.Callable;
+import java.util.concurrent.CountDownLatch;
 import java.util.concurrent.ThreadLocalRandom;
+import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
 import org.apache.curator.test.TestingCluster;
 import org.apache.ignite.internal.IgniteInternalFuture;
@@ -115,6 +117,43 @@ public class ZookeeperClientTest extends 
GridCommonAbstractTest {
     /**
      * @throws Exception If failed.
      */
+    public void testConnectionLoss4() throws Exception {
+        startZK(1);
+
+        CallbackFuture cb = new CallbackFuture();
+
+        final ZookeeperClient client = new ZookeeperClient(log, 
zkCluster.getConnectString(), 3000, cb);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        client.getChildrenAsync("/apacheIgnite1", false, new 
AsyncCallback.Children2Callback() {
+            @Override public void processResult(int rc, String path, Object 
ctx, List<String> children, Stat stat) {
+                closeZK();
+
+                try {
+                    client.createIfNeeded("/apacheIgnite2", null, 
CreateMode.PERSISTENT);
+                }
+                catch (ZookeeperClientFailedException e) {
+                    info("Expected error: " + e);
+
+                    l.countDown();
+                }
+                catch (Exception e) {
+                    fail("Unexpected error: " + e);
+                }
+            }
+        }, null);
+
+        assertTrue(l.await(10, TimeUnit.SECONDS));
+
+        cb.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
     public void testReconnect1() throws Exception {
         startZK(1);
 
@@ -146,6 +185,94 @@ public class ZookeeperClientTest extends 
GridCommonAbstractTest {
     /**
      * @throws Exception If failed.
      */
+    public void testReconnect1_Callback() throws Exception {
+        startZK(1);
+
+        ZookeeperClient client = new ZookeeperClient(log, 
zkCluster.getConnectString(), 30_000, null);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        zkCluster.getServers().get(0).stop();
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        client.getChildrenAsync("/apacheIgnite1", false, new 
AsyncCallback.Children2Callback() {
+            @Override public void processResult(int rc, String path, Object 
ctx, List<String> children, Stat stat) {
+                info("Callback: " + rc);
+
+                if (rc == 0)
+                    l.countDown();
+            }
+        }, null);
+
+        IgniteInternalFuture fut = GridTestUtils.runAsync(new Callable<Void>() 
{
+            @Override public Void call() throws Exception {
+                U.sleep(2000);
+
+                info("Restart zookeeper server");
+
+                zkCluster.getServers().get(0).restart();
+
+                info("Zookeeper server restarted");
+
+                return null;
+            }
+        }, "start-zk");
+
+        assertTrue(l.await(10, TimeUnit.SECONDS));
+
+        fut.get();
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
+    public void testReconnect1_InCallback() throws Exception {
+        startZK(1);
+
+        final ZookeeperClient client = new ZookeeperClient(log, 
zkCluster.getConnectString(), 30_000, null);
+
+        client.createIfNeeded("/apacheIgnite1", null, CreateMode.PERSISTENT);
+
+        final CountDownLatch l = new CountDownLatch(1);
+
+        client.getChildrenAsync("/apacheIgnite1", false, new 
AsyncCallback.Children2Callback() {
+            @Override public void processResult(int rc, String path, Object 
ctx, List<String> children, Stat stat) {
+                try {
+                    zkCluster.getServers().get(0).stop();
+
+                    IgniteInternalFuture fut = GridTestUtils.runAsync(new 
Callable<Void>() {
+                        @Override public Void call() throws Exception {
+                            U.sleep(2000);
+
+                            info("Restart zookeeper server");
+
+                            zkCluster.getServers().get(0).restart();
+
+                            info("Zookeeper server restarted");
+
+                            return null;
+                        }
+                    }, "start-zk");
+
+                    client.createIfNeeded("/apacheIgnite2", null, 
CreateMode.PERSISTENT);
+
+                    l.countDown();
+
+                    fut.get();
+                }
+                catch (Exception e) {
+                    fail("Unexpected error: " + e);
+                }
+            }
+        }, null);
+
+        assertTrue(l.await(10, TimeUnit.SECONDS));
+    }
+
+    /**
+     * @throws Exception If failed.
+     */
     public void testReconnect2() throws Exception {
         startZK(1);
 

Reply via email to