http://git-wip-us.apache.org/repos/asf/ignite/blob/5793ffbe/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java index 37b153d..0b17cab 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClient.java @@ -74,18 +74,32 @@ public class ZookeeperClient implements Watcher { /** */ private final ArrayDeque<ZkAsyncOperation> retryQ = new ArrayDeque<>(); + /** + * @param log Logger. + * @param connectString ZK connection string. + * @param sesTimeout ZK session timeout. + * @param connLostC Lost connection callback. + * @throws Exception If failed. + */ ZookeeperClient(IgniteLogger log, String connectString, int sesTimeout, IgniteRunnable connLostC) throws Exception { this(null, log, connectString, sesTimeout, connLostC); } /** + * @param igniteInstanceName Ignite instance name. * @param log Logger. * @param connectString ZK connection string. * @param sesTimeout ZK session timeout. + * @param connLostC Lost connection callback. * @throws Exception If failed. */ - ZookeeperClient(String igniteInstanceName, IgniteLogger log, String connectString, int sesTimeout, IgniteRunnable connLostC) - throws Exception { + ZookeeperClient(String igniteInstanceName, + IgniteLogger log, + String connectString, + int sesTimeout, + IgniteRunnable connLostC) + throws Exception + { this.log = log.getLogger(getClass()); this.connLostC = connLostC; @@ -185,13 +199,7 @@ public class ZookeeperClient implements Watcher { connLostC.run(); } - /** - * @param path - * @return - * @throws ZookeeperClientFailedException - * @throws InterruptedException - */ - public boolean exists(String path) throws ZookeeperClientFailedException, InterruptedException { + boolean exists(String path) throws ZookeeperClientFailedException, InterruptedException { for (;;) { long connStartTime = this.connStartTime; @@ -204,7 +212,7 @@ public class ZookeeperClient implements Watcher { } } - public void createIfNeeded(String path, byte[] data, CreateMode createMode) + String createIfNeeded(String path, byte[] data, CreateMode createMode) throws ZookeeperClientFailedException, InterruptedException { if (data == null) data = EMPTY_BYTES; @@ -213,14 +221,12 @@ public class ZookeeperClient implements Watcher { long connStartTime = this.connStartTime; try { - zk.create(path, data, ZK_ACL, createMode); - - break; + return zk.create(path, data, ZK_ACL, createMode); } catch (KeeperException.NodeExistsException e) { log.info("Node already exists: " + path); - break; + return path; } catch (Exception e) { onZookeeperError(connStartTime, e); @@ -228,10 +234,66 @@ public class ZookeeperClient implements Watcher { } } - public void getChildrenAsync(String path, boolean watch, AsyncCallback.Children2Callback cb, Object ctx) { - GetChildrenOperation op = new GetChildrenOperation(path, watch, cb, ctx); + List<String> getChildren(String path) + throws ZookeeperClientFailedException, InterruptedException + { + for (;;) { + long connStartTime = this.connStartTime; - zk.getChildren(path, watch, new ChildreCallbackWrapper(op), ctx); + try { + return zk.getChildren(path, false); + } + catch (Exception e) { + onZookeeperError(connStartTime, e); + } + } + } + + void setData(String path, byte[] data, int ver) + throws ZookeeperClientFailedException, InterruptedException + { + for (;;) { + long connStartTime = this.connStartTime; + + try { + zk.setData(path, data, ver); + + return; + } + catch (Exception e) { + onZookeeperError(connStartTime, e); + } + } + } + + byte[] getData(String path) + throws KeeperException.NoNodeException, ZookeeperClientFailedException, InterruptedException + { + for (;;) { + long connStartTime = this.connStartTime; + + try { + return zk.getData(path, false, null); + } + catch (KeeperException.NoNodeException e) { + throw e; + } + catch (Exception e) { + onZookeeperError(connStartTime, e); + } + } + } + + void getChildrenAsync(String path, Watcher watcher, AsyncCallback.Children2Callback cb) { + GetChildrenOperation op = new GetChildrenOperation(path, watcher, cb); + + zk.getChildren(path, watcher, new ChildreCallbackWrapper(op), null); + } + + void getDataAsync(String path, Watcher watcher, AsyncCallback.DataCallback cb) { + GetDataOperation op = new GetDataOperation(path, watcher, cb); + + zk.getData(path, watcher, new DataCallbackWrapper(op), null); } /** @@ -244,7 +306,9 @@ public class ZookeeperClient implements Watcher { /** * @param e Error. */ - private void onZookeeperError(long prevConnStartTime, Exception e) throws ZookeeperClientFailedException, InterruptedException { + private void onZookeeperError(long prevConnStartTime, Exception e) + throws ZookeeperClientFailedException, InterruptedException + { ZookeeperClientFailedException err = null; synchronized (stateMux) { @@ -330,6 +394,9 @@ public class ZookeeperClient implements Watcher { * */ interface ZkAsyncOperation { + /** + * + */ void execute(); } @@ -341,24 +408,45 @@ public class ZookeeperClient implements Watcher { private final String path; /** */ - private final boolean watch; + private final Watcher watcher; /** */ private final AsyncCallback.Children2Callback cb; + GetChildrenOperation(String path, Watcher watcher, AsyncCallback.Children2Callback cb) { + this.path = path; + this.watcher = watcher; + this.cb = cb; + } + + /** {@inheritDoc} */ + @Override public void execute() { + getChildrenAsync(path, watcher, cb); + } + } + + /** + * + */ + class GetDataOperation implements ZkAsyncOperation { /** */ - private final Object ctx; + private final String path; - public GetChildrenOperation(String path, boolean watch, AsyncCallback.Children2Callback cb, Object ctx) { + /** */ + private final Watcher watcher; + + /** */ + private final AsyncCallback.DataCallback cb; + + GetDataOperation(String path, Watcher watcher, AsyncCallback.DataCallback cb) { this.path = path; - this.watch = watch; + this.watcher = watcher; this.cb = cb; - this.ctx = ctx; } /** {@inheritDoc} */ @Override public void execute() { - getChildrenAsync(path, watch, cb, ctx); + getDataAsync(path, watcher, cb); } } @@ -416,6 +504,34 @@ public class ZookeeperClient implements Watcher { /** * */ + class DataCallbackWrapper implements AsyncCallback.DataCallback { + /** */ + private final GetDataOperation op; + + /** + * @param op Operation. + */ + private DataCallbackWrapper(GetDataOperation op) { + this.op = op; + } + + /** {@inheritDoc} */ + @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) { + if (needRetry(rc)) { + U.warn(log, "Failed to execute async operation, connection lost. Will retry after connection restore [path=" + path + ']'); + + retryQ.add(op); + } + else if (rc == KeeperException.Code.SESSIONEXPIRED.intValue()) + U.warn(log, "Failed to execute async operation, connection lost [path=" + path + ']'); + else + op.cb.processResult(rc, path, ctx, data, stat); + } + } + + /** + * + */ private class ConnectionTimeoutTask extends TimerTask { /** */ private final long connectStartTime;
http://git-wip-us.apache.org/repos/asf/ignite/blob/5793ffbe/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java index 3f8ee27..dc35752 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperClusterNode.java @@ -50,7 +50,7 @@ public class ZookeeperClusterNode implements ClusterNode, Serializable { private Serializable consistentId; /** */ - private long internalOrder; + private int internalId; /** */ private long order; @@ -160,20 +160,21 @@ public class ZookeeperClusterNode implements ClusterNode, Serializable { return order; } - public long internalOrder() { - return internalOrder; + /** + * @return Internal ID corresponds to Zookeeper sequential node. + */ + public int internalId() { + return internalId; } /** - * @param internalOrder Order of the node. + * @param internalId Internal ID corresponds to Zookeeper sequential node. */ - public void internalOrder(long internalOrder) { - assert internalOrder > 0 : internalOrder; - - this.internalOrder = internalOrder; + void internalId(int internalId) { + this.internalId = internalId; } - public void order(long order) { + void order(long order) { assert order > 0 : order; this.order = order; http://git-wip-us.apache.org/repos/asf/ignite/blob/5793ffbe/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java index aae2d06..14506a1 100644 --- a/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java +++ b/modules/zookeeper/src/main/java/org/apache/ignite/spi/discovery/zk/internal/ZookeeperDiscoveryImpl.java @@ -17,10 +17,22 @@ package org.apache.ignite.spi.discovery.zk.internal; +import java.io.Serializable; +import java.util.ArrayList; +import java.util.Collection; +import java.util.Collections; +import java.util.HashSet; import java.util.List; +import java.util.Map; +import java.util.TreeMap; +import java.util.UUID; import org.apache.curator.utils.PathUtils; import org.apache.ignite.IgniteCheckedException; +import org.apache.ignite.IgniteException; +import org.apache.ignite.IgniteInterruptedException; import org.apache.ignite.IgniteLogger; +import org.apache.ignite.cluster.ClusterNode; +import org.apache.ignite.events.EventType; import org.apache.ignite.internal.IgniteFutureTimeoutCheckedException; import org.apache.ignite.internal.util.future.GridFutureAdapter; import org.apache.ignite.internal.util.typedef.F; @@ -29,11 +41,17 @@ import org.apache.ignite.lang.IgniteRunnable; import org.apache.ignite.marshaller.jdk.JdkMarshaller; import org.apache.ignite.spi.IgniteSpiException; import org.apache.ignite.spi.discovery.DiscoveryDataBag; +import org.apache.ignite.spi.discovery.DiscoverySpiCustomMessage; import org.apache.ignite.spi.discovery.DiscoverySpiDataExchange; import org.apache.ignite.spi.discovery.DiscoverySpiListener; import org.apache.zookeeper.AsyncCallback; +import org.apache.zookeeper.KeeperException; +import org.apache.zookeeper.WatchedEvent; +import org.apache.zookeeper.Watcher; import org.apache.zookeeper.data.Stat; +import org.jetbrains.annotations.Nullable; +import static org.apache.zookeeper.CreateMode.EPHEMERAL; import static org.apache.zookeeper.CreateMode.EPHEMERAL_SEQUENTIAL; import static org.apache.zookeeper.CreateMode.PERSISTENT; @@ -42,19 +60,10 @@ import static org.apache.zookeeper.CreateMode.PERSISTENT; */ public class ZookeeperDiscoveryImpl { /** */ - private static final String JOIN_DATA_DIR = "joinData"; - - /** */ - private static final String ALIVE_NODES_DIR = "alive"; - - /** */ - private final String basePath; + private final JdkMarshaller marsh = new JdkMarshaller(); /** */ - private final String clusterName; - - /** */ - private final JdkMarshaller marsh = new JdkMarshaller(); + private final ZkPaths zkPaths; /** */ private final IgniteLogger log; @@ -74,6 +83,21 @@ public class ZookeeperDiscoveryImpl { /** */ private final GridFutureAdapter<Void> joinFut = new GridFutureAdapter<>(); + /** */ + private final ZkWatcher watcher; + + /** */ + private final ZKChildrenCallback childrenCallback; + + /** */ + private final ZkDataCallback dataCallback; + + /** */ + private final ZkClusterNodes top = new ZkClusterNodes(); + + /** */ + private long gridStartTime; + public ZookeeperDiscoveryImpl(IgniteLogger log, String basePath, String clusterName, @@ -87,20 +111,73 @@ public class ZookeeperDiscoveryImpl { PathUtils.validatePath(basePath); + zkPaths = new ZkPaths(basePath, clusterName); + this.log = log.getLogger(getClass()); - this.basePath = basePath; - this.clusterName = clusterName; this.locNode = locNode; this.lsnr = lsnr; this.exchange = exchange; + + watcher = new ZkWatcher(); + childrenCallback = new ZKChildrenCallback(); + dataCallback = new ZkDataCallback(); + } + + public ClusterNode localNode() { + return locNode; + } + + /** + * @param nodeId Node ID. + * @return Node instance. + */ + @Nullable public ClusterNode node(UUID nodeId) { + assert nodeId != null; + + return top.nodesById.get(nodeId); + } + + public boolean pingNode(UUID nodeId) { + // TODO ZK + return node(nodeId) != null; } /** - * @param path Relative path. - * @return Full path. + * @return Remote nodes. */ - private String zkPath(String path) { - return basePath + "/" + clusterName + "/" + path; + public Collection<ClusterNode> remoteNodes() { + return top.remoteNodes(); + } + + public boolean knownNode(UUID nodeId) { + try { + List<String> children = zkClient.getChildren(zkPaths.aliveNodesDir); + + for (int i = 0; i < children.size(); i++) { + UUID id = ZkPaths.aliveNodeId(children.get(i)); + + if (nodeId.equals(id)) + return true; + } + + return false; + } + catch (ZookeeperClientFailedException e) { + throw new IgniteException(e); + } + catch (InterruptedException e) { + Thread.currentThread().interrupt(); + + throw new IgniteInterruptedException(e); + } + } + + public void sendCustomEvent(DiscoverySpiCustomMessage msg) { + // TODO ZK + } + + public long gridStartTime() { + return gridStartTime; } /** @@ -116,7 +193,7 @@ public class ZookeeperDiscoveryImpl { exchange.collect(discoDataBag); - ZKJoiningNodeData joinData = new ZKJoiningNodeData(locNode, discoDataBag.joiningNodeData()); + ZkJoiningNodeData joinData = new ZkJoiningNodeData(locNode, discoDataBag.joiningNodeData()); byte[] joinDataBytes; @@ -138,53 +215,332 @@ public class ZookeeperDiscoveryImpl { throw new IgniteSpiException("Failed to create Zookeeper client", e); } + initZkNodes(); + + startJoin(joinDataBytes); + + for (;;) { + try { + joinFut.get(10_000); + + break; + } + catch (IgniteFutureTimeoutCheckedException e) { + U.warn(log, "Waiting for local join event [nodeId=" + locNode.id() + ", name=" + igniteInstanceName + ']'); + } + catch (Exception e) { + throw new IgniteSpiException("Failed to join cluster", e); + } + } + } + + /** + * @throws InterruptedException If interrupted. + */ + private void initZkNodes() throws InterruptedException { try { - initZkNodes(); + // TODO ZK: use multi. + if (zkClient.exists(zkPaths.aliveNodesDir)) + return; // This path is created last, assume all others dirs are created. - // TODO ZK: handle max size - zkClient.createIfNeeded(zkPath(JOIN_DATA_DIR) + "/" + locNode.id() + "-", joinDataBytes, EPHEMERAL_SEQUENTIAL); + zkClient.createIfNeeded(zkPaths.basePath, null, PERSISTENT); - zkClient.createIfNeeded(zkPath(ALIVE_NODES_DIR) + "/" + locNode.id() + "-", null, EPHEMERAL_SEQUENTIAL); + zkClient.createIfNeeded(zkPaths.evtsPath, null, PERSISTENT); - zkClient.getChildrenAsync(zkPath(ALIVE_NODES_DIR), false, new AsyncCallback.Children2Callback() { - @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { - assert rc == 0 : rc; + zkClient.createIfNeeded(zkPaths.joinDataDir, null, PERSISTENT); + zkClient.createIfNeeded(zkPaths.aliveNodesDir, null, PERSISTENT); + } + catch (ZookeeperClientFailedException e) { + throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e); + } + } - } - }, null); + /** */ + private ZkDiscoveryEventsData crdEvts; - for (;;) { - try { - joinFut.get(10_000); + /** + * @throws InterruptedException If interrupted. + */ + private void startJoin(byte[] joinDataBytes) throws InterruptedException { + try { + zkClient.getDataAsync(zkPaths.evtsPath, watcher, dataCallback); + + // TODO ZK: handle max size. + String path = zkClient.createIfNeeded(zkPaths.joinDataDir + "/" + locNode.id() + "|", joinDataBytes, EPHEMERAL_SEQUENTIAL); + + int seqNum = Integer.parseInt(path.substring(ZkPaths.ID_LEN + 2)); + + zkClient.createIfNeeded(zkPaths.aliveNodesDir + "/" + locNode.id() + "|" + seqNum + "|", null, EPHEMERAL); - break; + zkClient.getChildrenAsync(zkPaths.aliveNodesDir, null, new AsyncCallback.Children2Callback() { + @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { + nodeConnected(rc, children); } - catch (IgniteFutureTimeoutCheckedException e) { - U.warn(log, "Waiting for local join event [nodeId=" + locNode.id() + ", name=" + igniteInstanceName + ']'); + }); + } + catch (ZookeeperClientFailedException e) { + throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e); + } + } + + /** */ + private ZkDiscoveryEventsData evts; + + /** */ + private boolean crd; + + /** + * @param rc Async callback result. + * @param aliveNodes Alive nodes. + */ + private void nodeConnected(int rc, List<String> aliveNodes) { + try { + assert rc == 0 : rc; + assert !joined; + + TreeMap<Integer, String> alives = new TreeMap<>(); + + for (String child : aliveNodes) + alives.put(ZkPaths.aliveInternalId(child), child); + + assert !alives.isEmpty(); + + Map.Entry<Integer, String> crdE = alives.firstEntry(); + + UUID crdId = ZkPaths.aliveNodeId(crdE.getValue()); + + if (crdId.equals(locNode.id())) { + byte[] evtsData = zkClient.getData(zkPaths.evtsPath); + + if (evtsData.length > 0) + onEventsUpdate(evtsData, null); + + crd = true; + + if (joined) { + log.info("Node connected to zookeeper, node is new discovery coordinator " + + "[locId=" + locNode.id() + ']'); + + assert locNode.order() > 0 : locNode; + assert evts != null; } - catch (Exception e) { - throw new IgniteSpiException("Failed to join cluster", e); + else { + log.info("Node connected to zookeeper, node is first cluster node [locId=" + locNode.id() + ']'); + + newClusterStarted(crdE.getKey()); } + + zkClient.getChildrenAsync(zkPaths.aliveNodesDir, watcher, childrenCallback); + } + else { + } } - catch (ZookeeperClientFailedException e) { - throw new IgniteSpiException("Failed to initialize Zookeeper nodes", e); + catch (Throwable e) { + onFatalError(e); + } + } + + private void generateTopologyEvents(List<String> aliveNodes) throws Exception { + assert crd; + + TreeMap<Integer, String> alives = new TreeMap<>(); + + TreeMap<Long, ZookeeperClusterNode> curTop = new TreeMap<>(top.nodesByOrder); + + int evtCnt = evts.evts.size(); + + for (String child : aliveNodes) { + Integer inernalId = ZkPaths.aliveInternalId(child); + + Object old = alives.put(inernalId, child); + + assert old == null; + + if (!top.nodesByInternalId.containsKey(inernalId)) + processNodeJoin(curTop, inernalId, child); + } + + for (Map.Entry<Integer, ZookeeperClusterNode> e : top.nodesByInternalId.entrySet()) { + if (!alives.containsKey(e.getKey())) + processNodeFail(curTop, e.getValue()); + } + + if (evts.evts.size() > evtCnt) { + long start = System.currentTimeMillis(); + + zkClient.setData(zkPaths.evtsPath, marsh.marshal(evts), -1); + + // TODO ZK: on crd do not need listen for events path. + + long time = System.currentTimeMillis() - start; + + if (log.isInfoEnabled()) + log.info("Discovery coordinator saved new events [topVer=" + evts.topVer + ", saveTime=" + time + ']'); + } + } + + private void processNodeFail(TreeMap<Long, ZookeeperClusterNode> curTop, ZookeeperClusterNode failedNode) { + Object rmvd = curTop.remove(failedNode.order()); + + assert rmvd != null; + + evts.topVer++; + + ZkDiscoveryEventData evtData = new ZkDiscoveryNodeFailedEventData(evts.topVer, failedNode.internalId()); + + evts.addEvent(evtData); + + if (log.isInfoEnabled()) { + log.info("Generated NODE_FAILED event [topVer=" + evtData.topologyVersion() + + ", nodeId=" + failedNode.id() + ']'); + } + } + + private void processNodeJoin(TreeMap<Long, ZookeeperClusterNode> curTop, + int internalId, + String aliveNodePath) + throws Exception + { + UUID nodeId = ZkPaths.aliveNodeId(aliveNodePath); + int joinSeq = ZkPaths.aliveJoinSequence(aliveNodePath); + + String joinDataPath = zkPaths.zkPath(zkPaths.joinDataDir + '/' + nodeId.toString() + "|" + joinSeq); + + byte[] joinData; + + try { + joinData = zkClient.getData(joinDataPath); + } + catch (KeeperException.NoNodeException e) { + U.warn(log, "Failed to read joinin node data, node left before join process finished: " + nodeId); + + return; + } + + // TODO ZK: fail node if can not unmarshal. + ZkJoiningNodeData joiningNodeData = unmarshal(joinData); + + ZookeeperClusterNode joinedNode = joiningNodeData.node(); + + assert nodeId.equals(joinedNode.id()) : joiningNodeData.node(); + + evts.topVer++; + + joinedNode.order(evts.topVer); + joinedNode.internalId(internalId); + + Object old = curTop.put(joinedNode.order(), joinedNode); + + assert old == null; + + DiscoveryDataBag joiningNodeBag = new DiscoveryDataBag(nodeId); + + joiningNodeBag.joiningNodeData(joiningNodeData.discoveryData()); + + exchange.onExchange(joiningNodeBag); + + DiscoveryDataBag collectBag = new DiscoveryDataBag(nodeId, new HashSet<Integer>()); + + exchange.collect(collectBag); + + Map<Integer, Serializable> commonData = collectBag.commonData(); + + ZkJoinEventDataForJoined dataForJoined = new ZkJoinEventDataForJoined(new ArrayList<>(curTop.values()), + commonData); + + ZkDiscoveryEventData evtData = new ZkDiscoveryNodeJoinedEventData(evts.topVer, joinedNode.id()); + + evts.addEvent(evtData); + + String evtDataPath = zkPaths.evtsPath + "/" + evtData.topologyVersion(); + + long start = System.currentTimeMillis(); + + zkClient.createIfNeeded(evtDataPath, joinData, PERSISTENT); + zkClient.createIfNeeded(evtDataPath + "/joined", marshal(dataForJoined), PERSISTENT); + + long time = System.currentTimeMillis() - start; + + if (log.isInfoEnabled()) { + log.info("Generated NODE_JOINED event [topVer=" + evtData.topologyVersion() + + ", nodeId=" + joinedNode.id() + + ", addDataTime=" + time + ']'); } } - private void initZkNodes() throws ZookeeperClientFailedException, InterruptedException { - // TODO ZK: use multi. - String joinDir = zkPath(JOIN_DATA_DIR); + /** + * @param locInternalId Local node internal ID. + * @throws Exception If failed. + */ + @SuppressWarnings("unchecked") + private void newClusterStarted(int locInternalId) throws Exception { + joined = true; + + gridStartTime = U.currentTimeMillis(); + + evts = new ZkDiscoveryEventsData(gridStartTime, 1L, new TreeMap<Long, ZkDiscoveryEventData>()); + + // TODO ZK: old data cleanup. + + locNode.internalId(locInternalId); + locNode.order(1); + + top.addNode(locNode); + + lsnr.onDiscovery(EventType.EVT_NODE_JOINED, + 1L, + locNode, + (Collection)top.nodesByOrder.values(), + Collections.<Long, Collection<ClusterNode>>emptyMap(), + null); + + joinFut.onDone(); + } + + /** + * @param children + * @param stat + */ + private void onAliveNodesUpdate(List<String> children, Stat stat) throws Exception { + generateTopologyEvents(children); + } + + /** */ + private long lastProcEvt = -1; + + /** */ + private boolean joined; + + /** + * @param data + * @param stat + */ + private void onEventsUpdate(byte[] data, Stat stat) throws IgniteCheckedException { + ZkDiscoveryEventsData evtsData = unmarshal(data); + + TreeMap<Long, ZkDiscoveryEventData> evts = evtsData.evts; - if (zkClient.exists(joinDir)) - return; // Assume all others dirs are created. + for (Map.Entry<Long, ZkDiscoveryEventData> e : evts.tailMap(lastProcEvt).entrySet()) { + ZkDiscoveryEventData evtData = e.getValue(); - zkClient.createIfNeeded(basePath, null, PERSISTENT); + if (!joined) { + if (evtData.eventType() != EventType.EVT_NODE_JOINED) + continue; - zkClient.createIfNeeded(zkPath(ALIVE_NODES_DIR), null, PERSISTENT); + UUID joinedId = ((ZkDiscoveryNodeJoinedEventData)evtData).nodeId; - zkClient.createIfNeeded(joinDir, null, PERSISTENT); + boolean locJoin = evtData.eventType() == EventType.EVT_NODE_JOINED && + locNode.id().equals(joinedId); + + if (locJoin) { + + } + } + + lastProcEvt = e.getKey(); + } } /** @@ -196,6 +552,19 @@ public class ZookeeperDiscoveryImpl { } /** + * @param e Error. + */ + private void onFatalError(Throwable e) { + // TODO ZL + U.error(log, "Failed to process discovery data. Stopping the node in order to prevent cluster wide instability.", e); + + joinFut.onDone(e); + + if (e instanceof Error) + throw (Error)e; + } + + /** * @param bytes Bytes. * @return Unmarshalled object. * @throws IgniteCheckedException If failed. @@ -221,18 +590,62 @@ public class ZookeeperDiscoveryImpl { * */ private class ConnectionLossListener implements IgniteRunnable { + /** {@inheritDoc} */ @Override public void run() { + U.warn(log, "Zookeeper connection loss, local node is SEGMENTED"); + } + } + /** + * + */ + private class ZkWatcher implements Watcher { + /** {@inheritDoc} */ + @Override public void process(WatchedEvent evt) { + if (evt.getType() == Event.EventType.NodeDataChanged) + zkClient.getDataAsync(evt.getPath(), this, dataCallback); + else if (evt.getType() == Event.EventType.NodeChildrenChanged) + zkClient.getChildrenAsync(evt.getPath(), this, childrenCallback); } } /** * */ - private class ZKChildrenUpdateCallback implements AsyncCallback.Children2Callback { + private class ZKChildrenCallback implements AsyncCallback.Children2Callback { + /** {@inheritDoc} */ @Override public void processResult(int rc, String path, Object ctx, List<String> children, Stat stat) { + try { + assert rc == 0 : rc; + if (path.equals(zkPaths.aliveNodesDir)) + onAliveNodesUpdate(children, stat); + else + U.warn(log, "Children callback for unknown path: " + path); + } + catch (Throwable e) { + onFatalError(e); + } } } + /** + * + */ + private class ZkDataCallback implements AsyncCallback.DataCallback { + /** {@inheritDoc} */ + @Override public void processResult(int rc, String path, Object ctx, byte[] data, Stat stat) { + try { + assert rc == 0 : rc; + + if (path.equals(zkPaths.evtsPath)) + onEventsUpdate(data, stat); + else + U.warn(log, "Data callback for unknown path: " + path); + } + catch (Throwable e) { + onFatalError(e); + } + } + } } http://git-wip-us.apache.org/repos/asf/ignite/blob/5793ffbe/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiBasicTest.java ---------------------------------------------------------------------- diff --git a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiBasicTest.java b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiBasicTest.java index bc577d2..54dc1b4 100644 --- a/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiBasicTest.java +++ b/modules/zookeeper/src/test/java/org/apache/ignite/spi/discovery/zk/ZookeeperDiscoverySpiBasicTest.java @@ -51,6 +51,7 @@ import org.apache.ignite.resources.IgniteInstanceResource; import org.apache.ignite.testframework.GridTestUtils; import org.apache.ignite.testframework.junits.common.GridCommonAbstractTest; import org.apache.zookeeper.ZkTestClientCnxnSocketNIO; +import org.apache.zookeeper.ZooKeeper; import static org.apache.ignite.events.EventType.EVT_NODE_FAILED; import static org.apache.ignite.events.EventType.EVT_NODE_JOINED; @@ -103,10 +104,10 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest { if (USE_TEST_CLUSTER) { assert zkCluster != null; - zkSpi.setConnectString(zkCluster.getConnectString()); + zkSpi.setZkConnectionString(zkCluster.getConnectString()); } else - zkSpi.setConnectString("localhost:2181"); + zkSpi.setZkConnectionString("localhost:2181"); cfg.setDiscoverySpi(zkSpi); @@ -369,7 +370,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest { if (failWhenDisconnected) { ZookeeperDiscoverySpi spi = spis.get(getTestIgniteInstanceName(2)); - spi.closeClient(); + closeZkClient(spi); checkEvents(node0, failEvent(4)); } @@ -390,6 +391,20 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest { } /** + * @param spi Spi instance. + */ + private void closeZkClient(ZookeeperDiscoverySpi spi) { + ZooKeeper zk = GridTestUtils.getFieldValue(spi, "impl.zkClient.zk"); + + try { + zk.close(); + } + catch (Exception e) { + fail("Unexpected error: " + e); + } + } + + /** * @throws Exception If failed. */ public void testConnectionRestore_Coordinator1() throws Exception { @@ -455,7 +470,7 @@ public class ZookeeperDiscoverySpiBasicTest extends GridCommonAbstractTest { spi.waitConnectStart(); if (cnt < failCnt) - spi.closeClient(); + closeZkClient(spi); } c0.allowConnect();
