[ASTERIXDB-2198][REPL] Introduce Dynamic Replica Placement
- user model changes: no
- storage format changes: no
- interface changes: yes
- Add IReplicationMessage and IReplicaTask.
- Add notifyMetadataNodeChange to IFaultToleranceStrategy.
- Add register to IReplicationManager to allow registering
replicas at runtime.
Details:
- Add cluster APIs for:
- changing partition master node.
- changing metadata node.
- Add NC storage management API for promoting a partition replica
to master replica.
- Implement changing metadata node at runtime in
MetadataNodeFaultToleranceStrategy.
- Allow MetadataNodeFaultToleranceStrategy to have zero replica
at initialization.
- Add a flag to LangExecutionUtil to skip storage distribution
check at the end of each test.
- Add test case for metadata node failover as follows:
1- start with nc1 as metadata node.
2- add replica for metadata partition on nc2 at runtime.
3- performs metadata transactions on nc1.
4- promote metadata partition on nc2.
5- failover metadata node to nc2.
6- ensure the effects of the metadata transactions on (2) exists.
7- performs more metadata transactions on nc2.
8- ensure the effects of the metadata transactions on (7) exists.
Change-Id: I11f82efcad29d2c37324fe9d3c11d872b0348f49
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2215
Sonar-Qube: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Contrib: Jenkins <[email protected]>
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Till Westmann <[email protected]>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/cc7d2f0c
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/cc7d2f0c
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/cc7d2f0c
Branch: refs/heads/master
Commit: cc7d2f0ce46f2dd628a548b819d4e9a1f8ff99a9
Parents: 54249a8
Author: Murtadha Hubail <[email protected]>
Authored: Wed Dec 13 23:11:09 2017 +0300
Committer: Murtadha Hubail <[email protected]>
Committed: Wed Dec 13 20:30:36 2017 -0800
----------------------------------------------------------------------
.../api/http/server/ClusterApiServlet.java | 28 +++++
.../api/http/server/StorageApiServlet.java | 14 +++
.../asterix/app/nc/NCAppRuntimeContext.java | 2 +-
.../apache/asterix/app/nc/RecoveryManager.java | 4 +-
.../apache/asterix/app/nc/ReplicaManager.java | 17 ++-
.../replication/AutoFaultToleranceStrategy.java | 12 +--
.../MetadataNodeFaultToleranceStrategy.java | 41 +++++++
.../message/MetadataNodeRequestMessage.java | 78 ++++++++++++++
.../message/MetadataNodeResponseMessage.java | 60 +++++++++++
.../TakeoverMetadataNodeRequestMessage.java | 69 ------------
.../TakeoverMetadataNodeResponseMessage.java | 54 ----------
.../asterix-app/src/main/resources/cluster.xml | 3 -
.../asterix/test/runtime/LangExecutionUtil.java | 12 ++-
.../test/runtime/ReplicationExecutionTest.java | 46 +++++---
.../add_replica/add_replica.2.get.http | 19 ----
.../add_replica/add_replica.2.pollget.http | 21 ++++
.../metadata_failover.1.sto.cmd | 19 ++++
.../metadata_failover.10.ddl.sqlpp | 19 ++++
.../metadata_failover.11.query.sqlpp | 21 ++++
.../metadata_failover.2.pollget.http | 21 ++++
.../metadata_failover.3.ddl.sqlpp | 23 ++++
.../metadata_failover.4.sto.cmd | 19 ++++
.../metadata_failover.5.get.http | 19 ++++
.../metadata_failover.6.post.http | 19 ++++
.../metadata_failover.7.post.http | 19 ++++
.../metadata_failover.8.pollget.http | 21 ++++
.../metadata_failover.9.query.sqlpp | 21 ++++
.../test/resources/runtimets/replication.xml | 5 +
.../replication/add_replica/add_replica.2.adm | 2 +-
.../metadata_failover/metadata_failover.11.adm | 1 +
.../metadata_failover/metadata_failover.2.adm | 7 ++
.../metadata_failover/metadata_failover.5.adm | 10 ++
.../metadata_failover/metadata_failover.6.adm | 0
.../metadata_failover/metadata_failover.7.adm | 0
.../metadata_failover/metadata_failover.8.adm | 38 +++++++
.../metadata_failover/metadata_failover.9.adm | 1 +
.../replication/IFaultToleranceStrategy.java | 8 ++
.../common/replication/INCLifecycleMessage.java | 4 +-
.../replication/IReplicaResourcesManager.java | 3 +
.../common/replication/IReplicationManager.java | 7 ++
.../common/replication/IReplicationThread.java | 17 ++-
.../MetadataOnlyReplicationStrategy.java | 26 ++---
.../asterix/common/storage/IReplicaManager.java | 8 ++
.../common/storage/PartitionReplica.java | 100 -----------------
.../common/storage/ResourceReference.java | 4 +
.../asterix/common/utils/StorageConstants.java | 1 +
.../asterix/common/utils/StoragePathUtil.java | 10 +-
.../asterix/replication/api/IReplicaTask.java | 35 ++++++
.../replication/api/IReplicationMessage.java | 40 +++++++
.../functions/ReplicationProtocol.java | 103 +++++++++++++++++-
.../management/ReplicationChannel.java | 83 +++++++++-----
.../management/ReplicationManager.java | 22 ++++
.../CheckpointPartitionIndexesTask.java | 89 +++++++++++++++
.../replication/messaging/DeleteFileTask.java | 84 +++++++++++++++
.../PartitionResourcesListResponse.java | 74 +++++++++++++
.../messaging/PartitionResourcesListTask.java | 80 ++++++++++++++
.../messaging/ReplicateFileTask.java | 108 +++++++++++++++++++
.../replication/recovery/FileSynchronizer.java | 73 +++++++++++++
.../recovery/ReplicaFilesSynchronizer.java | 84 +++++++++++++++
.../recovery/ReplicaSynchronizer.java | 67 ++++++++++++
.../replication/storage/PartitionReplica.java | 67 +++++++++++-
.../storage/ReplicaResourcesManager.java | 1 +
62 files changed, 1633 insertions(+), 330 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
index 6826b26..18e837a 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/ClusterApiServlet.java
@@ -30,6 +30,7 @@ import java.util.logging.Logger;
import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.hyracks.api.config.IOption;
import org.apache.hyracks.api.config.Section;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.control.common.config.ConfigUtils;
import org.apache.hyracks.control.common.controllers.ControllerConfig;
import org.apache.hyracks.http.api.IServletRequest;
@@ -90,6 +91,21 @@ public class ClusterApiServlet extends AbstractServlet {
responseWriter.flush();
}
+ @Override
+ protected void post(IServletRequest request, IServletResponse response)
throws Exception {
+ switch (localPath(request)) {
+ case "/partition/master":
+ processPartitionMaster(request, response);
+ break;
+ case "/metadataNode":
+ processMetadataNode(request, response);
+ break;
+ default:
+ sendError(response, HttpResponseStatus.NOT_FOUND);
+ break;
+ }
+ }
+
protected ObjectNode getClusterStateSummaryJSON() {
return appCtx.getClusterStateManager().getClusterStateSummary();
}
@@ -143,4 +159,16 @@ public class ClusterApiServlet extends AbstractServlet {
&& option != ControllerConfig.Option.CONFIG_FILE_URL;
}
+ private void processPartitionMaster(IServletRequest request,
IServletResponse response) {
+ final String partition = request.getParameter("partition");
+ final String node = request.getParameter("node");
+
appCtx.getClusterStateManager().updateClusterPartition(Integer.valueOf(partition),
node, true);
+ response.setStatus(HttpResponseStatus.OK);
+ }
+
+ private void processMetadataNode(IServletRequest request, IServletResponse
response) throws HyracksDataException {
+ final String node = request.getParameter("node");
+ appCtx.getFaultToleranceStrategy().notifyMetadataNodeChange(node);
+ response.setStatus(HttpResponseStatus.OK);
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
index 8e73405..c2cda4e 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/server/StorageApiServlet.java
@@ -33,6 +33,7 @@ import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.replication.IPartitionReplica;
import org.apache.asterix.common.storage.IReplicaManager;
import org.apache.asterix.common.storage.ReplicaIdentifier;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.http.api.IServletRequest;
import org.apache.hyracks.http.api.IServletResponse;
import org.apache.hyracks.http.server.AbstractServlet;
@@ -91,6 +92,9 @@ public class StorageApiServlet extends AbstractServlet {
case "/removeReplica":
processRemoveReplica(request, response);
break;
+ case "/promote":
+ processPromote(request, response);
+ break;
default:
sendError(response, HttpResponseStatus.NOT_FOUND);
break;
@@ -160,4 +164,14 @@ public class StorageApiServlet extends AbstractServlet {
final InetSocketAddress replicaAddress = new InetSocketAddress(host,
Integer.valueOf(port));
return ReplicaIdentifier.of(Integer.valueOf(partition),
replicaAddress);
}
+
+ private void processPromote(IServletRequest request, IServletResponse
response) throws HyracksDataException {
+ final String partition = request.getParameter("partition");
+ if (partition == null) {
+ response.setStatus(HttpResponseStatus.BAD_REQUEST);
+ return;
+ }
+ appCtx.getReplicaManager().promote(Integer.valueOf(partition));
+ response.setStatus(HttpResponseStatus.OK);
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 75159af..81b232a 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -213,7 +213,7 @@ public class NCAppRuntimeContext implements
INcApplicationContext {
final ClusterPartition[] nodePartitions =
metadataProperties.getNodePartitions().get(nodeId);
final Set<Integer> nodePartitionsIds =
Arrays.stream(nodePartitions).map(ClusterPartition::getPartitionId)
.collect(Collectors.toSet());
- replicaManager = new ReplicaManager(nodePartitionsIds);
+ replicaManager = new ReplicaManager(this, nodePartitionsIds);
isShuttingdown = false;
activeManager = new ActiveManager(threadExecutor,
getServiceContext().getNodeId(),
activeProperties.getMemoryComponentGlobalBudget(),
compilerProperties.getFrameSize(),
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index f0ed5e9..22fa459 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -375,8 +375,8 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
} else {
maxDiskLastLsn =
resourceId2MaxLSNMap.get(resourceId);
}
-
- if (lsn > maxDiskLastLsn) {
+ // lsn @ maxDiskLastLsn is either a flush log
or a master replica log
+ if (lsn >= maxDiskLastLsn) {
redo(logRecord, datasetLifecycleManager);
redoCount++;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
index 0c84a6e..bf17a5b 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/ReplicaManager.java
@@ -25,14 +25,19 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.stream.Collectors;
+import java.util.stream.Stream;
+import org.apache.asterix.common.api.INcApplicationContext;
import org.apache.asterix.common.replication.IPartitionReplica;
+import org.apache.asterix.common.replication.IRemoteRecoveryManager;
import org.apache.asterix.common.storage.IReplicaManager;
import org.apache.asterix.common.storage.ReplicaIdentifier;
import org.apache.asterix.replication.storage.PartitionReplica;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
public class ReplicaManager implements IReplicaManager {
+ private final INcApplicationContext appCtx;
/**
* the partitions to which the current node is master
*/
@@ -42,7 +47,8 @@ public class ReplicaManager implements IReplicaManager {
*/
private final Map<ReplicaIdentifier, PartitionReplica> replicas = new
HashMap<>();
- public ReplicaManager(Set<Integer> partitions) {
+ public ReplicaManager(INcApplicationContext appCtx, Set<Integer>
partitions) {
+ this.appCtx = appCtx;
this.partitions.addAll(partitions);
}
@@ -52,7 +58,7 @@ public class ReplicaManager implements IReplicaManager {
throw new IllegalStateException(
"This node is not the current master of partition(" +
id.getPartition() + ")");
}
- replicas.computeIfAbsent(id, key -> new PartitionReplica(key));
+ replicas.computeIfAbsent(id, k -> new PartitionReplica(k, appCtx));
replicas.get(id).sync();
}
@@ -74,4 +80,11 @@ public class ReplicaManager implements IReplicaManager {
public Set<Integer> getPartitions() {
return Collections.unmodifiableSet(partitions);
}
+
+ @Override
+ public synchronized void promote(int partition) throws
HyracksDataException {
+ final IRemoteRecoveryManager remoteRecoveryManager =
appCtx.getRemoteRecoveryManager();
+
remoteRecoveryManager.replayReplicaPartitionLogs(Stream.of(partition).collect(Collectors.toSet()),
true);
+ partitions.add(partition);
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/AutoFaultToleranceStrategy.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/AutoFaultToleranceStrategy.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/AutoFaultToleranceStrategy.java
index 23f225e..e4a6f4b 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/AutoFaultToleranceStrategy.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/AutoFaultToleranceStrategy.java
@@ -45,8 +45,8 @@ import
org.apache.asterix.app.replication.message.PreparePartitionsFailbackReque
import
org.apache.asterix.app.replication.message.PreparePartitionsFailbackResponseMessage;
import
org.apache.asterix.app.replication.message.RegistrationTasksRequestMessage;
import
org.apache.asterix.app.replication.message.RegistrationTasksResponseMessage;
-import
org.apache.asterix.app.replication.message.TakeoverMetadataNodeRequestMessage;
-import
org.apache.asterix.app.replication.message.TakeoverMetadataNodeResponseMessage;
+import org.apache.asterix.app.replication.message.MetadataNodeRequestMessage;
+import org.apache.asterix.app.replication.message.MetadataNodeResponseMessage;
import
org.apache.asterix.app.replication.message.TakeoverPartitionsRequestMessage;
import
org.apache.asterix.app.replication.message.TakeoverPartitionsResponseMessage;
import org.apache.asterix.common.api.IClusterManagementWork.ClusterState;
@@ -337,7 +337,7 @@ public class AutoFaultToleranceStrategy implements
IFaultToleranceStrategy {
validateClusterState();
}
- public synchronized void process(TakeoverMetadataNodeResponseMessage
response) throws HyracksDataException {
+ public synchronized void process(MetadataNodeResponseMessage response)
throws HyracksDataException {
currentMetadataNode = response.getNodeId();
metadataNodeActive = true;
clusterManager.updateMetadataNode(currentMetadataNode,
metadataNodeActive);
@@ -403,7 +403,7 @@ public class AutoFaultToleranceStrategy implements
IFaultToleranceStrategy {
ICcApplicationContext appCtx = (ICcApplicationContext)
serviceCtx.getApplicationContext();
ClusterPartition metadataPartiton =
appCtx.getMetadataProperties().getMetadataPartition();
//request the metadataPartition node to register itself as the
metadata node
- TakeoverMetadataNodeRequestMessage takeoverRequest = new
TakeoverMetadataNodeRequestMessage();
+ MetadataNodeRequestMessage takeoverRequest = new
MetadataNodeRequestMessage(true);
try {
messageBroker.sendApplicationMessageToNC(takeoverRequest,
metadataPartiton.getActiveNodeId());
// Since the metadata node will be changed, we need to rebind the
proxy object
@@ -440,8 +440,8 @@ public class AutoFaultToleranceStrategy implements
IFaultToleranceStrategy {
case TAKEOVER_PARTITION_RESPONSE:
process((TakeoverPartitionsResponseMessage) message);
break;
- case TAKEOVER_METADATA_NODE_RESPONSE:
- process((TakeoverMetadataNodeResponseMessage) message);
+ case METADATA_NODE_RESPONSE:
+ process((MetadataNodeResponseMessage) message);
break;
case PREPARE_FAILBACK_RESPONSE:
process((PreparePartitionsFailbackResponseMessage) message);
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/MetadataNodeFaultToleranceStrategy.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/MetadataNodeFaultToleranceStrategy.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/MetadataNodeFaultToleranceStrategy.java
index 3341813..3470d7b 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/MetadataNodeFaultToleranceStrategy.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/MetadataNodeFaultToleranceStrategy.java
@@ -38,6 +38,8 @@ import org.apache.asterix.app.nc.task.RemoteRecoveryTask;
import org.apache.asterix.app.nc.task.ReportLocalCountersTask;
import org.apache.asterix.app.nc.task.StartLifecycleComponentsTask;
import org.apache.asterix.app.nc.task.StartReplicationServiceTask;
+import org.apache.asterix.app.replication.message.MetadataNodeRequestMessage;
+import org.apache.asterix.app.replication.message.MetadataNodeResponseMessage;
import org.apache.asterix.app.replication.message.NCLifecycleTaskReportMessage;
import
org.apache.asterix.app.replication.message.RegistrationTasksRequestMessage;
import
org.apache.asterix.app.replication.message.ReplayPartitionLogsRequestMessage;
@@ -132,6 +134,9 @@ public class MetadataNodeFaultToleranceStrategy implements
IFaultToleranceStrate
case REPLAY_LOGS_RESPONSE:
process((ReplayPartitionLogsResponseMessage) message);
break;
+ case METADATA_NODE_RESPONSE:
+ process((MetadataNodeResponseMessage) message);
+ break;
default:
throw new
RuntimeDataException(ErrorCode.UNSUPPORTED_MESSAGE_TYPE,
message.getType().name());
}
@@ -143,6 +148,26 @@ public class MetadataNodeFaultToleranceStrategy implements
IFaultToleranceStrate
this.metadataNodeId = clusterManager.getCurrentMetadataNodeId();
}
+ @Override
+ public void notifyMetadataNodeChange(String node) throws
HyracksDataException {
+ if (metadataNodeId.equals(node)) {
+ return;
+ }
+ // if current metadata node is active, we need to unbind its metadata
proxy object
+ if (clusterManager.isMetadataNodeActive()) {
+ MetadataNodeRequestMessage msg = new
MetadataNodeRequestMessage(false);
+ try {
+ messageBroker.sendApplicationMessageToNC(msg, metadataNodeId);
+ // when the current node responses, we will bind to the new one
+ metadataNodeId = node;
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ } else {
+ requestMetadataNodeTakeover(node);
+ }
+ }
+
private synchronized void process(ReplayPartitionLogsResponseMessage msg) {
hotStandbyMetadataReplica.add(msg.getNodeId());
if (LOGGER.isLoggable(Level.INFO)) {
@@ -256,4 +281,20 @@ public class MetadataNodeFaultToleranceStrategy implements
IFaultToleranceStrate
recoveryPlan.put(hotStandbyMetadataReplica.iterator().next(),
metadataPartition);
return new RemoteRecoveryTask(recoveryPlan);
}
+
+ private void process(MetadataNodeResponseMessage response) throws
HyracksDataException {
+ clusterManager.updateMetadataNode(response.getNodeId(),
response.isExported());
+ if (!response.isExported()) {
+ requestMetadataNodeTakeover(metadataNodeId);
+ }
+ }
+
+ private void requestMetadataNodeTakeover(String node) throws
HyracksDataException {
+ MetadataNodeRequestMessage msg = new MetadataNodeRequestMessage(true);
+ try {
+ messageBroker.sendApplicationMessageToNC(msg, node);
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
new file mode 100644
index 0000000..bebd133
--- /dev/null
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeRequestMessage.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.replication.message;
+
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.messaging.api.INCMessageBroker;
+import org.apache.asterix.common.messaging.api.INcAddressedMessage;
+import org.apache.asterix.common.replication.INCLifecycleMessage;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class MetadataNodeRequestMessage implements INCLifecycleMessage,
INcAddressedMessage {
+
+ private static final long serialVersionUID = 1L;
+ private static final Logger LOGGER =
Logger.getLogger(MetadataNodeRequestMessage.class.getName());
+ private final boolean export;
+
+ public MetadataNodeRequestMessage(boolean export) {
+ this.export = export;
+ }
+
+ @Override
+ public void handle(INcApplicationContext appContext) throws
HyracksDataException, InterruptedException {
+ INCMessageBroker broker = (INCMessageBroker)
appContext.getServiceContext().getMessageBroker();
+ HyracksDataException hde = null;
+ try {
+ if (export) {
+ appContext.initializeMetadata(false);
+ appContext.exportMetadataNodeStub();
+ } else {
+ appContext.unexportMetadataNodeStub();
+ }
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Failed taking over metadata", e);
+ hde = HyracksDataException.create(e);
+ } finally {
+ MetadataNodeResponseMessage reponse =
+ new
MetadataNodeResponseMessage(appContext.getTransactionSubsystem().getId(),
export);
+ try {
+ broker.sendMessageToCC(reponse);
+ } catch (Exception e) {
+ LOGGER.log(Level.SEVERE, "Failed taking over metadata", e);
+ hde = HyracksDataException.suppress(hde, e);
+ }
+ }
+ if (hde != null) {
+ throw hde;
+ }
+ }
+
+ @Override
+ public String toString() {
+ return MetadataNodeRequestMessage.class.getSimpleName();
+ }
+
+ @Override
+ public MessageType getType() {
+ return MessageType.METADATA_NODE_REQUEST;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java
new file mode 100644
index 0000000..ebde9b9
--- /dev/null
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/MetadataNodeResponseMessage.java
@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.replication.message;
+
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
+import org.apache.asterix.common.replication.INCLifecycleMessage;
+import org.apache.asterix.runtime.utils.CcApplicationContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class MetadataNodeResponseMessage implements INCLifecycleMessage,
ICcAddressedMessage {
+
+ private static final long serialVersionUID = 1L;
+ private final String nodeId;
+ private final boolean exported;
+
+ public MetadataNodeResponseMessage(String nodeId, boolean exported) {
+ this.nodeId = nodeId;
+ this.exported = exported;
+ }
+
+ public String getNodeId() {
+ return nodeId;
+ }
+
+ @Override
+ public void handle(ICcApplicationContext appCtx) throws
HyracksDataException, InterruptedException {
+ ((CcApplicationContext)
appCtx).getFaultToleranceStrategy().process(this);
+ }
+
+ @Override
+ public String toString() {
+ return MetadataNodeResponseMessage.class.getSimpleName();
+ }
+
+ @Override
+ public MessageType getType() {
+ return MessageType.METADATA_NODE_RESPONSE;
+ }
+
+ public boolean isExported() {
+ return exported;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/TakeoverMetadataNodeRequestMessage.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/TakeoverMetadataNodeRequestMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/TakeoverMetadataNodeRequestMessage.java
deleted file mode 100644
index 2137924..0000000
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/TakeoverMetadataNodeRequestMessage.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.app.replication.message;
-
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-import org.apache.asterix.common.api.INcApplicationContext;
-import org.apache.asterix.common.messaging.api.INCMessageBroker;
-import org.apache.asterix.common.messaging.api.INcAddressedMessage;
-import org.apache.asterix.common.replication.INCLifecycleMessage;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class TakeoverMetadataNodeRequestMessage implements
INCLifecycleMessage, INcAddressedMessage {
-
- private static final long serialVersionUID = 1L;
- private static final Logger LOGGER =
Logger.getLogger(TakeoverMetadataNodeRequestMessage.class.getName());
-
- @Override
- public void handle(INcApplicationContext appContext) throws
HyracksDataException, InterruptedException {
- INCMessageBroker broker = (INCMessageBroker)
appContext.getServiceContext().getMessageBroker();
- HyracksDataException hde = null;
- try {
- appContext.initializeMetadata(false);
- appContext.exportMetadataNodeStub();
- } catch (Exception e) {
- LOGGER.log(Level.SEVERE, "Failed taking over metadata", e);
- hde = HyracksDataException.create(e);
- } finally {
- TakeoverMetadataNodeResponseMessage reponse =
- new
TakeoverMetadataNodeResponseMessage(appContext.getTransactionSubsystem().getId());
- try {
- broker.sendMessageToCC(reponse);
- } catch (Exception e) {
- LOGGER.log(Level.SEVERE, "Failed taking over metadata", e);
- hde = HyracksDataException.suppress(hde, e);
- }
- }
- if (hde != null) {
- throw hde;
- }
- }
-
- @Override
- public String toString() {
- return TakeoverMetadataNodeRequestMessage.class.getSimpleName();
- }
-
- @Override
- public MessageType getType() {
- return MessageType.TAKEOVER_METADATA_NODE_REQUEST;
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/TakeoverMetadataNodeResponseMessage.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/TakeoverMetadataNodeResponseMessage.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/TakeoverMetadataNodeResponseMessage.java
deleted file mode 100644
index ff1b2d2..0000000
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/replication/message/TakeoverMetadataNodeResponseMessage.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.app.replication.message;
-
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
-import org.apache.asterix.common.messaging.api.ICcAddressedMessage;
-import org.apache.asterix.common.replication.INCLifecycleMessage;
-import org.apache.asterix.runtime.utils.CcApplicationContext;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-
-public class TakeoverMetadataNodeResponseMessage implements
INCLifecycleMessage, ICcAddressedMessage {
-
- private static final long serialVersionUID = 1L;
- private final String nodeId;
-
- public TakeoverMetadataNodeResponseMessage(String nodeId) {
- this.nodeId = nodeId;
- }
-
- public String getNodeId() {
- return nodeId;
- }
-
- @Override
- public void handle(ICcApplicationContext appCtx) throws
HyracksDataException, InterruptedException {
- ((CcApplicationContext)
appCtx).getFaultToleranceStrategy().process(this);
- }
-
- @Override
- public String toString() {
- return TakeoverMetadataNodeResponseMessage.class.getSimpleName();
- }
-
- @Override
- public MessageType getType() {
- return MessageType.TAKEOVER_METADATA_NODE_RESPONSE;
- }
-}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/main/resources/cluster.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/main/resources/cluster.xml
b/asterixdb/asterix-app/src/main/resources/cluster.xml
index 7b7d52a..41be696 100644
--- a/asterixdb/asterix-app/src/main/resources/cluster.xml
+++ b/asterixdb/asterix-app/src/main/resources/cluster.xml
@@ -30,9 +30,6 @@
</data_replication>
<fault_tolerance>
<strategy>metadata_node</strategy>
- <replica>
- <node_id>nc2</node_id>
- </replica>
</fault_tolerance>
</high_availability>
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
index 9d73407..09761a0 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/LangExecutionUtil.java
@@ -32,7 +32,6 @@ import java.util.Collection;
import java.util.List;
import org.apache.asterix.app.external.ExternalUDFLibrarian;
-import org.apache.asterix.common.config.ClusterProperties;
import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.test.common.TestExecutor;
@@ -40,8 +39,8 @@ import
org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.commons.lang.SystemUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hyracks.api.io.IODeviceHandle;
-import org.apache.hyracks.util.ThreadDumpUtil;
import org.apache.hyracks.control.nc.NodeControllerService;
+import org.apache.hyracks.util.ThreadDumpUtil;
/**
* Utils for running SQL++ or AQL runtime tests.
@@ -59,6 +58,7 @@ public class LangExecutionUtil {
private static ExternalUDFLibrarian librarian;
private static final int repeat = Integer.getInteger("test.repeat", 1);
+ private static boolean checkStorageDistribution = true;
public static void setUp(String configFile, TestExecutor executor) throws
Exception {
testExecutor = executor;
@@ -126,7 +126,9 @@ public class LangExecutionUtil {
testExecutor.executeTest(PATH_ACTUAL, tcCtx, null, false,
ExecutionTestUtil.FailedGroup);
try {
- checkStorageFiles();
+ if (checkStorageDistribution) {
+ checkStorageFiles();
+ }
} finally {
testExecutor.cleanup(tcCtx.toString(), badTestCases);
}
@@ -223,6 +225,10 @@ public class LangExecutionUtil {
}
}
+ public static void setCheckStorageDistribution(boolean
checkStorageDistribution) {
+ LangExecutionUtil.checkStorageDistribution = checkStorageDistribution;
+ }
+
private static void outputLeakedOpenFiles(String processId) throws
IOException {
Process process =
Runtime.getRuntime().exec(new String[] { "bash", "-c", "lsof
-p " + processId + "|grep waf" });
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ReplicationExecutionTest.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ReplicationExecutionTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ReplicationExecutionTest.java
index 56c7bc0..32b756b 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ReplicationExecutionTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/runtime/ReplicationExecutionTest.java
@@ -25,10 +25,12 @@ import java.util.HashMap;
import java.util.Map;
import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.config.ClusterProperties;
import org.apache.asterix.test.common.TestExecutor;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.hyracks.control.nc.NodeControllerService;
-import org.junit.AfterClass;
+import org.junit.After;
+import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -39,28 +41,38 @@ import org.junit.runners.Parameterized.Parameters;
public class ReplicationExecutionTest {
protected static final String TEST_CONFIG_FILE_NAME =
"asterix-build-configuration.xml";
private static final TestExecutor testExecutor = new TestExecutor();
+ private static boolean configured = false;
@BeforeClass
- public static void setUp() throws Exception {
+ public static void setUp() {
+
ClusterProperties.INSTANCE.getCluster().getHighAvailability().setEnabled(String.valueOf(true));
+ LangExecutionUtil.setCheckStorageDistribution(false);
+ }
+
+ @Before
+ public void before() throws Exception {
LangExecutionUtil.setUp(TEST_CONFIG_FILE_NAME, testExecutor);
- final NodeControllerService[] ncs =
ExecutionTestUtil.integrationUtil.ncs;
- Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
- Map<String, InetSocketAddress> replicationAddress = new HashMap<>();
- final String ip = InetAddress.getLoopbackAddress().getHostAddress();
- for (NodeControllerService nc : ncs) {
- final String nodeId = nc.getId();
- final INcApplicationContext appCtx = (INcApplicationContext)
nc.getApplicationContext();
- int apiPort = appCtx.getExternalProperties().getNcApiPort();
- int replicationPort =
appCtx.getReplicationProperties().getDataReplicationPort(nodeId);
- ncEndPoints.put(nodeId, InetSocketAddress.createUnresolved(ip,
apiPort));
- replicationAddress.put(nodeId,
InetSocketAddress.createUnresolved(ip, replicationPort));
+ if (!configured) {
+ final NodeControllerService[] ncs =
ExecutionTestUtil.integrationUtil.ncs;
+ Map<String, InetSocketAddress> ncEndPoints = new HashMap<>();
+ Map<String, InetSocketAddress> replicationAddress = new
HashMap<>();
+ final String ip =
InetAddress.getLoopbackAddress().getHostAddress();
+ for (NodeControllerService nc : ncs) {
+ final String nodeId = nc.getId();
+ final INcApplicationContext appCtx = (INcApplicationContext)
nc.getApplicationContext();
+ int apiPort = appCtx.getExternalProperties().getNcApiPort();
+ int replicationPort =
appCtx.getReplicationProperties().getDataReplicationPort(nodeId);
+ ncEndPoints.put(nodeId, InetSocketAddress.createUnresolved(ip,
apiPort));
+ replicationAddress.put(nodeId,
InetSocketAddress.createUnresolved(ip, replicationPort));
+ }
+ testExecutor.setNcEndPoints(ncEndPoints);
+ testExecutor.setNcReplicationAddress(replicationAddress);
+ configured = true;
}
- testExecutor.setNcEndPoints(ncEndPoints);
- testExecutor.setNcReplicationAddress(replicationAddress);
}
- @AfterClass
- public static void tearDown() throws Exception {
+ @After
+ public void after() throws Exception {
LangExecutionUtil.tearDown();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/add_replica/add_replica.2.get.http
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/add_replica/add_replica.2.get.http
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/add_replica/add_replica.2.get.http
deleted file mode 100644
index d287fad..0000000
---
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/add_replica/add_replica.2.get.http
+++ /dev/null
@@ -1,19 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-nc:asterix_nc1 /admin/storage/partition/0
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/add_replica/add_replica.2.pollget.http
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/add_replica/add_replica.2.pollget.http
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/add_replica/add_replica.2.pollget.http
new file mode 100644
index 0000000..6867a5d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/add_replica/add_replica.2.pollget.http
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+//polltimeoutsecs=30
+
+nc:asterix_nc1 /admin/storage/partition/0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.1.sto.cmd
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.1.sto.cmd
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.1.sto.cmd
new file mode 100644
index 0000000..7ddaa20
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.1.sto.cmd
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+nc:asterix_nc1 /addReplica 0 asterix_nc2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.10.ddl.sqlpp
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.10.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.10.ddl.sqlpp
new file mode 100644
index 0000000..f96d5a8
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.10.ddl.sqlpp
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+CREATE DATASET ds_2(MyType) PRIMARY KEY id;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.11.query.sqlpp
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.11.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.11.query.sqlpp
new file mode 100644
index 0000000..555954d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.11.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+SELECT value count(*)
+FROM Metadata.`Dataset`
+WHERE DatasetName = 'ds_2';
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.2.pollget.http
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.2.pollget.http
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.2.pollget.http
new file mode 100644
index 0000000..6867a5d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.2.pollget.http
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+//polltimeoutsecs=30
+
+nc:asterix_nc1 /admin/storage/partition/0
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.3.ddl.sqlpp
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.3.ddl.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.3.ddl.sqlpp
new file mode 100644
index 0000000..15bc3c5
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.3.ddl.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+CREATE TYPE MyType AS {
+ id : int
+};
+
+CREATE DATASET ds_1(MyType) PRIMARY KEY id;
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.4.sto.cmd
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.4.sto.cmd
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.4.sto.cmd
new file mode 100644
index 0000000..a5753f0
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.4.sto.cmd
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+nc:asterix_nc2 /promote 0 asterix_nc2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.5.get.http
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.5.get.http
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.5.get.http
new file mode 100644
index 0000000..4a53aed
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.5.get.http
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+nc:asterix_nc2 /admin/storage
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.6.post.http
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.6.post.http
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.6.post.http
new file mode 100644
index 0000000..2e8fc63
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.6.post.http
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/admin/cluster/partition/master?partition=0&node=asterix_nc2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.7.post.http
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.7.post.http
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.7.post.http
new file mode 100644
index 0000000..e8dca0b
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.7.post.http
@@ -0,0 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+/admin/cluster/metadataNode?node=asterix_nc2
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.8.pollget.http
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.8.pollget.http
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.8.pollget.http
new file mode 100644
index 0000000..32e2f78
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.8.pollget.http
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+//polltimeoutsecs=30
+
+/admin/cluster/summary
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.9.query.sqlpp
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.9.query.sqlpp
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.9.query.sqlpp
new file mode 100644
index 0000000..a612cbb
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/replication/metadata_failover/metadata_failover.9.query.sqlpp
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+SELECT value count(*)
+FROM Metadata.`Dataset`
+WHERE DatasetName = 'ds_1';
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
----------------------------------------------------------------------
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
index a635676..c3dfb3c 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/replication.xml
@@ -24,5 +24,10 @@
<output-dir compare="Text">add_replica</output-dir>
</compilation-unit>
</test-case>
+ <test-case FilePath="replication">
+ <compilation-unit name="metadata_failover">
+ <output-dir compare="Text">metadata_failover</output-dir>
+ </compilation-unit>
+ </test-case>
</test-group>
</test-suite>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/add_replica/add_replica.2.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/add_replica/add_replica.2.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/add_replica/add_replica.2.adm
index 3553d9c..6836f71 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/add_replica/add_replica.2.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/add_replica/add_replica.2.adm
@@ -2,6 +2,6 @@
"partition" : 0,
"replicas" : [ {
"location" : "127.0.0.1:2017",
- "status" : "DISCONNECTED"
+ "status" : "IN_SYNC"
} ]
} ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.11.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.11.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.11.adm
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.11.adm
@@ -0,0 +1 @@
+1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.2.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.2.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.2.adm
new file mode 100644
index 0000000..6836f71
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.2.adm
@@ -0,0 +1,7 @@
+[ {
+ "partition" : 0,
+ "replicas" : [ {
+ "location" : "127.0.0.1:2017",
+ "status" : "IN_SYNC"
+ } ]
+} ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.5.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.5.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.5.adm
new file mode 100644
index 0000000..3d3204d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.5.adm
@@ -0,0 +1,10 @@
+[ {
+ "partition" : 0,
+ "replicas" : [ ]
+}, {
+ "partition" : 2,
+ "replicas" : [ ]
+}, {
+ "partition" : 3,
+ "replicas" : [ ]
+} ]
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.6.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.6.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.6.adm
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.7.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.7.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.7.adm
new file mode 100644
index 0000000..e69de29
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.8.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.8.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.8.adm
new file mode 100644
index 0000000..fa5cfb4
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.8.adm
@@ -0,0 +1,38 @@
+{
+ "metadata_node" : "asterix_nc2",
+ "partitions" : {
+ "0" : {
+ "active" : true,
+ "activeNodeId" : "asterix_nc2",
+ "iodeviceNum" : 0,
+ "nodeId" : "asterix_nc1",
+ "partitionId" : 0,
+ "pendingActivation" : false
+ },
+ "1" : {
+ "active" : true,
+ "activeNodeId" : "asterix_nc1",
+ "iodeviceNum" : 1,
+ "nodeId" : "asterix_nc1",
+ "partitionId" : 1,
+ "pendingActivation" : false
+ },
+ "2" : {
+ "active" : true,
+ "activeNodeId" : "asterix_nc2",
+ "iodeviceNum" : 0,
+ "nodeId" : "asterix_nc2",
+ "partitionId" : 2,
+ "pendingActivation" : false
+ },
+ "3" : {
+ "active" : true,
+ "activeNodeId" : "asterix_nc2",
+ "iodeviceNum" : 1,
+ "nodeId" : "asterix_nc2",
+ "partitionId" : 3,
+ "pendingActivation" : false
+ }
+ },
+ "state" : "ACTIVE"
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.9.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.9.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.9.adm
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/replication/metadata_failover/metadata_failover.9.adm
@@ -0,0 +1 @@
+1
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IFaultToleranceStrategy.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IFaultToleranceStrategy.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IFaultToleranceStrategy.java
index 5c286cc..e871374 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IFaultToleranceStrategy.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IFaultToleranceStrategy.java
@@ -64,4 +64,12 @@ public interface IFaultToleranceStrategy {
*/
IFaultToleranceStrategy from(ICCServiceContext serviceCtx,
IReplicationStrategy replicationStrategy);
+ /**
+ * Performs the required steps to change the metadata node to {@code node}
+ *
+ * @param node
+ */
+ default void notifyMetadataNodeChange(String node) throws
HyracksDataException {
+ throw new UnsupportedOperationException(getClass() + " does not
support metadata node change");
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INCLifecycleMessage.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INCLifecycleMessage.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INCLifecycleMessage.java
index cb9fa8f..372a88a 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INCLifecycleMessage.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/INCLifecycleMessage.java
@@ -34,8 +34,8 @@ public interface INCLifecycleMessage extends IMessage {
REGISTRATION_TASKS_RESULT,
TAKEOVER_PARTITION_REQUEST,
TAKEOVER_PARTITION_RESPONSE,
- TAKEOVER_METADATA_NODE_REQUEST,
- TAKEOVER_METADATA_NODE_RESPONSE
+ METADATA_NODE_REQUEST,
+ METADATA_NODE_RESPONSE
}
/**
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
index 1c3f030..72a7f9d 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.replication;
+import java.util.List;
import java.util.Set;
import org.apache.hyracks.api.exceptions.HyracksDataException;
@@ -30,4 +31,6 @@ public interface IReplicaResourcesManager {
* @throws HyracksDataException
*/
long getPartitionsMinLSN(Set<Integer> partitions) throws
HyracksDataException;
+
+ List<String> getPartitionIndexesFiles(int partition, boolean relativePath)
throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationManager.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationManager.java
index b969bef..04c4437 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationManager.java
@@ -127,4 +127,11 @@ public interface IReplicationManager extends
IIOReplicationManager {
* @param buffer
*/
public void replicateTxnLogBatch(ByteBuffer buffer);
+
+ /**
+ * Registers {@code replica}. After registration, the replica will be
included in all replication events
+ *
+ * @param replica
+ */
+ void register(IPartitionReplica replica);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationThread.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationThread.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationThread.java
index a88b82a..5b9d4fa 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationThread.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicationThread.java
@@ -18,6 +18,7 @@
*/
package org.apache.asterix.common.replication;
+import java.nio.ByteBuffer;
import java.nio.channels.SocketChannel;
import org.apache.asterix.common.transactions.LogRecord;
@@ -27,13 +28,19 @@ public interface IReplicationThread extends Runnable {
/**
* Sends a notification to this thread that logRecord has been flushed.
*
- * @param logRecord
- * The log that has been flushed.
+ * @param logRecord The log that has been flushed.
*/
- public void notifyLogReplicationRequester(LogRecord logRecord);
+ void notifyLogReplicationRequester(LogRecord logRecord);
/**
- * @return The replication client socket channel.
+ * @return The replication socket channel.
*/
- public SocketChannel getReplicationClientSocket();
+ SocketChannel getChannel();
+
+ /**
+ * Gets a reusable buffer that can be used to send data
+ *
+ * @return the reusable buffer
+ */
+ ByteBuffer getReusableBuffer();
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/MetadataOnlyReplicationStrategy.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/MetadataOnlyReplicationStrategy.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/MetadataOnlyReplicationStrategy.java
index bd4b32f..f0bba41 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/MetadataOnlyReplicationStrategy.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/MetadataOnlyReplicationStrategy.java
@@ -69,21 +69,16 @@ public class MetadataOnlyReplicationStrategy implements
IReplicationStrategy {
if (metadataNode == null) {
throw new IllegalStateException("Invalid metadata node specified");
}
-
- if (cluster.getHighAvailability().getFaultTolerance().getReplica() ==
null
- ||
cluster.getHighAvailability().getFaultTolerance().getReplica().getNodeId() ==
null
- ||
cluster.getHighAvailability().getFaultTolerance().getReplica().getNodeId().isEmpty())
{
- throw new RuntimeDataException(ErrorCode.INVALID_CONFIGURATION,
- "One or more replicas must be specified for metadata
node.");
- }
-
final Set<Replica> replicas = new HashSet<>();
- for (String nodeId :
cluster.getHighAvailability().getFaultTolerance().getReplica().getNodeId()) {
- Node node = ClusterProperties.INSTANCE.getNodeById(nodeId);
- if (node == null) {
- throw new
RuntimeDataException(ErrorCode.INVALID_CONFIGURATION, "Invalid replica
specified: " + nodeId);
+ if (cluster.getHighAvailability().getFaultTolerance().getReplica() !=
null) {
+ for (String nodeId :
cluster.getHighAvailability().getFaultTolerance().getReplica().getNodeId()) {
+ Node node = ClusterProperties.INSTANCE.getNodeById(nodeId);
+ if (node == null) {
+ throw new
RuntimeDataException(ErrorCode.INVALID_CONFIGURATION,
+ "Invalid replica specified: " + nodeId);
+ }
+ replicas.add(new Replica(node));
}
- replicas.add(new Replica(node));
}
MetadataOnlyReplicationStrategy st = new
MetadataOnlyReplicationStrategy();
st.metadataNodeId = cluster.getMetadataNode();
@@ -91,4 +86,9 @@ public class MetadataOnlyReplicationStrategy implements
IReplicationStrategy {
st.metadataNodeReplicas = replicas;
return st;
}
+
+ @Override
+ public boolean isParticipant(String nodeId) {
+ return true;
+ }
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
index a3b2b50..19eee8f 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IReplicaManager.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Set;
import org.apache.asterix.common.replication.IPartitionReplica;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
public interface IReplicaManager {
@@ -54,4 +55,11 @@ public interface IReplicaManager {
* @return The list of partition
*/
Set<Integer> getPartitions();
+
+ /**
+ * Promotes a partition by making this node its master replica
+ *
+ * @param partition
+ */
+ void promote(int partition) throws HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/PartitionReplica.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/PartitionReplica.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/PartitionReplica.java
deleted file mode 100644
index 18733ce..0000000
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/PartitionReplica.java
+++ /dev/null
@@ -1,100 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.storage;
-
-import static
org.apache.asterix.common.storage.PartitionReplica.PartitionReplicaStatus.CATCHING_UP;
-import static
org.apache.asterix.common.storage.PartitionReplica.PartitionReplicaStatus.DISCONNECTED;
-import static
org.apache.asterix.common.storage.PartitionReplica.PartitionReplicaStatus.IN_SYNC;
-
-import org.apache.hyracks.util.JSONUtil;
-import org.apache.hyracks.util.annotations.ThreadSafe;
-
-import com.fasterxml.jackson.core.JsonProcessingException;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
-@ThreadSafe
-public class PartitionReplica {
-
- public enum PartitionReplicaStatus {
- /* replica is in-sync with master */
- IN_SYNC,
- /* replica is still catching up with master */
- CATCHING_UP,
- /* replica is not connected with master */
- DISCONNECTED
- }
-
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
- private final ReplicaIdentifier id;
- private PartitionReplicaStatus status = DISCONNECTED;
-
- public PartitionReplica(ReplicaIdentifier id) {
- this.id = id;
- }
-
- public synchronized PartitionReplicaStatus getStatus() {
- return status;
- }
-
- public ReplicaIdentifier getIdentifier() {
- return id;
- }
-
- public synchronized void sync() {
- if (status == IN_SYNC || status == CATCHING_UP) {
- return;
- }
- //TODO complete implementation
- }
-
- public JsonNode asJson() {
- ObjectNode json = OBJECT_MAPPER.createObjectNode();
- json.put("id", id.toString());
- json.put("state", status.name());
- return json;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- }
- if (o == null || getClass() != o.getClass()) {
- return false;
- }
- PartitionReplica that = (PartitionReplica) o;
- return id.equals(that.id);
- }
-
- @Override
- public int hashCode() {
- return id.hashCode();
- }
-
- @Override
- public String toString() {
- try {
- return JSONUtil.convertNode(asJson());
- } catch (JsonProcessingException e) {
- throw new IllegalStateException(e);
- }
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/cc7d2f0c/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
index 4aa6982..8cc11a8 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
@@ -71,6 +71,10 @@ public class ResourceReference {
return Paths.get(root, partition, dataverse, dataset, rebalance,
index);
}
+ public Path getFileRelativePath() {
+ return Paths.get(root, partition, dataverse, dataset, rebalance,
index, name);
+ }
+
protected static void parse(ResourceReference ref, String path) {
// format:
root/partition/dataverse/dataset/rebalanceCount/index/fileName
final String[] tokens = path.split(File.separator);
