Change in asterixdb[master]: Enable Replication Tests on Multi-NC Setup

[Murtadha Hubail (Code Review)]

Murtadha Hubail has submitted this change and it was merged.

Change subject: Enable Replication Tests on Multi-NC Setup
......................................................................


Enable Replication Tests on Multi-NC Setup

- Move replication IT from vagrant VMs to multi-NC setup.
- Add 'node' command to kill specific node process.
- Fix file path passing to replication manager.

Change-Id: I542b212e04469e2701690f464f821a5189b97f12
Reviewed-on: https://asterix-gerrit.ics.uci.edu/1364
Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamou...@gmail.com>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
---
M 
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
M 
asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
M 
asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
M 
asterixdb/asterix-installer/src/main/java/org/apache/asterix/installer/command/ConfigureCommand.java
A 
asterixdb/asterix-installer/src/main/resources/clusters/local/local_with_replication.xml
M 
asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
M 
asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
M 
asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixLifecycleIT.java
M 
asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java
M 
asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
A 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/data/fbu.adm
M 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.10.get.http
M 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.2.update.aql
C 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.3.node.aql
D 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.3.vscript.aql
M 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.5.get.http
R 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.8.mgx.aql
M 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.9.sleep.aql
M 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.2.update.aql
R 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.3.node.aql
R 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.4.sleep.aql
D 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.4.vscript.aql
R 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.5.query.aql
M 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.2.update.aql
C 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.3.node.aql
R 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.4.sleep.aql
D 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.4.vscript.aql
R 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.5.query.aql
D 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.6.txnqar.aql
C 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.2.node.aql
D 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.2.txnqbc.aql
R 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.3.sleep.aql
D 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.3.vscript.aql
R 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.4.query.aql
M 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.10.adm
M 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.5.adm
A 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/bulkload/bulkload.5.aql
A 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/mem_component_recovery/mem_component_recovery.5.aql
A 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/metadata_node/metadata_node.4.aql
M 
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/util/ClusterStateManager.java
M 
asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
M asterixdb/pom.xml
42 files changed, 276 insertions(+), 605 deletions(-)

Approvals:
  abdullah alamoudi: Looks good to me, approved
  Jenkins: Verified; No violations found; Verified



diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
index a8095af..b0b2e36 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/AsterixHyracksIntegrationUtil.java
@@ -78,9 +78,10 @@
         // Starts ncs.
         List<String> nodes = propertiesAccessor.getNodeNames();
         List<NodeControllerService> nodeControllers = new ArrayList<>();
+        List<Thread> startupThreads = new ArrayList<>();
         for (String ncName : nodes) {
-            NodeControllerService nodeControllerService =
-                    new 
NodeControllerService(fixupIODevices(createNCConfig(ncName)));
+            NodeControllerService nodeControllerService = new 
NodeControllerService(
+                    fixupIODevices(createNCConfig(ncName)));
             nodeControllers.add(nodeControllerService);
             Thread ncStartThread = new Thread("IntegrationUtil-" + ncName) {
                 @Override
@@ -93,7 +94,11 @@
                 }
             };
             ncStartThread.start();
-            ncStartThread.join();
+            startupThreads.add(ncStartThread);
+        }
+        //wait until all NCs complete their startup
+        for (Thread thread : startupThreads) {
+            thread.join();
         }
         hcc = new HyracksConnection(cc.getConfig().clientNetIpAddress, 
cc.getConfig().clientNetPort);
         ncs = nodeControllers.toArray(new 
NodeControllerService[nodeControllers.size()]);
@@ -156,7 +161,6 @@
         }
         return ncConfig;
     }
-
 
     public String[] getNcNames() {
         return propertiesAccessor.getNodeNames().toArray(new 
String[propertiesAccessor.getNodeNames().size()]);
diff --git 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
index 49730a0..158317b 100644
--- 
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
+++ 
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/http/servlet/ClusterAPIServlet.java
@@ -77,9 +77,11 @@
                 case "/replication":
                     json = getReplicationJSON();
                     break;
+                case "/summary":
+                    json = getClusterStateSummaryJSON();
+                    break;
                 default:
                     throw new IllegalArgumentException();
-
             }
             response.setStatus(HttpServletResponse.SC_OK);
             responseWriter.write(json.toString(4));
@@ -90,6 +92,10 @@
             response.sendError(HttpServletResponse.SC_INTERNAL_SERVER_ERROR, 
e.toString());
         }
         responseWriter.flush();
+    }
+
+    protected JSONObject getClusterStateSummaryJSON() throws JSONException {
+        return ClusterStateManager.INSTANCE.getClusterStateSummary();
     }
 
     protected JSONObject getReplicationJSON() throws JSONException {
@@ -117,8 +123,7 @@
         return AbstractAsterixProperties.getImplementations();
     }
 
-    protected JSONObject getClusterStateJSON(HttpServletRequest request, 
String pathToNode)
-            throws JSONException {
+    protected JSONObject getClusterStateJSON(HttpServletRequest request, 
String pathToNode) throws JSONException {
         JSONObject json;
         json = ClusterStateManager.INSTANCE.getClusterStateDescription();
         Map<String, Object> allProperties = getAllClusterProperties();
diff --git 
a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
 
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
index 4080325..2da57e3 100644
--- 
a/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
+++ 
b/asterixdb/asterix-common/src/test/java/org/apache/asterix/test/aql/TestExecutor.java
@@ -80,11 +80,11 @@
     // see
     // 
https://stackoverflow.com/questions/417142/what-is-the-maximum-length-of-a-url-in-different-browsers/417184
     private static final long MAX_URL_LENGTH = 2000l;
-    private static final Pattern JAVA_BLOCK_COMMENT_PATTERN =
-            Pattern.compile("/\\*.*\\*/", Pattern.MULTILINE | Pattern.DOTALL);
+    private static final Pattern JAVA_BLOCK_COMMENT_PATTERN = 
Pattern.compile("/\\*.*\\*/",
+            Pattern.MULTILINE | Pattern.DOTALL);
     private static final Pattern REGEX_LINES_PATTERN = 
Pattern.compile("^(-)?/(.*)/([im]*)$");
-    private static final Pattern POLL_TIMEOUT_PATTERN =
-            Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)", 
Pattern.MULTILINE);
+    private static final Pattern POLL_TIMEOUT_PATTERN = 
Pattern.compile("polltimeoutsecs=(\\d+)(\\D|$)",
+            Pattern.MULTILINE);
     private static final Pattern POLL_DELAY_PATTERN = 
Pattern.compile("polldelaysecs=(\\d+)(\\D|$)", Pattern.MULTILINE);
     public static final int TRUNCATE_THRESHOLD = 16384;
 
@@ -199,9 +199,8 @@
 
     private void throwLineChanged(File scriptFile, String lineExpected, String 
lineActual, int num)
             throws ComparisonException {
-        throw new ComparisonException(
-                "Result for " + scriptFile + " changed at line " + num + ":\n< 
"
-                        + truncateIfLong(lineExpected) + "\n> " + 
truncateIfLong(lineActual));
+        throw new ComparisonException("Result for " + scriptFile + " changed 
at line " + num + ":\n< "
+                + truncateIfLong(lineExpected) + "\n> " + 
truncateIfLong(lineActual));
     }
 
     private String truncateIfLong(String string) {
@@ -211,8 +210,7 @@
         final StringBuilder truncatedString = new StringBuilder(string);
         truncatedString.setLength(TRUNCATE_THRESHOLD);
         truncatedString.append("\n<truncated ")
-                .append(StorageUtil.toHumanReadableSize(string.length() - 
TRUNCATE_THRESHOLD))
-                .append("...>");
+                .append(StorageUtil.toHumanReadableSize(string.length() - 
TRUNCATE_THRESHOLD)).append("...>");
         return truncatedString.toString();
     }
 
@@ -327,8 +325,8 @@
                 }
                 Matcher m = REGEX_LINES_PATTERN.matcher(lineExpected);
                 if (!m.matches()) {
-                    throw new IllegalArgumentException("Each line of regex 
file must conform to: [-]/regex/[flags]: "
-                            + expectedFile);
+                    throw new IllegalArgumentException(
+                            "Each line of regex file must conform to: 
[-]/regex/[flags]: " + expectedFile);
                 }
                 String negateStr = m.group(1);
                 String expression = m.group(2);
@@ -346,8 +344,8 @@
                 if (match && !negate || negate && !match) {
                     continue;
                 }
-                throw new Exception("Result for " + scriptFile + ": expected 
pattern '" + expression +
-                        "' not found in result.");
+                throw new Exception(
+                        "Result for " + scriptFile + ": expected pattern '" + 
expression + "' not found in result.");
             }
         } catch (Exception e) {
             System.err.println("Actual results file: " + 
actualFile.toString());
@@ -404,14 +402,12 @@
                 String[] errors = { 
result.getJSONArray("error-code").getString(0), result.getString("summary"),
                         result.getString("stacktrace") };
                 GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, errors[2]);
-                exceptionMsg = "HTTP operation failed: " + errors[0]
-                        + "\nSTATUS LINE: " + httpResponse.getStatusLine()
+                exceptionMsg = "HTTP operation failed: " + errors[0] + 
"\nSTATUS LINE: " + httpResponse.getStatusLine()
                         + "\nSUMMARY: " + errors[1] + "\nSTACKTRACE: " + 
errors[2];
             } catch (Exception e) {
                 // whoops, not JSON (e.g. 404) - just include the body
                 GlobalConfig.ASTERIX_LOGGER.log(Level.SEVERE, errorBody);
-                exceptionMsg = "HTTP operation failed:"
-                        + "\nSTATUS LINE: " + httpResponse.getStatusLine()
+                exceptionMsg = "HTTP operation failed:" + "\nSTATUS LINE: " + 
httpResponse.getStatusLine()
                         + "\nERROR_BODY: " + errorBody;
             }
             throw new Exception(exceptionMsg);
@@ -457,7 +453,7 @@
     }
 
     private List<CompilationUnit.Parameter> injectStatement(String statement, 
String stmtParamName,
-                                                            
List<CompilationUnit.Parameter> otherParams) {
+            List<CompilationUnit.Parameter> otherParams) {
         CompilationUnit.Parameter stmtParam = new CompilationUnit.Parameter();
         stmtParam.setName(stmtParamName);
         stmtParam.setValue(statement);
@@ -488,7 +484,7 @@
     }
 
     private HttpUriRequest constructGetMethod(String endpoint, OutputFormat 
fmt,
-                                              List<CompilationUnit.Parameter> 
params) {
+            List<CompilationUnit.Parameter> params) {
 
         HttpUriRequest method = constructGetMethod(endpoint, params);
         // Set accepted output response type
@@ -506,7 +502,7 @@
     }
 
     private HttpUriRequest constructPostMethod(String endpoint, OutputFormat 
fmt,
-                                              List<CompilationUnit.Parameter> 
params) {
+            List<CompilationUnit.Parameter> params) {
 
         HttpUriRequest method = constructPostMethod(endpoint, params);
         // Set accepted output response type
@@ -565,8 +561,7 @@
     // Insert and Delete statements are executed here
     public void executeUpdate(String str, String url) throws Exception {
         // Create a method instance.
-        HttpUriRequest request = RequestBuilder.post(url)
-                .setEntity(new StringEntity(str, StandardCharsets.UTF_8))
+        HttpUriRequest request = RequestBuilder.post(url).setEntity(new 
StringEntity(str, StandardCharsets.UTF_8))
                 .build();
 
         // Execute the method.
@@ -593,10 +588,8 @@
         final String url = getEndpoint(Servlets.QUERY_RESULT);
 
         // Create a method instance.
-        HttpUriRequest request = RequestBuilder.get(url)
-                .addParameter("handle", handle)
-                .setHeader("Accept", fmt.mimeType())
-                .build();
+        HttpUriRequest request = 
RequestBuilder.get(url).addParameter("handle", handle)
+                .setHeader("Accept", fmt.mimeType()).build();
 
         HttpResponse response = executeAndCheckHttpRequest(request);
         return response.getEntity().getContent();
@@ -610,8 +603,7 @@
     // create function statement
     public void executeDDL(String str, String url) throws Exception {
         // Create a method instance.
-        HttpUriRequest request = RequestBuilder.post(url)
-                .setEntity(new StringEntity(str, StandardCharsets.UTF_8))
+        HttpUriRequest request = RequestBuilder.post(url).setEntity(new 
StringEntity(str, StandardCharsets.UTF_8))
                 .build();
 
         // Execute the method.
@@ -622,8 +614,8 @@
     // and returns the contents as a string
     // This string is later passed to REST API for execution.
     public String readTestFile(File testFile) throws Exception {
-        BufferedReader reader =
-                new BufferedReader(new InputStreamReader(new 
FileInputStream(testFile), StandardCharsets.UTF_8));
+        BufferedReader reader = new BufferedReader(
+                new InputStreamReader(new FileInputStream(testFile), 
StandardCharsets.UTF_8));
         String line;
         StringBuilder stringBuilder = new StringBuilder();
         String ls = System.getProperty("line.separator");
@@ -746,7 +738,7 @@
                 long limitTime = startTime + 
TimeUnit.SECONDS.toMillis(timeoutSecs);
                 ctx.setType(ctx.getType().substring("poll".length()));
                 Exception finalException;
-                LOGGER.fine("polling for up to " + timeoutSecs + " seconds w/ 
" + retryDelaySecs  + " second(s) delay");
+                LOGGER.fine("polling for up to " + timeoutSecs + " seconds w/ 
" + retryDelaySecs + " second(s) delay");
                 while (true) {
                     try {
                         executeTest(testCaseCtx, ctx, statement, 
isDmlRecoveryTest, pb, cUnit, queryCount,
@@ -901,8 +893,8 @@
             case "get":
             case "post":
                 if (!"http".equals(ctx.extension())) {
-                    throw new IllegalArgumentException("Unexpected format for 
method " + ctx.getType() + ": "
-                            + ctx.extension());
+                    throw new IllegalArgumentException(
+                            "Unexpected format for method " + ctx.getType() + 
": " + ctx.extension());
                 }
                 fmt = OutputFormat.forCompilationUnit(cUnit);
                 String endpoint = stripJavaComments(statement).trim();
@@ -924,7 +916,7 @@
                 queryCount.increment();
                 break;
             case "server": // (start <test server name> <port>
-                           // [<arg1>][<arg2>][<arg3>]...|stop (<port>|all))
+                               // [<arg1>][<arg2>][<arg3>]...|stop 
(<port>|all))
                 try {
                     lines = statement.trim().split("\n");
                     String[] command = lines[lines.length - 1].trim().split(" 
");
@@ -1001,11 +993,35 @@
                         throw new Exception("invalid library format");
                 }
                 break;
+            case "node":
+                command = stripJavaComments(statement).trim().split(" ");
+                String commandType = command[0];
+                String nodeId = command[1];
+                if (commandType.equals("kill")) {
+                    killNC(nodeId, cUnit);
+                }
+                break;
             default:
                 throw new IllegalArgumentException("No statements of type " + 
ctx.getType());
         }
     }
 
+    private void killNC(String nodeId, CompilationUnit cUnit) throws Exception 
{
+        //get node process id
+        OutputFormat fmt = OutputFormat.forCompilationUnit(cUnit);
+        String endpoint = "/admin/cluster/node/" + nodeId + "/config";
+        InputStream executeJSONGet = executeJSONGet(fmt, "http://"; + host + 
":" + port + endpoint);
+        StringWriter actual = new StringWriter();
+        IOUtils.copy(executeJSONGet, actual, StandardCharsets.UTF_8);
+        String config = actual.toString();
+        String nodePid = StringUtils.substringBetween(config, "\"pid\": ", 
",").trim();
+        if (nodePid == null) {
+            throw new IllegalArgumentException("Coud not find process for node 
id: " + nodeId);
+        }
+        ProcessBuilder pb = new ProcessBuilder("kill", "-9", nodePid);
+        pb.start().waitFor();
+    }
+
     public void executeTest(String actualPath, TestCaseContext testCaseCtx, 
ProcessBuilder pb,
             boolean isDmlRecoveryTest, TestGroup failedGroup) throws Exception 
{
         File testFile;
diff --git 
a/asterixdb/asterix-installer/src/main/java/org/apache/asterix/installer/command/ConfigureCommand.java
 
b/asterixdb/asterix-installer/src/main/java/org/apache/asterix/installer/command/ConfigureCommand.java
index aeb118f..6d2b4b9 100644
--- 
a/asterixdb/asterix-installer/src/main/java/org/apache/asterix/installer/command/ConfigureCommand.java
+++ 
b/asterixdb/asterix-installer/src/main/java/org/apache/asterix/installer/command/ConfigureCommand.java
@@ -21,6 +21,8 @@
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileOutputStream;
+import java.nio.file.Files;
+import java.nio.file.Paths;
 
 import javax.xml.bind.JAXBContext;
 import javax.xml.bind.JAXBException;
@@ -42,6 +44,7 @@
     @Override
     protected void execCommand() throws Exception {
         configureCluster("local", "local.xml");
+        configureCluster("local", "local_with_replication.xml");
         configureCluster("demo", "demo.xml");
 
         String installerConfPath = InstallerDriver.getManagixHome() + 
File.separator + InstallerDriver.MANAGIX_CONF_XML;
@@ -51,9 +54,8 @@
 
         configuration.setConfigured(true);
         
configuration.getBackup().setBackupDir(InstallerDriver.getManagixHome() + 
File.separator + "backup");
-        configuration.getZookeeper().setHomeDir(
-                InstallerDriver.getManagixHome() + File.separator + 
InstallerDriver.MANAGIX_INTERNAL_DIR
-                        + File.separator + "zookeeper_home");
+        
configuration.getZookeeper().setHomeDir(InstallerDriver.getManagixHome() + 
File.separator
+                + InstallerDriver.MANAGIX_INTERNAL_DIR + File.separator + 
"zookeeper_home");
         
configuration.getZookeeper().getServers().setJavaHome(System.getProperty("java.home"));
 
         Marshaller marshaller = ctx.createMarshaller();
@@ -61,11 +63,14 @@
         marshaller.marshal(configuration, new 
FileOutputStream(installerConfPath));
     }
 
-    private void configureCluster(String dir, String file) throws 
JAXBException, PropertyException,
-            FileNotFoundException {
+    private void configureCluster(String dir, String file)
+            throws JAXBException, PropertyException, FileNotFoundException {
         String clusterDir = InstallerDriver.getManagixHome() + File.separator 
+ "clusters" + File.separator + dir;
         String localClusterPath = clusterDir + File.separator + file;
 
+        if (!Files.exists(Paths.get(localClusterPath))) {
+            return;
+        }
         Cluster cluster = EventUtil.getCluster(localClusterPath);
         String workingDir = clusterDir + File.separator + "working_dir";
         cluster.setWorkingDir(new WorkingDir(workingDir, true));
diff --git 
a/asterixdb/asterix-installer/src/main/resources/clusters/local/local_with_replication.xml
 
b/asterixdb/asterix-installer/src/main/resources/clusters/local/local_with_replication.xml
new file mode 100644
index 0000000..e6a3547
--- /dev/null
+++ 
b/asterixdb/asterix-installer/src/main/resources/clusters/local/local_with_replication.xml
@@ -0,0 +1,76 @@
+<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
+<!--
+ ! Licensed to the Apache Software Foundation (ASF) under one
+ ! or more contributor license agreements.  See the NOTICE file
+ ! distributed with this work for additional information
+ ! regarding copyright ownership.  The ASF licenses this file
+ ! to you under the Apache License, Version 2.0 (the
+ ! "License"); you may not use this file except in compliance
+ ! with the License.  You may obtain a copy of the License at
+ !
+ !   http://www.apache.org/licenses/LICENSE-2.0
+ !
+ ! Unless required by applicable law or agreed to in writing,
+ ! software distributed under the License is distributed on an
+ ! "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ ! KIND, either express or implied.  See the License for the
+ ! specific language governing permissions and limitations
+ ! under the License.
+ !-->
+<cluster xmlns="cluster">
+  <java_home>/usr/lib/jvm/jdk-8-oracle-x64/jre</java_home>
+  <log_dir>/tmp/asterix/logs</log_dir>
+  <txn_log_dir>/tmp/asterix/txnLogs</txn_log_dir>
+  <store>storage</store>
+  <working_dir>
+    <dir>/tmp/asterix-installer</dir>
+    <NFS>true</NFS>
+  </working_dir>
+  <!-- Sets the time duration between two heartbeats from each node controller 
in milliseconds (default: 10000) -->
+  <heartbeat_period>1000</heartbeat_period>
+  <!-- Sets the maximum number of missed heartbeats before a node is marked as 
dead (default: 5) -->
+  <max_heartbeat_lapse_periods>5</max_heartbeat_lapse_periods>
+  <!-- Sets the time duration between two profile dumps from each node 
controller in milliseconds. 0 to disable. (default: 0) -->
+  <profile_dump_period>0</profile_dump_period>
+  <!-- Sets the default number of job attempts allowed if not specified in the 
job specification. (default: 5) -->
+  <default_max_job_attempts>5</default_max_job_attempts>
+  <!-- Limits the number of historical jobs remembered by the system to the 
specified value. (default: 10) -->
+  <job_history_size>10</job_history_size>
+  <!-- Limits the amount of time results for asynchronous jobs should be 
retained by the system in milliseconds. (default: 24 hours) -->
+  <result_time_to_live>86400000</result_time_to_live>
+  <!-- The duration within which an instance of the result cleanup should be 
invoked in milliseconds. (default: 1 minute) -->
+  <result_sweep_threshold>60000</result_sweep_threshold>
+
+  <metadata_node>nc1</metadata_node>
+
+  <data_replication>
+    <enabled>true</enabled>
+    <replication_port>2000</replication_port>
+    <replication_factor>2</replication_factor>
+    <auto_failover>true</auto_failover>
+    <replication_time_out>10</replication_time_out>
+  </data_replication>
+
+  <master_node>
+    <id>master</id>
+    <client_ip>127.0.0.1</client_ip>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <client_port>1098</client_port>
+    <cluster_port>1099</cluster_port>
+    <http_port>8888</http_port>
+  </master_node>
+  <node>
+    <id>nc1</id>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <txn_log_dir>/tmp/asterix/nc1/txnLogs</txn_log_dir>
+    <iodevices>/tmp/asterix/nc1/p1,/tmp/asterix/nc1/p2</iodevices>
+    <replication_port>2000</replication_port>
+  </node>
+  <node>
+    <id>nc2</id>
+    <cluster_ip>127.0.0.1</cluster_ip>
+    <txn_log_dir>/tmp/asterix/nc2/txnLogs</txn_log_dir>
+    <iodevices>/tmp/asterix/nc2/p1,/tmp/asterix/nc2/p2</iodevices>
+    <replication_port>2001</replication_port>
+  </node>
+</cluster>
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
index ae98d19..5dde951 100644
--- 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
+++ 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixExternalLibraryIT.java
@@ -55,7 +55,7 @@
     @BeforeClass
     public static void setUp() throws Exception {
         try {
-            AsterixInstallerIntegrationUtil.init();
+            
AsterixInstallerIntegrationUtil.init(AsterixInstallerIntegrationUtil.LOCAL_CLUSTER_PATH);
             File asterixInstallerProjectDir = new 
File(System.getProperty("user.dir"));
             String asterixExternalLibraryPath = 
asterixInstallerProjectDir.getAbsolutePath() + File.separator
                     + LIBRARY_PATH;
diff --git 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
index 34a8733..e840796 100644
--- 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
+++ 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixInstallerIntegrationUtil.java
@@ -53,19 +53,22 @@
     private static final int zookeeperClientPort = 2900;
     private static final int zookeeperTestClientPort = 3945;
     private static IHyracksClientConnection hcc;
+    private static final String CLUSTERS_BASE_PATH = "clusters" + 
File.separator + "local" + File.separator;
+    public static final String LOCAL_CLUSTER_PATH = CLUSTERS_BASE_PATH + 
File.separator + "local.xml";
+    public static final String LOCAL_CLUSTER_WITH_REPLICATION_PATH = 
CLUSTERS_BASE_PATH + File.separator
+            + "local_with_replication.xml";
 
     public static void deinit() throws Exception {
         deleteInstance();
         stopZookeeper();
     }
 
-    public static void init() throws Exception {
+    public static void init(String clusterPath) throws Exception {
         managixHome = getManagixHome();
         System.setProperty("log4j.configuration",
                 managixHome + File.separator + "conf" + File.separator + 
"log4j.properties");
 
-        clusterConfigurationPath = managixHome + File.separator + "clusters" + 
File.separator + "local" + File.separator
-                + "local.xml";
+        clusterConfigurationPath = managixHome + File.separator + clusterPath;
 
         InstallerDriver.setManagixHome(managixHome);
 
diff --git 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixLifecycleIT.java
 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixLifecycleIT.java
index 04e73ed..4dfc5c5 100644
--- 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixLifecycleIT.java
+++ 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixLifecycleIT.java
@@ -53,7 +53,7 @@
 
     @BeforeClass
     public static void setUp() throws Exception {
-        AsterixInstallerIntegrationUtil.init();
+        
AsterixInstallerIntegrationUtil.init(AsterixInstallerIntegrationUtil.LOCAL_CLUSTER_PATH);
         TestCaseContext.Builder b = new TestCaseContext.Builder();
         testCaseCollection = b.build(new File(PATH_BASE));
         File outdir = new File(PATH_ACTUAL);
diff --git 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java
 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java
index d567145..96b420b 100644
--- 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java
+++ 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/AsterixRestartIT.java
@@ -21,17 +21,20 @@
 import java.io.File;
 import java.util.ArrayList;
 import java.util.Collection;
-import java.util.List;
 import java.util.Map;
 import java.util.logging.Logger;
 
-import org.apache.asterix.common.config.AsterixStorageProperties;
 import org.apache.asterix.event.model.AsterixInstance.State;
 import org.apache.asterix.test.aql.TestExecutor;
 import org.apache.asterix.test.base.RetainLogsRule;
 import org.apache.asterix.testframework.context.TestCaseContext;
 import org.apache.commons.lang3.StringUtils;
-import org.junit.*;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.Test;
 import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
@@ -73,7 +76,7 @@
                     + "resources" + File.separator + "integrationts" + 
File.separator + "restart"
                     + File.separator + "scripts";
             env.put("SCRIPT_HOME", scriptHomePath);
-            AsterixInstallerIntegrationUtil.init();
+            
AsterixInstallerIntegrationUtil.init(AsterixInstallerIntegrationUtil.LOCAL_CLUSTER_PATH);
         } catch (Throwable th) {
             th.printStackTrace();
             throw th;
diff --git 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
index 37aa59d..5f47849 100644
--- 
a/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
+++ 
b/asterixdb/asterix-installer/src/test/java/org/apache/asterix/installer/test/ReplicationIT.java
@@ -19,152 +19,78 @@
 package org.apache.asterix.installer.test;
 
 import java.io.File;
-import java.io.FilenameFilter;
-import java.io.IOException;
-import java.io.InputStream;
-import java.nio.charset.StandardCharsets;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Map;
 import java.util.logging.Logger;
 
+import org.apache.asterix.event.model.AsterixInstance.State;
 import org.apache.asterix.test.aql.TestExecutor;
+import org.apache.asterix.test.base.RetainLogsRule;
 import org.apache.asterix.testframework.context.TestCaseContext;
-import org.apache.commons.io.IOUtils;
 import org.apache.commons.lang3.StringUtils;
-import org.codehaus.plexus.util.FileUtils;
 import org.junit.After;
-import org.junit.AfterClass;
 import org.junit.Before;
 import org.junit.BeforeClass;
+import org.junit.Rule;
 import org.junit.Test;
+import org.junit.rules.TestRule;
 import org.junit.runner.RunWith;
 import org.junit.runners.Parameterized;
-import org.junit.runners.Parameterized.Parameters;
 
 @RunWith(Parameterized.class)
 public class ReplicationIT {
 
-    private static final String PATH_BASE = StringUtils
-            .join(new String[] { "src", "test", "resources", "integrationts", 
"replication" }, File.separator);
-    private static final String CLUSTER_BASE = StringUtils
-            .join(new String[] { "src", "test", "resources", "clusterts" }, 
File.separator);
-    private static final String PATH_ACTUAL = "target" + File.separator + 
"repliationtest" + File.separator;
-    private static String managixFolderName;
+    private static final String PATH_BASE = 
"src/test/resources/integrationts/replication/";
+    private static final String PATH_ACTUAL = "target" + File.separator + 
"ittest" + File.separator;
     private static final Logger LOGGER = 
Logger.getLogger(ReplicationIT.class.getName());
-    private static File asterixProjectDir = new 
File(System.getProperty("user.dir"));
-    private static final String CLUSTER_CC_ADDRESS = "10.10.0.2";
-    private static final int CLUSTER_CC_API_PORT = 19002;
+    private static String reportPath = new File(
+            StringUtils.join(new String[] { "target", "failsafe-reports" }, 
File.separator)).getAbsolutePath();
+
+    private final TestExecutor testExecutor = new TestExecutor();
+    private TestCaseContext tcCtx;
+    private static String scriptHomePath;
+    private static File asterixInstallerPath;
     private static ProcessBuilder pb;
     private static Map<String, String> env;
-    private final static TestExecutor testExecutor = new 
TestExecutor(CLUSTER_CC_ADDRESS, CLUSTER_CC_API_PORT);
-    private static String SCRIPT_HOME = "/vagrant/scripts/";
-    private static String MANAGIX_HOME = "/tmp/asterix/bin/managix ";
-    private static final String INSTANCE_NAME = "asterix";
-    protected TestCaseContext tcCtx;
 
     public ReplicationIT(TestCaseContext tcCtx) {
         this.tcCtx = tcCtx;
     }
 
+    @Rule
+    public TestRule retainLogs = new 
RetainLogsRule(AsterixInstallerIntegrationUtil.getManagixHome(), reportPath);
+
     @BeforeClass
     public static void setUp() throws Exception {
-        File outdir = new File(PATH_ACTUAL);
-        outdir.mkdirs();
-
-        // vagrant setup
-        File installerTargetDir = new File(asterixProjectDir, "target");
-        String[] installerFiles = installerTargetDir.list(new FilenameFilter() 
{
-            @Override
-            public boolean accept(File dir, String name) {
-                return new File(dir, name).isDirectory() && 
name.startsWith("asterix-installer")
-                        && name.endsWith("binary-assembly");
-            }
-        });
-
-        if (installerFiles == null || installerFiles.length == 0) {
-            throw new Exception("Couldn't find installer binaries");
+        try {
+            pb = new ProcessBuilder();
+            env = pb.environment();
+            asterixInstallerPath = new File(System.getProperty("user.dir"));
+            scriptHomePath = asterixInstallerPath + File.separator + "src" + 
File.separator + "test" + File.separator
+                    + "resources" + File.separator + "integrationts" + 
File.separator + "replication" + File.separator
+                    + "scripts";
+            env.put("SCRIPT_HOME", scriptHomePath);
+        } catch (Throwable th) {
+            th.printStackTrace();
+            throw th;
         }
-
-        managixFolderName = installerFiles[0];
-
-        //copy tests data
-        FileUtils.copyDirectoryStructure(
-                new File(StringUtils.join(
-                        new String[] { "..", "asterix-replication", "src", 
"test", "resources", "data" },
-                        File.separator)),
-                new File(StringUtils.join(new String[] { "src", "test", 
"resources", "clusterts", "data" },
-                        File.separator)));
-
-        //copy tests scripts
-        FileUtils.copyDirectoryStructure(
-                new File(StringUtils.join(
-                        new String[] { "..", "asterix-replication", "src", 
"test", "resources", "scripts" },
-                        File.separator)),
-                new File(StringUtils.join(new String[] { "src", "test", 
"resources", "clusterts", "scripts" },
-                        File.separator)));
-
-        invoke("cp", "-r", installerTargetDir.toString() + "/" + 
managixFolderName,
-                asterixProjectDir + "/" + CLUSTER_BASE);
-
-        remoteInvoke("cp -r /vagrant/" + managixFolderName + " /tmp/asterix");
-
-        pb = new ProcessBuilder();
-        env = pb.environment();
-        env.put("SCRIPT_HOME", SCRIPT_HOME);
-        env.put("MANAGIX_HOME", MANAGIX_HOME);
-        File cwd = new File(asterixProjectDir.toString() + "/" + CLUSTER_BASE);
-        pb.directory(cwd);
-        pb.redirectErrorStream(true);
-
-        //make scripts executable
-        String chmodScriptsCmd = "chmod -R +x " + SCRIPT_HOME;
-        remoteInvoke(chmodScriptsCmd, "cc");
-        remoteInvoke(chmodScriptsCmd, "nc1");
-        remoteInvoke(chmodScriptsCmd, "nc2");
-
-        //managix configure
-        logOutput(managixInvoke("configure").getInputStream());
-
-        //managix validate
-        String validateOutput = 
IOUtils.toString(managixInvoke("validate").getInputStream(),
-                StandardCharsets.UTF_8.name());
-        if (validateOutput.contains("ERROR")) {
-            throw new Exception("Managix validate error: " + validateOutput);
-        }
-    }
-
-    @AfterClass
-    public static void tearDown() throws Exception {
-        //remove files
-        remoteInvoke("rm -rf /vagrant/asterix");
     }
 
     @Before
-    public void beforeTest() throws Exception {
-        //create instance
-        managixInvoke("create -n " + INSTANCE_NAME + " -c 
/vagrant/cluster_with_replication.xml").getInputStream();
+    public void before() throws Exception {
+        LOGGER.info("Creating new instance...");
+        
AsterixInstallerIntegrationUtil.init(AsterixInstallerIntegrationUtil.LOCAL_CLUSTER_WITH_REPLICATION_PATH);
+        LOGGER.info("Instacne created.");
+        
AsterixInstallerIntegrationUtil.transformIntoRequiredState(State.ACTIVE);
+        LOGGER.info("Instance is in ACTIVE state.");
     }
 
     @After
-    public void afterTest() throws Exception {
-        //stop instance
-        managixInvoke("stop -n " + INSTANCE_NAME);
-
-        //verify that all processes have been stopped
-        String killProcesses = "kill_cc_and_nc.sh";
-        executeVagrantScript("cc", killProcesses);
-        executeVagrantScript("nc1", killProcesses);
-        executeVagrantScript("nc2", killProcesses);
-
-        //delete storage
-        String deleteStorage = "delete_storage.sh";
-        executeVagrantScript("cc", deleteStorage);
-        executeVagrantScript("nc1", deleteStorage);
-        executeVagrantScript("nc2", deleteStorage);
-
-        //delete instance
-        managixInvoke("delete -n " + INSTANCE_NAME);
+    public void after() throws Exception {
+        LOGGER.info("Destroying instance...");
+        AsterixInstallerIntegrationUtil.deinit();
+        LOGGER.info("Instance destroyed.");
     }
 
     @Test
@@ -172,7 +98,7 @@
         testExecutor.executeTest(PATH_ACTUAL, tcCtx, pb, false);
     }
 
-    @Parameters(name = "ReplicationIT {index}: {0}")
+    @Parameterized.Parameters(name = "ReplicationIT {index}: {0}")
     public static Collection<Object[]> tests() throws Exception {
         Collection<Object[]> testArgs = 
buildTestsInXml(TestCaseContext.DEFAULT_TESTSUITE_XML_NAME);
         if (testArgs.size() == 0) {
@@ -182,78 +108,11 @@
     }
 
     protected static Collection<Object[]> buildTestsInXml(String xmlfile) 
throws Exception {
-        Collection<Object[]> testArgs = new ArrayList<Object[]>();
+        Collection<Object[]> testArgs = new ArrayList<>();
         TestCaseContext.Builder b = new TestCaseContext.Builder();
         for (TestCaseContext ctx : b.build(new File(PATH_BASE), xmlfile)) {
             testArgs.add(new Object[] { ctx });
         }
         return testArgs;
-    }
-
-    public static boolean checkOutput(InputStream input, String 
requiredSubString) {
-        String candidate;
-        try {
-            candidate = IOUtils.toString(input, StandardCharsets.UTF_8.name());
-        } catch (IOException e) {
-            LOGGER.warning("Could not check output of subprocess");
-            return false;
-        }
-        return candidate.contains(requiredSubString);
-    }
-
-    public static boolean checkOutput(String candidate, String 
requiredSubString) {
-        return candidate.contains(requiredSubString);
-    }
-
-    public static String processOut(Process p) throws IOException {
-        InputStream input = p.getInputStream();
-        return IOUtils.toString(input, StandardCharsets.UTF_8.name());
-    }
-
-    public static void logOutput(InputStream input) {
-        try {
-            LOGGER.info(IOUtils.toString(input, 
StandardCharsets.UTF_8.name()));
-        } catch (IOException e) {
-            LOGGER.warning("Could not print output of subprocess");
-        }
-    }
-
-    private static Process invoke(String... args) throws Exception {
-        ProcessBuilder pb = new ProcessBuilder(args);
-        pb.redirectErrorStream(true);
-        Process p = pb.start();
-        return p;
-    }
-
-    private static Process remoteInvoke(String cmd) throws Exception {
-        ProcessBuilder pb = new ProcessBuilder("vagrant", "ssh", "cc", "-c", 
"MANAGIX_HOME=/tmp/asterix/ " + cmd);
-        File cwd = new File(asterixProjectDir.toString() + "/" + CLUSTER_BASE);
-        pb.directory(cwd);
-        pb.redirectErrorStream(true);
-        Process p = pb.start();
-        p.waitFor();
-        return p;
-    }
-
-    private static Process remoteInvoke(String cmd, String node) throws 
Exception {
-        ProcessBuilder pb = new ProcessBuilder("vagrant", "ssh", node, "--", 
cmd);
-        File cwd = new File(asterixProjectDir.toString() + "/" + CLUSTER_BASE);
-        pb.directory(cwd);
-        pb.redirectErrorStream(true);
-        Process p = pb.start();
-        p.waitFor();
-        return p;
-    }
-
-    private static Process managixInvoke(String cmd) throws Exception {
-        return remoteInvoke(MANAGIX_HOME + cmd);
-    }
-
-    private static String executeVagrantScript(String node, String scriptName) 
throws Exception {
-        pb.command("vagrant", "ssh", node, "--", SCRIPT_HOME + scriptName);
-        Process p = pb.start();
-        p.waitFor();
-        InputStream input = p.getInputStream();
-        return IOUtils.toString(input, StandardCharsets.UTF_8.name());
     }
 }
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/data/fbu.adm
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/data/fbu.adm
new file mode 100644
index 0000000..7e99ea4
--- /dev/null
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/data/fbu.adm
@@ -0,0 +1,10 @@
+{"id":1,"alias":"Margarita","name":"MargaritaStoddard","user-since":datetime("2012-08-20T10:10:00"),"friend-ids":{{2,3,6,10}},"employment":[{"organization-name":"Codetechno","start-date":date("2006-08-06")}]}
+{"id":2,"alias":"Isbel","name":"IsbelDull","user-since":datetime("2011-01-22T10:10:00"),"friend-ids":{{1,4}},"employment":[{"organization-name":"Hexviafind","start-date":date("2010-04-27")}]}
+{"id":3,"alias":"Emory","name":"EmoryUnk","user-since":datetime("2012-07-10T10:10:00"),"friend-ids":{{1,5,8,9}},"employment":[{"organization-name":"geomedia","start-date":date("2010-06-17"),"end-date":date("2010-01-26")}]}
+{"id":4,"alias":"Nicholas","name":"NicholasStroh","user-since":datetime("2010-12-27T10:10:00"),"friend-ids":{{2}},"employment":[{"organization-name":"Zamcorporation","start-date":date("2010-06-08")}]}
+{"id":5,"alias":"Von","name":"VonKemble","user-since":datetime("2010-01-05T10:10:00"),"friend-ids":{{3,6,10}},"employment":[{"organization-name":"Kongreen","start-date":date("2010-11-27")}]}
+{"id":6,"alias":"Willis","name":"WillisWynne","user-since":datetime("2005-01-17T10:10:00"),"friend-ids":{{1,3,7}},"employment":[{"organization-name":"jaydax","start-date":date("2009-05-15")}]}
+{"id":7,"alias":"Suzanna","name":"SuzannaTillson","user-since":datetime("2012-08-07T10:10:00"),"friend-ids":{{6}},"employment":[{"organization-name":"Labzatron","start-date":date("2011-04-19")}]}
+{"id":8,"alias":"Nila","name":"NilaMilliron","user-since":datetime("2008-01-01T10:10:00"),"friend-ids":{{3}},"employment":[{"organization-name":"Plexlane","start-date":date("2010-02-28")}]}
+{"id":9,"alias":"Woodrow","name":"WoodrowNehling","user-since":datetime("2005-09-20T10:10:00"),"friend-ids":{{3,10}},"employment":[{"organization-name":"Zuncan","start-date":date("2003-04-22"),"end-date":date("2009-12-13")}]}
+{"id":10,"alias":"Bram","name":"BramHatch","user-since":datetime("2010-10-16T10:10:00"),"friend-ids":{{1,5,9}},"employment":[{"organization-name":"physcane","start-date":date("2007-06-05"),"end-date":date("2011-11-05")}]}
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.10.get.http
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.10.get.http
index 2bd5e45..3faa945 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.10.get.http
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.10.get.http
@@ -27,4 +27,4 @@
  * Expected Result : Success
  * Date            : February 3 2016
  */
-/admin/cluster
+/admin/cluster/summary
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.2.update.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.2.update.aql
index c0b4919..e10896a 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.2.update.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.2.update.aql
@@ -30,7 +30,7 @@
 use dataverse TinySocial;
 
 load dataset FacebookUsers using localfs
-(("path"="asterix_nc1:///vagrant/data/fbu.adm"),
+(("path"="asterix_nc1://../../../../../src/test/resources/integrationts/replication/data/fbu.adm"),
 ("format"="adm"));
 
 insert into dataset TinySocial.FacebookUsersInMemory(for $x in dataset 
TinySocial.FacebookUsers return $x);
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.3.node.aql
similarity index 93%
copy from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
copy to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.3.node.aql
index 9c8cb96..9d5dc9b 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.3.node.aql
@@ -25,7 +25,4 @@
  * Expected Result : Success
  * Date            : January 6 2016
  */
-
-use dataverse TinySocial;
-
-count (for $x in dataset FacebookUsers return $x);
\ No newline at end of file
+kill asterix_nc1
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.3.vscript.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.3.vscript.aql
deleted file mode 100644
index 5eec164..0000000
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.3.vscript.aql
+++ /dev/null
@@ -1 +0,0 @@
-nc1 kill_cc_and_nc.sh
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.5.get.http
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.5.get.http
index 9d19b69..3faa945 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.5.get.http
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.5.get.http
@@ -27,4 +27,4 @@
  * Expected Result : Success
  * Date            : February 3 2016
  */
-admin/cluster
+/admin/cluster/summary
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.8.vmgx.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.8.mgx.aql
similarity index 100%
rename from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.8.vmgx.aql
rename to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.8.mgx.aql
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.9.sleep.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.9.sleep.aql
index 1746da6..d0000aa 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.9.sleep.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failback/node_failback/node_failback.9.sleep.aql
@@ -1 +1 @@
-10000
\ No newline at end of file
+20000
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.2.update.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.2.update.aql
index 94ecc27..90ef6a7 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.2.update.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.2.update.aql
@@ -29,4 +29,5 @@
 use dataverse TinySocial;
 
 load dataset FacebookUsers using localfs
-(("path"="asterix_nc1:///vagrant/data/fbu.adm"),("format"="adm"));
\ No newline at end of file
+(("path"="asterix_nc1://../../../../../src/test/resources/integrationts/replication/data/fbu.adm"),
+("format"="adm"));
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.3.node.aql
similarity index 93%
rename from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
rename to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.3.node.aql
index 9c8cb96..118b7e8 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.3.node.aql
@@ -25,7 +25,4 @@
  * Expected Result : Success
  * Date            : January 6 2016
  */
-
-use dataverse TinySocial;
-
-count (for $x in dataset FacebookUsers return $x);
\ No newline at end of file
+kill asterix_nc2
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.5.sleep.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.4.sleep.aql
similarity index 100%
rename from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.5.sleep.aql
rename to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.4.sleep.aql
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.4.vscript.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.4.vscript.aql
deleted file mode 100644
index 5695ed7..0000000
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.4.vscript.aql
+++ /dev/null
@@ -1 +0,0 @@
-nc2 kill_cc_and_nc.sh
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.3.txnqbc.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.5.query.aql
similarity index 100%
rename from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.3.txnqbc.aql
rename to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.5.query.aql
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.2.update.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.2.update.aql
index d97f786..18ce908 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.2.update.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.2.update.aql
@@ -29,6 +29,6 @@
 use dataverse TinySocial;
 
 load dataset FacebookUsers using localfs
-(("path"="asterix_nc1:///vagrant/data/fbu.adm"),("format"="adm"));
+(("path"="asterix_nc1://../../../../../src/test/resources/integrationts/replication/data/fbu.adm"),("format"="adm"));
 
 insert into dataset TinySocial.FacebookUsersInMemory(for $x in dataset 
TinySocial.FacebookUsers return $x);
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.3.node.aql
similarity index 93%
copy from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
copy to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.3.node.aql
index 9c8cb96..118b7e8 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.3.node.aql
@@ -25,7 +25,4 @@
  * Expected Result : Success
  * Date            : January 6 2016
  */
-
-use dataverse TinySocial;
-
-count (for $x in dataset FacebookUsers return $x);
\ No newline at end of file
+kill asterix_nc2
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.5.sleep.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.4.sleep.aql
similarity index 100%
rename from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.5.sleep.aql
rename to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.4.sleep.aql
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.4.vscript.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.4.vscript.aql
deleted file mode 100644
index 5695ed7..0000000
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.4.vscript.aql
+++ /dev/null
@@ -1 +0,0 @@
-nc2 kill_cc_and_nc.sh
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.3.txnqbc.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.5.query.aql
similarity index 100%
rename from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.3.txnqbc.aql
rename to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.5.query.aql
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.6.txnqar.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.6.txnqar.aql
deleted file mode 100644
index e25e409..0000000
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/mem_component_recovery/mem_component_recovery.6.txnqar.aql
+++ /dev/null
@@ -1,32 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Test case Name  : mem_component_recovery.aql
- * Description     : Check that Memory LSM component are replicated and 
recovered correclty.
-                     The test goes as follows:
-                     start 2 nodes, bulkload a dataset, copy it to in-memory 
dataset, query
-                     data from memory, kill one node and wait until the 
failover complete,
-                     query the data again.
- * Expected Result : Success
- * Date            : January 6 2016
- */
-
-use dataverse TinySocial;
-
-count (for $x in dataset FacebookUsersInMemory return $x);
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.2.node.aql
similarity index 93%
copy from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
copy to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.2.node.aql
index 9c8cb96..9d5dc9b 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/bulkload/bulkload.6.txnqar.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.2.node.aql
@@ -25,7 +25,4 @@
  * Expected Result : Success
  * Date            : January 6 2016
  */
-
-use dataverse TinySocial;
-
-count (for $x in dataset FacebookUsers return $x);
\ No newline at end of file
+kill asterix_nc1
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.2.txnqbc.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.2.txnqbc.aql
deleted file mode 100644
index 76bdcfe..0000000
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.2.txnqbc.aql
+++ /dev/null
@@ -1,30 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements.  See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership.  The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License.  You may obtain a copy of the License at
- *
- *   http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied.  See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/*
- * Test case Name  : metadata_node_recovery.aql
- * Description     : Check that metadata node failover is done correctly.
-                     The test goes as follows:
-                     start 2 nodes, create a dataset, kill metadata node
-                     and wait until the failover complete, verify the
-                     dataset still exists.
- * Expected Result : Success
- * Date            : January 6 2016
- */
-
-for $x in dataset Metadata.Dataset where $x.DatasetName ='FacebookUsers' 
return $x.DatasetName;
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.4.sleep.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.3.sleep.aql
similarity index 100%
rename from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.4.sleep.aql
rename to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.3.sleep.aql
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.3.vscript.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.3.vscript.aql
deleted file mode 100644
index 5eec164..0000000
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.3.vscript.aql
+++ /dev/null
@@ -1 +0,0 @@
-nc1 kill_cc_and_nc.sh
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.5.txnqar.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.4.query.aql
similarity index 92%
rename from 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.5.txnqar.aql
rename to 
asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.4.query.aql
index ac1c593..3d525f6 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.5.txnqar.aql
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/queries/failover/metadata_node/metadata_node.4.query.aql
@@ -29,4 +29,8 @@
 
 use dataverse TinySocial;
 
-for $x in dataset Metadata.Dataset where $x.DatasetName ='FacebookUsers' 
return $x.DatasetName;
\ No newline at end of file
+count(
+for $x in dataset Metadata.Dataset
+where $x.DatasetName ='FacebookUsers'
+return $x.DatasetName
+);
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.10.adm
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.10.adm
index 2c3c039..b155305 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.10.adm
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.10.adm
@@ -1,131 +1,10 @@
 {
-    "cc": {
-        "configUri": "http://10.10.0.2:19002/admin/cluster/cc/config";,
-        "statsUri": "http://10.10.0.2:19002/admin/cluster/cc/stats";,
-        "threadDumpUri": "http://10.10.0.2:19002/admin/cluster/cc/threaddump";
-    },
-    "config": {
-        "api.port": 19002,
-        "cluster.partitions": {
-            "0": "ID:0, Original Node: asterix_nc1, IODevice: 0, Active Node: 
asterix_nc1",
-            "1": "ID:1, Original Node: asterix_nc1, IODevice: 1, Active Node: 
asterix_nc1",
-            "2": "ID:2, Original Node: asterix_nc2, IODevice: 0, Active Node: 
asterix_nc2",
-            "3": "ID:3, Original Node: asterix_nc2, IODevice: 1, Active Node: 
asterix_nc2"
-        },
-        "compiler.framesize": 131072,
-        "compiler.groupmemory": 33554432,
-        "compiler.joinmemory": 33554432,
-        "compiler.pregelix.home": "~/pregelix",
-        "compiler.sortmemory": 33554432,
-        "core.dump.paths": {
-            "asterix_nc1": "/home/vagrant/asterix/logs//asterix_nc1",
-            "asterix_nc2": "/home/vagrant/asterix/logs//asterix_nc2"
-        },
-        "feed.central.manager.port": 4500,
-        "feed.max.threshold.period": 5,
-        "feed.memory.available.wait.timeout": 10,
-        "feed.memory.global.budget": 67108864,
-        "feed.pending.work.threshold": 50,
-        "feed.port": 19003,
-        "instance.name": "asterix",
-        "log.level": "INFO",
-        "max.wait.active.cluster": 60,
-        "metadata.callback.port": 0,
-        "metadata.node": "asterix_nc1",
-        "metadata.partition": "ID:0, Original Node: asterix_nc1, IODevice: 0, 
Active Node: asterix_nc1",
-        "metadata.port": 0,
-        "metadata.registration.timeout.secs": 60,
-        "node.partitions": {
-            "asterix_nc1": [
-                "ID:0, Original Node: asterix_nc1, IODevice: 0, Active Node: 
asterix_nc1",
-                "ID:1, Original Node: asterix_nc1, IODevice: 1, Active Node: 
asterix_nc1"
-            ],
-            "asterix_nc2": [
-                "ID:2, Original Node: asterix_nc2, IODevice: 0, Active Node: 
asterix_nc2",
-                "ID:3, Original Node: asterix_nc2, IODevice: 1, Active Node: 
asterix_nc2"
-            ]
-        },
-        "node.stores": {
-            "asterix_nc1": [
-                "/home/vagrant/asterix/p1/storage",
-                "/home/vagrant/asterix/p2/storage"
-            ],
-            "asterix_nc2": [
-                "/home/vagrant/asterix/p1/storage",
-                "/home/vagrant/asterix/p2/storage"
-            ]
-        },
-        "plot.activate": false,
-        "storage.buffercache.maxopenfiles": 214748364,
-        "storage.buffercache.pagesize": 131072,
-        "storage.buffercache.size": 536870912,
-        "storage.lsm.bloomfilter.falsepositiverate": 0.01,
-        "storage.memorycomponent.globalbudget": 1073741824,
-        "storage.memorycomponent.numcomponents": 2,
-        "storage.memorycomponent.numpages": 256,
-        "storage.memorycomponent.pagesize": 131072,
-        "storage.metadata.memorycomponent.numpages": 64,
-        "transaction.log.dirs": {
-            "asterix_nc1": "/home/vagrant/asterix/tx_logs",
-            "asterix_nc2": "/home/vagrant/asterix/tx_logs"
-        },
-        "txn.commitprofiler.reportinterval": 5,
-        "txn.job.recovery.memorysize": 67108864,
-        "txn.lock.escalationthreshold": 1000,
-        "txn.lock.shrinktimer": 5000,
-        "txn.lock.timeout.sweepthreshold": 10000,
-        "txn.lock.timeout.waitthreshold": 60000,
-        "txn.log.buffer.numpages": 8,
-        "txn.log.buffer.pagesize": 524288,
-        "txn.log.checkpoint.history": 0,
-        "txn.log.checkpoint.lsnthreshold": 67108864,
-        "txn.log.checkpoint.pollfrequency": 120,
-        "txn.log.partitionsize": 2147483648,
-        "web.port": 19001,
-        "web.queryinterface.port": 19006,
-        "web.secondary.port": 19005
-    },
-    "diagnosticsUri": "http://10.10.0.2:19002/admin/diagnostics";,
-    "fullShutdownUri": "http://10.10.0.2:19002/admin/shutdown?all=true";,
     "metadata_node": "asterix_nc1",
-    "ncs": [
-        {
-            "configUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc1/config";,
-            "node_id": "asterix_nc1",
-            "partitions": [
-                {
-                    "active": true,
-                    "partition_id": "partition_0"
-                },
-                {
-                    "active": true,
-                    "partition_id": "partition_1"
-                }
-            ],
-            "state": "ACTIVE",
-            "statsUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc1/stats";,
-            "threadDumpUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc1/threaddump";
-        },
-        {
-            "configUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc2/config";,
-            "node_id": "asterix_nc2",
-            "partitions": [
-                {
-                    "active": true,
-                    "partition_id": "partition_2"
-                },
-                {
-                    "active": true,
-                    "partition_id": "partition_3"
-                }
-            ],
-            "state": "ACTIVE",
-            "statsUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc2/stats";,
-            "threadDumpUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc2/threaddump";
-        }
-    ],
-    "replicationUri": "http://10.10.0.2:19002/admin/cluster/replication";,
-    "shutdownUri": "http://10.10.0.2:19002/admin/shutdown";,
-    "state": "ACTIVE",
-    "versionUri": "http://10.10.0.2:19002/admin/version";
-}
+    "partitions": {
+        "0": "ID:0, Original Node: asterix_nc1, IODevice: 0, Active Node: 
asterix_nc1",
+        "1": "ID:1, Original Node: asterix_nc1, IODevice: 1, Active Node: 
asterix_nc1",
+        "2": "ID:2, Original Node: asterix_nc2, IODevice: 0, Active Node: 
asterix_nc2",
+        "3": "ID:3, Original Node: asterix_nc2, IODevice: 1, Active Node: 
asterix_nc2"
+    },
+    "state": "ACTIVE"
+}
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.5.adm
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.5.adm
index 076c05b..6b31475 100644
--- 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.5.adm
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failback/node_failback/node_failback.cluster_state.5.adm
@@ -1,131 +1,10 @@
 {
-    "cc": {
-        "configUri": "http://10.10.0.2:19002/admin/cluster/cc/config";,
-        "statsUri": "http://10.10.0.2:19002/admin/cluster/cc/stats";,
-        "threadDumpUri": "http://10.10.0.2:19002/admin/cluster/cc/threaddump";
-    },
-    "config": {
-        "api.port": 19002,
-        "cluster.partitions": {
-            "0": "ID:0, Original Node: asterix_nc1, IODevice: 0, Active Node: 
asterix_nc2",
-            "1": "ID:1, Original Node: asterix_nc1, IODevice: 1, Active Node: 
asterix_nc2",
-            "2": "ID:2, Original Node: asterix_nc2, IODevice: 0, Active Node: 
asterix_nc2",
-            "3": "ID:3, Original Node: asterix_nc2, IODevice: 1, Active Node: 
asterix_nc2"
-        },
-        "compiler.framesize": 131072,
-        "compiler.groupmemory": 33554432,
-        "compiler.joinmemory": 33554432,
-        "compiler.pregelix.home": "~/pregelix",
-        "compiler.sortmemory": 33554432,
-        "core.dump.paths": {
-            "asterix_nc1": "/home/vagrant/asterix/logs//asterix_nc1",
-            "asterix_nc2": "/home/vagrant/asterix/logs//asterix_nc2"
-        },
-        "feed.central.manager.port": 4500,
-        "feed.max.threshold.period": 5,
-        "feed.memory.available.wait.timeout": 10,
-        "feed.memory.global.budget": 67108864,
-        "feed.pending.work.threshold": 50,
-        "feed.port": 19003,
-        "instance.name": "asterix",
-        "log.level": "INFO",
-        "max.wait.active.cluster": 60,
-        "metadata.callback.port": 0,
-        "metadata.node": "asterix_nc1",
-        "metadata.partition": "ID:0, Original Node: asterix_nc1, IODevice: 0, 
Active Node: asterix_nc2",
-        "metadata.port": 0,
-        "metadata.registration.timeout.secs": 60,
-        "node.partitions": {
-            "asterix_nc1": [
-                "ID:0, Original Node: asterix_nc1, IODevice: 0, Active Node: 
asterix_nc2",
-                "ID:1, Original Node: asterix_nc1, IODevice: 1, Active Node: 
asterix_nc2"
-            ],
-            "asterix_nc2": [
-                "ID:2, Original Node: asterix_nc2, IODevice: 0, Active Node: 
asterix_nc2",
-                "ID:3, Original Node: asterix_nc2, IODevice: 1, Active Node: 
asterix_nc2"
-            ]
-        },
-        "node.stores": {
-            "asterix_nc1": [
-                "/home/vagrant/asterix/p1/storage",
-                "/home/vagrant/asterix/p2/storage"
-            ],
-            "asterix_nc2": [
-                "/home/vagrant/asterix/p1/storage",
-                "/home/vagrant/asterix/p2/storage"
-            ]
-        },
-        "plot.activate": false,
-        "storage.buffercache.maxopenfiles": 214748364,
-        "storage.buffercache.pagesize": 131072,
-        "storage.buffercache.size": 536870912,
-        "storage.lsm.bloomfilter.falsepositiverate": 0.01,
-        "storage.memorycomponent.globalbudget": 1073741824,
-        "storage.memorycomponent.numcomponents": 2,
-        "storage.memorycomponent.numpages": 256,
-        "storage.memorycomponent.pagesize": 131072,
-        "storage.metadata.memorycomponent.numpages": 64,
-        "transaction.log.dirs": {
-            "asterix_nc1": "/home/vagrant/asterix/tx_logs",
-            "asterix_nc2": "/home/vagrant/asterix/tx_logs"
-        },
-        "txn.commitprofiler.reportinterval": 5,
-        "txn.job.recovery.memorysize": 67108864,
-        "txn.lock.escalationthreshold": 1000,
-        "txn.lock.shrinktimer": 5000,
-        "txn.lock.timeout.sweepthreshold": 10000,
-        "txn.lock.timeout.waitthreshold": 60000,
-        "txn.log.buffer.numpages": 8,
-        "txn.log.buffer.pagesize": 524288,
-        "txn.log.checkpoint.history": 0,
-        "txn.log.checkpoint.lsnthreshold": 67108864,
-        "txn.log.checkpoint.pollfrequency": 120,
-        "txn.log.partitionsize": 2147483648,
-        "web.port": 19001,
-        "web.queryinterface.port": 19006,
-        "web.secondary.port": 19005
-    },
-    "diagnosticsUri": "http://10.10.0.2:19002/admin/diagnostics";,
-    "fullShutdownUri": "http://10.10.0.2:19002/admin/shutdown?all=true";,
     "metadata_node": "asterix_nc2",
-    "ncs": [
-        {
-            "configUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc1/config";,
-            "node_id": "asterix_nc1",
-            "partitions": [
-                {
-                    "active": true,
-                    "partition_id": "partition_0"
-                },
-                {
-                    "active": true,
-                    "partition_id": "partition_1"
-                }
-            ],
-            "state": "FAILED",
-            "statsUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc1/stats";,
-            "threadDumpUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc1/threaddump";
-        },
-        {
-            "configUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc2/config";,
-            "node_id": "asterix_nc2",
-            "partitions": [
-                {
-                    "active": true,
-                    "partition_id": "partition_2"
-                },
-                {
-                    "active": true,
-                    "partition_id": "partition_3"
-                }
-            ],
-            "state": "ACTIVE",
-            "statsUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc2/stats";,
-            "threadDumpUri": 
"http://10.10.0.2:19002/admin/cluster/node/asterix_nc2/threaddump";
-        }
-    ],
-    "replicationUri": "http://10.10.0.2:19002/admin/cluster/replication";,
-    "shutdownUri": "http://10.10.0.2:19002/admin/shutdown";,
-    "state": "ACTIVE",
-    "versionUri": "http://10.10.0.2:19002/admin/version";
-}
+    "partitions": {
+        "0": "ID:0, Original Node: asterix_nc1, IODevice: 0, Active Node: 
asterix_nc2",
+        "1": "ID:1, Original Node: asterix_nc1, IODevice: 1, Active Node: 
asterix_nc2",
+        "2": "ID:2, Original Node: asterix_nc2, IODevice: 0, Active Node: 
asterix_nc2",
+        "3": "ID:3, Original Node: asterix_nc2, IODevice: 1, Active Node: 
asterix_nc2"
+    },
+    "state": "ACTIVE"
+}
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/bulkload/bulkload.5.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/bulkload/bulkload.5.aql
new file mode 100644
index 0000000..9a03714
--- /dev/null
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/bulkload/bulkload.5.aql
@@ -0,0 +1 @@
+10
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/mem_component_recovery/mem_component_recovery.5.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/mem_component_recovery/mem_component_recovery.5.aql
new file mode 100644
index 0000000..9a03714
--- /dev/null
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/mem_component_recovery/mem_component_recovery.5.aql
@@ -0,0 +1 @@
+10
\ No newline at end of file
diff --git 
a/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/metadata_node/metadata_node.4.aql
 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/metadata_node/metadata_node.4.aql
new file mode 100644
index 0000000..56a6051
--- /dev/null
+++ 
b/asterixdb/asterix-installer/src/test/resources/integrationts/replication/results/failover/metadata_node/metadata_node.4.aql
@@ -0,0 +1 @@
+1
\ No newline at end of file
diff --git 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/util/ClusterStateManager.java
 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/util/ClusterStateManager.java
index 376c82a..9f2612f 100644
--- 
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/util/ClusterStateManager.java
+++ 
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/util/ClusterStateManager.java
@@ -662,4 +662,12 @@
         }
         return stateDescription;
     }
+
+    public synchronized JSONObject getClusterStateSummary() throws 
JSONException {
+        JSONObject stateDescription = new JSONObject();
+        stateDescription.put("state", state.name());
+        stateDescription.put("metadata_node", currentMetadataNode);
+        stateDescription.put("partitions", clusterPartitions);
+        return stateDescription;
+    }
 }
diff --git 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
index 6ccc6d0..6646defb 100644
--- 
a/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
+++ 
b/asterixdb/asterix-transactions/src/main/java/org/apache/asterix/transaction/management/resource/PersistentLocalResourceRepository.java
@@ -72,8 +72,8 @@
     private static final String STORAGE_METADATA_FILE_NAME_PREFIX = "." + 
StorageConstants.METADATA_ROOT;
     private static final long STORAGE_LOCAL_RESOURCE_ID = -4321;
     private static final int MAX_CACHED_RESOURCES = 1000;
-    private static final FilenameFilter METADATA_FILES_FILTER = (File dir, 
String name) -> name.equalsIgnoreCase(
-            METADATA_FILE_NAME);
+    private static final FilenameFilter METADATA_FILES_FILTER = (File dir, 
String name) -> name
+            .equalsIgnoreCase(METADATA_FILE_NAME);
     // Finals
     private final IIOManager ioManager;
     private final String[] mountPoints;
@@ -121,9 +121,9 @@
     @Override
     public String toString() {
         StringBuilder aString = new 
StringBuilder().append(PersistentLocalResourceRepository.class.getSimpleName())
-                
.append(Character.LINE_SEPARATOR).append(ioManager.getClass().getSimpleName()).append(':').append(
-                        
Character.LINE_SEPARATOR).append(ioManager.toString()).append(Character.LINE_SEPARATOR).append(
-                                "Cached 
Resources:").append(Character.LINE_SEPARATOR);
+                
.append(Character.LINE_SEPARATOR).append(ioManager.getClass().getSimpleName()).append(':')
+                
.append(Character.LINE_SEPARATOR).append(ioManager.toString()).append(Character.LINE_SEPARATOR)
+                .append("Cached Resources:").append(Character.LINE_SEPARATOR);
         for (Entry<String, LocalResource> pair : 
resourceCache.asMap().entrySet()) {
             
aString.append(pair.getKey()).append("->").append(pair.getValue()).append(Character.LINE_SEPARATOR);
         }
@@ -158,8 +158,7 @@
 
             String storageRootDirPath;
             if (storageRootDirName.startsWith(File.separator)) {
-                storageRootDirPath = mountPoints[i]
-                        + 
storageRootDirName.substring(File.separator.length());
+                storageRootDirPath = mountPoints[i] + 
storageRootDirName.substring(File.separator.length());
             } else {
                 storageRootDirPath = mountPoints[i] + storageRootDirName;
             }
@@ -209,8 +208,7 @@
 
         //if replication enabled, send resource metadata info to remote nodes
         if (isReplicationEnabled && resource.getId() != 
STORAGE_LOCAL_RESOURCE_ID) {
-            String filePath = getFileName(resource.getPath(), 
resource.getId());
-            createReplicationJob(ReplicationOperation.REPLICATE, filePath);
+            createReplicationJob(ReplicationOperation.REPLICATE, resourceFile);
         }
     }
 
@@ -222,9 +220,9 @@
             resourceCache.invalidate(relativePath);
 
             //if replication enabled, delete resource from remote replicas
-            if (isReplicationEnabled && 
!resourceFile.getFile().getName().startsWith(
-                    STORAGE_METADATA_FILE_NAME_PREFIX)) {
-                createReplicationJob(ReplicationOperation.DELETE, 
resourceFile.getAbsolutePath());
+            if (isReplicationEnabled
+                    && 
!resourceFile.getFile().getName().startsWith(STORAGE_METADATA_FILE_NAME_PREFIX))
 {
+                createReplicationJob(ReplicationOperation.DELETE, 
resourceFile);
             }
         } else {
             throw new HyracksDataException("Resource doesn't exist");
@@ -365,19 +363,20 @@
         }
     }
 
-    private void createReplicationJob(ReplicationOperation operation, String 
filePath) throws HyracksDataException {
+    private void createReplicationJob(ReplicationOperation operation, 
FileReference fileRef)
+            throws HyracksDataException {
         /**
          * Durable resources path format:
          * /partition/dataverse/idx/fileName
          * Temporary resources path format:
          * /partition/TEMP_DATASETS_STORAGE_FOLDER/dataverse/idx/fileName
          */
-        String[] fileNameTokens = filePath.split(File.separator);
+        String[] fileNameTokens = 
fileRef.getAbsolutePath().split(File.separator);
         String partitionDir = fileNameTokens[fileNameTokens.length - 4];
         //exclude temporary datasets resources
         if 
(!partitionDir.equals(StoragePathUtil.TEMP_DATASETS_STORAGE_FOLDER)) {
             filesToBeReplicated.clear();
-            filesToBeReplicated.add(filePath);
+            filesToBeReplicated.add(fileRef.getAbsolutePath());
             AsterixReplicationJob job = new 
AsterixReplicationJob(ReplicationJobType.METADATA, operation,
                     ReplicationExecutionType.SYNC, filesToBeReplicated);
             try {
@@ -421,11 +420,9 @@
      * @param ioDeviceId
      * @return A file reference to the storage metadata file.
      */
-    private static FileReference getStorageMetadataFile(IIOManager ioManager, 
String nodeId,
-            int ioDeviceId) {
+    private static FileReference getStorageMetadataFile(IIOManager ioManager, 
String nodeId, int ioDeviceId) {
         String storageMetadataFileName = STORAGE_METADATA_DIRECTORY + 
File.separator + nodeId + "_" + "iodevice"
-                + ioDeviceId + File.separator
-                + STORAGE_METADATA_FILE_NAME_PREFIX;
+                + ioDeviceId + File.separator + 
STORAGE_METADATA_FILE_NAME_PREFIX;
         return new FileReference(ioManager.getIODevices().get(ioDeviceId), 
storageMetadataFileName);
     }
 
@@ -436,8 +433,7 @@
      * @return A file reference to the storage root directory if exists, 
otherwise null.
      * @throws HyracksDataException
      */
-    public static File getStorageRootDirectoryIfExists(IIOManager ioManager, 
String nodeId,
-            int ioDeviceId)
+    public static File getStorageRootDirectoryIfExists(IIOManager ioManager, 
String nodeId, int ioDeviceId)
             throws HyracksDataException {
         File storageRootDir = null;
         FileReference storageMetadataFile = getStorageMetadataFile(ioManager, 
nodeId, ioDeviceId);
diff --git a/asterixdb/pom.xml b/asterixdb/pom.xml
index 8862644..ee5067d 100644
--- a/asterixdb/pom.xml
+++ b/asterixdb/pom.xml
@@ -60,7 +60,6 @@
     <failsafe.test.excludes>**/DmlRecoveryIT.java</failsafe.test.excludes>
     
<cluster.test.excludes>**/AsterixClusterLifeCycleIT.java</cluster.test.excludes>
     
<cluster.extest.excludes>**/ClusterExecutionIT.java</cluster.extest.excludes>
-    
<replication.test.excludes>**/ReplicationIT.java</replication.test.excludes>
 
     <!-- Versions under dependencymanagement or used in many projects via 
properties -->
     <algebricks.version>0.2.18-SNAPSHOT</algebricks.version>
@@ -115,7 +114,6 @@
             <exclude>${failsafe.test.excludes}</exclude>
             <exclude>${cluster.test.excludes}</exclude>
             <exclude>${cluster.extest.excludes}</exclude>
-            <exclude>${replication.test.excludes}</exclude>
           </excludes>
         </configuration>
         <executions>

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/1364
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-MessageType: merged
Gerrit-Change-Id: I542b212e04469e2701690f464f821a5189b97f12
Gerrit-PatchSet: 7
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Owner: Murtadha Hubail <hubail...@gmail.com>
Gerrit-Reviewer: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Gerrit-Reviewer: Murtadha Hubail <hubail...@gmail.com>
Gerrit-Reviewer: abdullah alamoudi <bamou...@gmail.com>