[NO ISSUE][STO] Adapt Storage Structure To Rebalance
- user model changes: no
- storage format changes: no
- interface changes: yes
-- Added IResource#setPath to use for the resource
storage migration.
Details:
- Unify storage structure to support dataset rebalance:
Old format:
./storage/partition_#/dataverse/datasetName_idx_indexName
New format:
./storage/partition_#/dataverse/datasetName/rebalanaceNum/indexName
- Adapt recovery and replication to new storage structure.
- Add old structure -> new structure NC migration task.
- Add CompatibilityUtil to ensure NC can be upgraded during
NC startup.
- Centralize the logic for parsing file path to its components in
ResourceReference/DatasetResourceReference.
- Add storage structure migration test case.
- Add test case for recovery after rebalance.
Change-Id: I0f968b9f493bf5aa2d49f503afe21f0d438bb7f0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/2181
Sonar-Qube: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Contrib: Jenkins <jenk...@fulliautomatix.ics.uci.edu>
Reviewed-by: abdullah alamoudi <bamou...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo
Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/f9e6bae9
Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/f9e6bae9
Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/f9e6bae9
Branch: refs/heads/master
Commit: f9e6bae98de7664cab3054a9e06168323ebd894c
Parents: 98b9d60
Author: Murtadha Hubail <mhub...@apache.org>
Authored: Sat Nov 25 23:25:30 2017 +0300
Committer: Murtadha Hubail <mhub...@apache.org>
Committed: Sun Nov 26 15:45:13 2017 -0800
----------------------------------------------------------------------
.../apache/asterix/app/nc/RecoveryManager.java | 2 +-
.../asterix/app/nc/TransactionSubsystem.java | 5 -
.../nc/task/MigrateStorageResourcesTask.java | 143 +++++++++++++
.../hyracks/bootstrap/NCApplication.java | 8 +-
.../apache/asterix/utils/CompatibilityUtil.java | 66 ++++++
.../http/servlet/ConnectorApiServletTest.java | 2 +-
.../org/apache/asterix/common/TestDataUtil.java | 124 +++++++++++
.../MigrateStorageResourcesTaskTest.java | 104 ++++++++++
.../asterix/test/txn/RecoveryManagerTest.java | 77 ++++---
.../rebalance/all_datasets/all_datasets.11.adm | 2 +-
.../rebalance/all_datasets/all_datasets.12.adm | 2 +-
.../rebalance/all_datasets/all_datasets.5.adm | 2 +-
.../rebalance/all_datasets/all_datasets.6.adm | 2 +-
.../single_dataset/single_dataset.5.adm | 2 +-
.../single_dataset/single_dataset.9.adm | 2 +-
.../single_dataset_with_index.10.adm | 2 +-
.../single_dataset_with_index.5.adm | 2 +-
.../single_dataverse/single_dataverse.11.adm | 2 +-
.../single_dataverse/single_dataverse.12.adm | 2 +-
.../single_dataverse/single_dataverse.5.adm | 2 +-
.../single_dataverse/single_dataverse.6.adm | 2 +-
.../common/dataflow/DatasetLocalResource.java | 5 +
.../replication/IReplicaResourcesManager.java | 5 +-
.../storage/DatasetResourceReference.java | 77 +++++++
.../common/storage/IndexFileProperties.java | 74 -------
.../common/storage/IndexPathElements.java | 44 ++++
.../common/storage/ResourceReference.java | 110 ++++++++++
.../asterix/common/utils/StorageConstants.java | 18 +-
.../asterix/common/utils/StoragePathUtil.java | 23 ++-
asterixdb/asterix-replication/pom.xml | 5 -
.../management/ReplicationChannel.java | 4 +-
.../management/ReplicationManager.java | 4 +-
.../storage/LSMComponentProperties.java | 41 +---
.../storage/LSMIndexFileProperties.java | 46 +----
.../storage/ReplicaResourcesManager.java | 80 +++----
.../PersistentLocalResourceRepository.java | 207 +++++++------------
.../recovery/ReplicationCheckpointManager.java | 2 +-
.../am/btree/dataflow/BTreeResource.java | 7 +-
.../am/lsm/common/dataflow/LsmResource.java | 17 +-
.../am/rtree/dataflow/RTreeResource.java | 6 +-
.../hyracks/storage/common/IResource.java | 7 +
41 files changed, 904 insertions(+), 433 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
index 19966fe..e29e3fe 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/RecoveryManager.java
@@ -443,7 +443,7 @@ public class RecoveryManager implements IRecoveryManager,
ILifeCycleComponent {
return minFirstLSN;
}
- private long getRemoteMinFirstLSN() {
+ private long getRemoteMinFirstLSN() throws HyracksDataException {
IReplicaResourcesManager remoteResourcesManager =
txnSubsystem.getAsterixAppRuntimeContextProvider().getAppContext().getReplicaResourcesManager();
return
remoteResourcesManager.getPartitionsMinLSN(localResourceRepository.getInactivePartitions());
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
index f922832..47f9315 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/TransactionSubsystem.java
@@ -81,11 +81,6 @@ public class TransactionSubsystem implements
ITransactionSubsystem {
checkpointManager = CheckpointManagerFactory.create(this,
checkpointProperties, replicationEnabled);
final Checkpoint latestCheckpoint = checkpointManager.getLatest();
if (latestCheckpoint != null) {
- if (latestCheckpoint.getStorageVersion() !=
StorageConstants.VERSION) {
- throw new IllegalStateException(
- String.format("Storage version mismatch. Current
version (%s). On disk version: (%s)",
- StorageConstants.VERSION,
latestCheckpoint.getStorageVersion()));
- }
transactionManager.ensureMaxTxnId(latestCheckpoint.getMaxTxnId());
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MigrateStorageResourcesTask.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MigrateStorageResourcesTask.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MigrateStorageResourcesTask.java
new file mode 100644
index 0000000..503b8de
--- /dev/null
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/task/MigrateStorageResourcesTask.java
@@ -0,0 +1,143 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.app.nc.task;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.List;
+import java.util.function.Predicate;
+import java.util.logging.Logger;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import org.apache.asterix.common.api.INCLifecycleTask;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.dataflow.DatasetLocalResource;
+import org.apache.asterix.common.storage.DatasetResourceReference;
+import org.apache.asterix.common.transactions.Checkpoint;
+import org.apache.asterix.common.transactions.ICheckpointManager;
+import org.apache.asterix.common.utils.StorageConstants;
+import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.commons.io.FileUtils;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.io.FileReference;
+import org.apache.hyracks.api.io.IIOManager;
+import org.apache.hyracks.api.io.IODeviceHandle;
+import org.apache.hyracks.api.service.IControllerService;
+import org.apache.hyracks.storage.common.ILocalResourceRepository;
+import org.apache.hyracks.storage.common.LocalResource;
+
+/**
+ * Migrates a legacy storage structure to the current one
+ */
+public class MigrateStorageResourcesTask implements INCLifecycleTask {
+
+ private static final Logger LOGGER =
Logger.getLogger(MigrateStorageResourcesTask.class.getName());
+ private static final long serialVersionUID = 1L;
+ public static final int LEGACY_RESOURCES_TREE_DEPTH_FROM_STORAGE_ROOT = 5;
+
+ @Override
+ public void perform(IControllerService cs) throws HyracksDataException {
+ final INcApplicationContext appCtx = (INcApplicationContext)
cs.getApplicationContext();
+ final ICheckpointManager checkpointMgr =
appCtx.getTransactionSubsystem().getCheckpointManager();
+ final Checkpoint latestCheckpoint = checkpointMgr.getLatest();
+ if (latestCheckpoint == null) {
+ // nothing to migrate
+ return;
+ }
+ final IIOManager ioManager = appCtx.getIoManager();
+ final List<IODeviceHandle> ioDevices = ioManager.getIODevices();
+ for (IODeviceHandle ioDeviceHandle : ioDevices) {
+ final Path root =
Paths.get(ioDeviceHandle.getMount().getAbsolutePath());
+ if (!root.toFile().exists()) {
+ continue;
+ }
+ try (Stream<Path> stream = Files.find(root,
LEGACY_RESOURCES_TREE_DEPTH_FROM_STORAGE_ROOT,
+ (path, attr) ->
path.getFileName().toString().equals(StorageConstants.METADATA_FILE_NAME))) {
+ final List<Path> resourceToMigrate =
stream.map(Path::getParent).collect(Collectors.toList());
+ for (Path src : resourceToMigrate) {
+ final Path dest =
+ migrateResourceMetadata(root.relativize(src),
appCtx, latestCheckpoint.getStorageVersion());
+ copyResourceFiles(root.resolve(src), root.resolve(dest),
+
PersistentLocalResourceRepository.INDEX_COMPONENTS);
+ FileUtils.deleteDirectory(root.resolve(src).toFile());
+ }
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+ }
+
+ /**
+ * Migrates the resource metadata file at {@code resourcePath} to the new
storage structure
+ * and updates the migrated resource's metadata to reflect the new path.
+ *
+ * @param resourcePath
+ * @param appCtx
+ * @param resourceVersion
+ * @return The migrated resource relative path
+ * @throws HyracksDataException
+ */
+ private Path migrateResourceMetadata(Path resourcePath,
INcApplicationContext appCtx, int resourceVersion)
+ throws HyracksDataException {
+ final ILocalResourceRepository localResourceRepository =
appCtx.getLocalResourceRepository();
+ final LocalResource srcResource =
localResourceRepository.get(resourcePath.toFile().getPath());
+ final DatasetLocalResource lsmResource = (DatasetLocalResource)
srcResource.getResource();
+ // recreate the resource with the new path and version
+ final DatasetResourceReference lrr =
DatasetResourceReference.of(srcResource, resourceVersion);
+ final Path destPath = lrr.getRelativePath();
+ final FileReference destDir =
appCtx.getIoManager().resolve(destPath.toString());
+ // ensure the new dest dir is empty
+ if (destDir.getFile().exists()) {
+ FileUtils.deleteQuietly(destDir.getFile());
+ }
+ lsmResource.setPath(destPath.toString());
+
+ final LocalResource destResource =
+ new LocalResource(srcResource.getId(),
srcResource.getVersion(), srcResource.isDurable(), lsmResource);
+ LOGGER.info(() -> "Migrating resource from: " + srcResource.getPath()
+ " to " + destResource.getPath());
+ localResourceRepository.insert(destResource);
+ return destPath;
+ }
+
+ /**
+ * Copies the files matching {@code filter} at {@code src} path to {@code
dest}
+ *
+ * @param src
+ * @param dest
+ * @param filter
+ * @throws IOException
+ */
+ private void copyResourceFiles(Path src, Path dest, Predicate<Path>
filter) throws IOException {
+ try (Stream<Path> stream = Files.list(src)) {
+ final List<Path> srcFiles =
stream.filter(filter).collect(Collectors.toList());
+ for (Path srcFile : srcFiles) {
+ Path fileDest = Paths.get(dest.toString(),
srcFile.getFileName().toString());
+ Files.copy(srcFile, fileDest);
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "{ \"class\" : \"" + getClass().getSimpleName() + "\" }";
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
index 63f5bfc..47e5ac9 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/hyracks/bootstrap/NCApplication.java
@@ -37,6 +37,7 @@ import org.apache.asterix.common.config.MetadataProperties;
import org.apache.asterix.common.config.NodeProperties;
import org.apache.asterix.common.config.StorageProperties;
import org.apache.asterix.common.config.TransactionProperties;
+import org.apache.asterix.common.transactions.Checkpoint;
import org.apache.asterix.common.transactions.IRecoveryManager;
import org.apache.asterix.common.transactions.IRecoveryManager.SystemState;
import org.apache.asterix.common.utils.PrintUtil;
@@ -46,6 +47,7 @@ import org.apache.asterix.event.schema.cluster.Node;
import org.apache.asterix.messaging.MessagingChannelInterfaceFactory;
import org.apache.asterix.messaging.NCMessageBroker;
import
org.apache.asterix.transaction.management.resource.PersistentLocalResourceRepository;
+import org.apache.asterix.utils.CompatibilityUtil;
import org.apache.hyracks.api.application.INCServiceContext;
import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.client.NodeStatus;
@@ -54,7 +56,6 @@ import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.io.IFileDeviceResolver;
import org.apache.hyracks.api.job.resource.NodeCapacity;
import org.apache.hyracks.api.messages.IMessageBroker;
-import org.apache.hyracks.api.util.IoUtil;
import org.apache.hyracks.control.common.controllers.NCConfig;
import org.apache.hyracks.control.nc.BaseNCApplication;
import org.apache.hyracks.control.nc.NodeControllerService;
@@ -115,7 +116,10 @@ public class NCApplication extends BaseNCApplication {
MessagingChannelInterfaceFactory interfaceFactory =
new MessagingChannelInterfaceFactory((NCMessageBroker)
messageBroker, messagingProperties);
this.ncServiceCtx.setMessagingChannelInterfaceFactory(interfaceFactory);
-
+ final Checkpoint latestCheckpoint =
runtimeContext.getTransactionSubsystem().getCheckpointManager().getLatest();
+ if (latestCheckpoint != null) {
+ CompatibilityUtil.ensureCompatibility(controllerService,
latestCheckpoint.getStorageVersion());
+ }
IRecoveryManager recoveryMgr =
runtimeContext.getTransactionSubsystem().getRecoveryManager();
final SystemState stateOnStartup = recoveryMgr.getSystemState();
if (stateOnStartup == SystemState.PERMANENT_DATA_LOSS) {
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/CompatibilityUtil.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/CompatibilityUtil.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/CompatibilityUtil.java
new file mode 100644
index 0000000..5d44fc9
--- /dev/null
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/CompatibilityUtil.java
@@ -0,0 +1,66 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.utils;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.logging.Logger;
+
+import org.apache.asterix.app.nc.task.MigrateStorageResourcesTask;
+import org.apache.asterix.common.api.INCLifecycleTask;
+import org.apache.asterix.common.utils.StorageConstants;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.control.nc.NodeControllerService;
+
+public class CompatibilityUtil {
+
+ private static final Logger LOGGER =
Logger.getLogger(CompatibilityUtil.class.getName());
+ private static final int MIN_COMPATIBLE_VERSION = 1;
+
+ private CompatibilityUtil() {
+ }
+
+ public static void ensureCompatibility(NodeControllerService ncs, int
onDiskVerson) throws HyracksDataException {
+ if (onDiskVerson == StorageConstants.VERSION) {
+ return;
+ }
+ ensureUpgradability(onDiskVerson);
+ LOGGER.info(() -> "Upgrading from storage version " + onDiskVerson + "
to " + StorageConstants.VERSION);
+ final List<INCLifecycleTask> upgradeTasks =
getUpgradeTasks(onDiskVerson);
+ for (INCLifecycleTask task : upgradeTasks) {
+ task.perform(ncs);
+ }
+ }
+
+ private static void ensureUpgradability(int onDiskVerson) {
+ if (onDiskVerson < MIN_COMPATIBLE_VERSION) {
+ throw new IllegalStateException(String.format(
+ "Storage cannot be upgraded to new version. Current
version (%s). On disk version: (%s)",
+ StorageConstants.VERSION, onDiskVerson));
+ }
+ }
+
+ private static List<INCLifecycleTask> getUpgradeTasks(int fromVersion) {
+ List<INCLifecycleTask> upgradeTasks = new ArrayList<>();
+ if (fromVersion < StorageConstants.REBALANCE_STORAGE_VERSION) {
+ upgradeTasks.add(new MigrateStorageResourcesTask());
+ }
+ return upgradeTasks;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiServletTest.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiServletTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiServletTest.java
index 33f3a42..87ee2ef 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiServletTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/api/http/servlet/ConnectorApiServletTest.java
@@ -122,7 +122,7 @@ public class ConnectorApiServletTest {
// Checks the correctness of results.
ArrayNode splits = (ArrayNode) actualResponse.get("splits");
String path = (splits.get(0)).get("path").asText();
- Assert.assertTrue(path.endsWith("Metadata/Dataset_idx_Dataset"));
+ Assert.assertTrue(path.endsWith("Metadata/Dataset/0/Dataset"));
}
@Test
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
new file mode 100644
index 0000000..e24ef2d
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/common/TestDataUtil.java
@@ -0,0 +1,124 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common;
+
+import java.io.InputStream;
+import java.util.Arrays;
+import java.util.LinkedHashSet;
+
+import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
+import org.apache.asterix.app.active.ActiveNotificationHandler;
+import org.apache.asterix.common.api.IMetadataLockManager;
+import org.apache.asterix.common.dataflow.ICcApplicationContext;
+import org.apache.asterix.common.utils.Servlets;
+import org.apache.asterix.metadata.declared.MetadataProvider;
+import org.apache.asterix.rebalance.NoOpDatasetRebalanceCallback;
+import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.testframework.context.TestCaseContext;
+import org.apache.asterix.utils.RebalanceUtil;
+import org.junit.Assert;
+
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+
+public class TestDataUtil {
+
+ private static final TestExecutor TEST_EXECUTOR = new TestExecutor();
+ private static final TestCaseContext.OutputFormat OUTPUT_FORMAT =
TestCaseContext.OutputFormat.CLEAN_JSON;
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
+ private TestDataUtil() {
+ }
+
+ /**
+ * Creates dataset with a single field called id as its primary key.
+ *
+ * @param dataset
+ * @throws Exception
+ */
+ public static void createIdOnlyDataset(String dataset) throws Exception {
+ TEST_EXECUTOR.executeSqlppUpdateOrDdl("CREATE TYPE KeyType IF NOT
EXISTS AS { id: int };", OUTPUT_FORMAT);
+ TEST_EXECUTOR.executeSqlppUpdateOrDdl("CREATE DATASET " + dataset +
"(KeyType) PRIMARY KEY id;", OUTPUT_FORMAT);
+ }
+
+ /**
+ * Upserts {@code count} ids into {@code dataset}
+ *
+ * @param dataset
+ * @param count
+ * @throws Exception
+ */
+ public static void upsertData(String dataset, long count) throws Exception
{
+ for (int i = 0; i < count; i++) {
+ TEST_EXECUTOR.executeSqlppUpdateOrDdl("UPSERT INTO " + dataset + "
({\"id\": " + i + "});",
+ TestCaseContext.OutputFormat.CLEAN_JSON);
+ }
+ }
+
+ /**
+ * Gets the number of records in dataset {@code dataset}
+ *
+ * @param datasetName
+ * @return The count
+ * @throws Exception
+ */
+ public static long getDatasetCount(String datasetName) throws Exception {
+ final String query = "SELECT VALUE COUNT(*) FROM `" + datasetName +
"`;";
+ final InputStream responseStream = TEST_EXECUTOR
+ .executeQueryService(query,
TEST_EXECUTOR.getEndpoint(Servlets.QUERY_SERVICE), OUTPUT_FORMAT);
+ final ObjectNode response = OBJECT_MAPPER.readValue(responseStream,
ObjectNode.class);
+ final JsonNode result = response.get("results");
+ // make sure there is a single value in result
+ Assert.assertEquals(1, result.size());
+ return result.get(0).asInt();
+ }
+
+ /**
+ * Rebalances a dataset to {@code targetNodes}
+ *
+ * @param integrationUtil
+ * @param dataverseName
+ * @param datasetName
+ * @param targetNodes
+ * @throws Exception
+ */
+ public static void rebalanceDataset(AsterixHyracksIntegrationUtil
integrationUtil, String dataverseName,
+ String datasetName, String[] targetNodes) throws Exception {
+ ICcApplicationContext ccAppCtx =
+ (ICcApplicationContext)
integrationUtil.getClusterControllerService().getApplicationContext();
+ MetadataProvider metadataProvider = new MetadataProvider(ccAppCtx,
null);
+ try {
+ ActiveNotificationHandler activeNotificationHandler =
+ (ActiveNotificationHandler)
ccAppCtx.getActiveNotificationHandler();
+ activeNotificationHandler.suspend(metadataProvider);
+ try {
+ IMetadataLockManager lockManager =
ccAppCtx.getMetadataLockManager();
+
lockManager.acquireDatasetExclusiveModificationLock(metadataProvider.getLocks(),
+ dataverseName + '.' + datasetName);
+ RebalanceUtil.rebalance(dataverseName, datasetName, new
LinkedHashSet<>(Arrays.asList(targetNodes)),
+ metadataProvider, ccAppCtx.getHcc(),
NoOpDatasetRebalanceCallback.INSTANCE);
+ } finally {
+ activeNotificationHandler.resume(metadataProvider);
+ }
+ } finally {
+ metadataProvider.getLocks().unlock();
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/MigrateStorageResourcesTaskTest.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/MigrateStorageResourcesTaskTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/MigrateStorageResourcesTaskTest.java
new file mode 100644
index 0000000..5b53041
--- /dev/null
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/storage/MigrateStorageResourcesTaskTest.java
@@ -0,0 +1,104 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.storage;
+
+import java.io.BufferedWriter;
+import java.io.File;
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.function.Function;
+
+import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
+import org.apache.asterix.common.TestDataUtil;
+import org.apache.asterix.common.api.INcApplicationContext;
+import org.apache.asterix.common.config.GlobalConfig;
+import org.apache.asterix.common.storage.IndexPathElements;
+import org.apache.asterix.common.transactions.Checkpoint;
+import org.apache.asterix.common.utils.StorageConstants;
+import org.apache.asterix.common.utils.StoragePathUtil;
+import
org.apache.asterix.transaction.management.service.recovery.AbstractCheckpointManager;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class MigrateStorageResourcesTaskTest {
+
+ private static final String DEFAULT_TEST_CONFIG_FILE_NAME =
"asterix-build-configuration.xml";
+ private static final AsterixHyracksIntegrationUtil integrationUtil = new
AsterixHyracksIntegrationUtil();
+
+ @Before
+ public void setUp() throws Exception {
+ System.setProperty(GlobalConfig.CONFIG_FILE_PROPERTY,
DEFAULT_TEST_CONFIG_FILE_NAME);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ integrationUtil.deinit(true);
+ }
+
+ @Test
+ public void storageStructureMigration() throws Exception {
+ Function<IndexPathElements, String> legacyIndexPathProvider =
(pathElements) ->
+ (pathElements.getRebalanceCount().equals("0") ? "" :
pathElements.getRebalanceCount() + File.separator)
+ + pathElements.getDatasetName() +
StoragePathUtil.DATASET_INDEX_NAME_SEPARATOR + pathElements
+ .getIndexName();
+ StoragePathUtil.setIndexPathProvider(legacyIndexPathProvider);
+ integrationUtil.init(true);
+ // create dataset and insert data using legacy structure
+ String datasetName = "ds";
+ TestDataUtil.createIdOnlyDataset(datasetName);
+ TestDataUtil.upsertData(datasetName, 100);
+ final long countBeforeMigration =
TestDataUtil.getDatasetCount(datasetName);
+ // stop NCs
+ integrationUtil.deinit(false);
+ // forge a checkpoint with old version to force migration to new
storage structure on all ncs
+ final INcApplicationContext nc1AppCtx = (INcApplicationContext)
integrationUtil.ncs[0].getApplicationContext();
+ final AbstractCheckpointManager nc1CheckpointManager =
+ (AbstractCheckpointManager)
nc1AppCtx.getTransactionSubsystem().getCheckpointManager();
+ forgeOldVersionCheckpoint(nc1CheckpointManager);
+ final INcApplicationContext nc2AppCtx = (INcApplicationContext)
integrationUtil.ncs[1].getApplicationContext();
+ final AbstractCheckpointManager nc2CheckpointManager =
+ (AbstractCheckpointManager)
nc2AppCtx.getTransactionSubsystem().getCheckpointManager();
+ forgeOldVersionCheckpoint(nc2CheckpointManager);
+
+ // remove the legacy path provider to use the new default structure
+ StoragePathUtil.setIndexPathProvider(null);
+ // start the NCs to do the migration
+ integrationUtil.init(false);
+ final long countAfterMigration =
TestDataUtil.getDatasetCount(datasetName);
+ // ensure data migrated to new structure without issues
+ Assert.assertEquals(countBeforeMigration, countAfterMigration);
+ }
+
+ private void forgeOldVersionCheckpoint(AbstractCheckpointManager manger)
throws HyracksDataException {
+ Checkpoint cp = new Checkpoint(-1, -1, 0, System.currentTimeMillis(),
true,
+ StorageConstants.REBALANCE_STORAGE_VERSION - 1);
+ Path path = manger.getCheckpointPath(cp.getTimeStamp());
+ // Write checkpoint file to disk
+ try (BufferedWriter writer = Files.newBufferedWriter(path)) {
+ writer.write(cp.asJson());
+ writer.flush();
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
index 723786c..29efa47 100644
---
a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
+++
b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/txn/RecoveryManagerTest.java
@@ -19,38 +19,30 @@
package org.apache.asterix.test.txn;
import java.io.File;
-import java.io.InputStream;
-import java.util.Random;
+import java.util.logging.ConsoleHandler;
+import java.util.logging.Level;
+import java.util.logging.Logger;
import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil;
+import org.apache.asterix.common.TestDataUtil;
import org.apache.asterix.common.config.GlobalConfig;
import org.apache.asterix.common.configuration.AsterixConfiguration;
import org.apache.asterix.common.configuration.Property;
-import org.apache.asterix.common.utils.Servlets;
-import org.apache.asterix.test.common.TestExecutor;
+import org.apache.asterix.metadata.bootstrap.MetadataBuiltinEntities;
import org.apache.asterix.test.common.TestHelper;
-import org.apache.asterix.testframework.context.TestCaseContext;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import com.fasterxml.jackson.databind.JsonNode;
-import com.fasterxml.jackson.databind.ObjectMapper;
-import com.fasterxml.jackson.databind.node.ObjectNode;
-
public class RecoveryManagerTest {
- private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
private static final String DEFAULT_TEST_CONFIG_FILE_NAME =
"asterix-build-configuration.xml";
private static final String TEST_CONFIG_FILE_NAME =
"asterix-test-configuration.xml";
private static final String TEST_CONFIG_PATH =
System.getProperty("user.dir") + File.separator + "target" +
File.separator + "config";
private static final String TEST_CONFIG_FILE_PATH = TEST_CONFIG_PATH +
File.separator + TEST_CONFIG_FILE_NAME;
- private static final TestExecutor testExecutor = new TestExecutor();
private static final AsterixHyracksIntegrationUtil integrationUtil = new
AsterixHyracksIntegrationUtil();
- private static final Random random = new Random();
- private static final int numRecords = 1;
@Before
public void setUp() throws Exception {
@@ -74,52 +66,53 @@ public class RecoveryManagerTest {
@Test
public void multiDatasetRecovery() throws Exception {
String datasetNamePrefix = "ds_";
- final TestCaseContext.OutputFormat format =
TestCaseContext.OutputFormat.CLEAN_JSON;
- testExecutor.executeSqlppUpdateOrDdl("CREATE TYPE KeyType AS { id: int
};", format);
int numDatasets = 50;
String datasetName = null;
for (int i = 1; i <= numDatasets; i++) {
datasetName = datasetNamePrefix + i;
- testExecutor.executeSqlppUpdateOrDdl("CREATE DATASET " +
datasetName + "(KeyType) PRIMARY KEY id;", format);
- insertData(datasetName);
+ TestDataUtil.createIdOnlyDataset(datasetName);
+ TestDataUtil.upsertData(datasetName, 10);
}
+ final long countBeforeFirstRecovery =
TestDataUtil.getDatasetCount(datasetName);
// do ungraceful shutdown to enforce recovery
integrationUtil.deinit(false);
integrationUtil.init(false);
- validateRecovery(datasetName);
-
+ final long countAfterFirstRecovery =
TestDataUtil.getDatasetCount(datasetName);
+ Assert.assertEquals(countBeforeFirstRecovery, countAfterFirstRecovery);
// create more datasets after recovery
numDatasets = 100;
for (int i = 51; i <= numDatasets; i++) {
datasetName = datasetNamePrefix + i;
- testExecutor.executeSqlppUpdateOrDdl("CREATE DATASET " +
datasetName + "(KeyType) PRIMARY KEY id;", format);
- insertData(datasetName);
+ TestDataUtil.createIdOnlyDataset(datasetName);
+ TestDataUtil.upsertData(datasetName, 1);
}
+ final long countBeforeSecondRecovery =
TestDataUtil.getDatasetCount(datasetName);
// do ungraceful shutdown to enforce recovery again
integrationUtil.deinit(false);
integrationUtil.init(false);
- validateRecovery(datasetName);
- }
-
- private void insertData(String datasetName) throws Exception {
- for (int i = 0; i < numRecords; i++) {
- testExecutor.executeSqlppUpdateOrDdl("UPSERT INTO " + datasetName
+ " ({\"id\": " + random.nextInt() + "})",
- TestCaseContext.OutputFormat.CLEAN_JSON);
- }
+ final long countAfterSecondRecovery =
TestDataUtil.getDatasetCount(datasetName);
+ Assert.assertEquals(countBeforeSecondRecovery,
countAfterSecondRecovery);
}
- private void validateRecovery(String datasetName) throws Exception {
- final String query = "select value count(*) from `" + datasetName +
"`;";
- final InputStream inputStream = testExecutor
- .executeQueryService(query,
testExecutor.getEndpoint(Servlets.QUERY_SERVICE),
- TestCaseContext.OutputFormat.CLEAN_JSON);
- final ObjectNode jsonNodes = OBJECT_MAPPER.readValue(inputStream,
ObjectNode.class);
- JsonNode result = jsonNodes.get("results");
- // make sure there is result
- Assert.assertEquals(1, result.size());
- for (int i = 0; i < result.size(); i++) {
- JsonNode json = result.get(i);
- Assert.assertEquals(numRecords, json.asInt());
- }
+ @Test
+ public void reoveryAfterRebalance() throws Exception {
+ String datasetName = "ds";
+ TestDataUtil.createIdOnlyDataset(datasetName);
+ TestDataUtil.upsertData(datasetName, 10);
+ final long countBeforeRebalance =
TestDataUtil.getDatasetCount(datasetName);
+ // rebalance dataset to single nc
+ TestDataUtil.rebalanceDataset(integrationUtil,
MetadataBuiltinEntities.DEFAULT_DATAVERSE.getDataverseName(),
+ datasetName, new String[] { "asterix_nc2" });
+ // check data after rebalance
+ final long countAfterRebalance =
TestDataUtil.getDatasetCount(datasetName);
+ Assert.assertEquals(countBeforeRebalance, countAfterRebalance);
+ // insert data after rebalance
+ TestDataUtil.upsertData(datasetName, 20);
+ final long countBeforeRecovery =
TestDataUtil.getDatasetCount(datasetName);
+ // do ungraceful shutdown to enforce recovery
+ integrationUtil.deinit(false);
+ integrationUtil.init(false);
+ final long countAfterRecovery =
TestDataUtil.getDatasetCount(datasetName);
+ Assert.assertEquals(countBeforeRecovery, countAfterRecovery);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.11.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.11.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.11.adm
index f57b230..d970119 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.11.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.11.adm
@@ -1 +1 @@
-{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch1/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch1/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch1/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch1/2/LineItem_
idx_LineItem"}]}
\ No newline at end of file
+{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch1/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch1/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch1/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch1/LineItem/2/LineItem"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.12.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.12.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.12.adm
index 39cad23..38d2eac 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.12.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.12.adm
@@ -1 +1 @@
-{"keys":"o_orderkey","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"OrderType","open":false,"fields":[{"o_orderkey":{"type":"AInt64"}},{"o_custkey":{"type":"AInt64"}},{"o_orderstatus":{"type":"AString"}},{"o_totalprice":{"type":"ADouble"}},{"o_orderdate":{"type":"AString"}},{"o_orderpriority":{"type":"AString"}},{"o_clerk":{"type":"AString"}},{"o_shippriority":{"type":"AInt64"}},{"o_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch2/2/Orders_idx_Orders"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch2/2/Orders_idx_Orders"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch2/2/Orders_idx_Orders"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch2/2/Orders_idx_Orders"}]}
\ No newline at end of file
+{"keys":"o_orderkey","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"OrderType","open":false,"fields":[{"o_orderkey":{"type":"AInt64"}},{"o_custkey":{"type":"AInt64"}},{"o_orderstatus":{"type":"AString"}},{"o_totalprice":{"type":"ADouble"}},{"o_orderdate":{"type":"AString"}},{"o_orderpriority":{"type":"AString"}},{"o_clerk":{"type":"AString"}},{"o_shippriority":{"type":"AInt64"}},{"o_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch2/Orders/2/Orders"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch2/Orders/2/Orders"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch2/Orders/2/Orders"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch2/Orders/2/Orders"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.5.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.5.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.5.adm
index fa5763e..af9759b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.5.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.5.adm
@@ -1 +1 @@
-{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch1/1/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch1/1/LineItem_idx_LineItem"}]}
\ No newline at end of file
+{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch1/LineItem/1/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch1/LineItem/1/LineItem"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.6.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.6.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.6.adm
index b3dea05..0003101 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.6.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/all_datasets/all_datasets.6.adm
@@ -1 +1 @@
-{"keys":"o_orderkey","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"OrderType","open":false,"fields":[{"o_orderkey":{"type":"AInt64"}},{"o_custkey":{"type":"AInt64"}},{"o_orderstatus":{"type":"AString"}},{"o_totalprice":{"type":"ADouble"}},{"o_orderdate":{"type":"AString"}},{"o_orderpriority":{"type":"AString"}},{"o_clerk":{"type":"AString"}},{"o_shippriority":{"type":"AInt64"}},{"o_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch2/1/Orders_idx_Orders"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch2/1/Orders_idx_Orders"}]}
\ No newline at end of file
+{"keys":"o_orderkey","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"OrderType","open":false,"fields":[{"o_orderkey":{"type":"AInt64"}},{"o_custkey":{"type":"AInt64"}},{"o_orderstatus":{"type":"AString"}},{"o_totalprice":{"type":"ADouble"}},{"o_orderdate":{"type":"AString"}},{"o_orderpriority":{"type":"AString"}},{"o_clerk":{"type":"AString"}},{"o_shippriority":{"type":"AInt64"}},{"o_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch2/Orders/1/Orders"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch2/Orders/1/Orders"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.5.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.5.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.5.adm
index 8ff99de..374e1af 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.5.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.5.adm
@@ -1 +1 @@
-{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/1/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/1/LineItem_idx_LineItem"}]}
+{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/LineItem/1/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/LineItem/1/LineItem"}]}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.9.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.9.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.9.adm
index 6eece47..ae6e2fb 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.9.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset/single_dataset.9.adm
@@ -1 +1 @@
-{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/2/LineItem_idx_
LineItem"}]}
\ No newline at end of file
+{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/LineItem/2/LineItem"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm
index 6eece47..ae6e2fb 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.10.adm
@@ -1 +1 @@
-{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/2/LineItem_idx_
LineItem"}]}
\ No newline at end of file
+{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/LineItem/2/LineItem"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm
index 8ff99de..374e1af 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataset_with_index/single_dataset_with_index.5.adm
@@ -1 +1 @@
-{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/1/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/1/LineItem_idx_LineItem"}]}
+{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/LineItem/1/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/LineItem/1/LineItem"}]}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.11.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.11.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.11.adm
index 6eece47..ae6e2fb 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.11.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.11.adm
@@ -1 +1 @@
-{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/2/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/2/LineItem_idx_
LineItem"}]}
\ No newline at end of file
+{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/LineItem/2/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/LineItem/2/LineItem"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.12.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.12.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.12.adm
index 5a549ef..f9cd003 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.12.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.12.adm
@@ -1 +1 @@
-{"keys":"o_orderkey","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"OrderType","open":false,"fields":[{"o_orderkey":{"type":"AInt64"}},{"o_custkey":{"type":"AInt64"}},{"o_orderstatus":{"type":"AString"}},{"o_totalprice":{"type":"ADouble"}},{"o_orderdate":{"type":"AString"}},{"o_orderpriority":{"type":"AString"}},{"o_clerk":{"type":"AString"}},{"o_shippriority":{"type":"AInt64"}},{"o_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/2/Orders_idx_Orders"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/2/Orders_idx_Orders"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/2/Orders_idx_Orders"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/2/Orders_idx_Orders"}]}
\ No newline at end of file
+{"keys":"o_orderkey","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"OrderType","open":false,"fields":[{"o_orderkey":{"type":"AInt64"}},{"o_custkey":{"type":"AInt64"}},{"o_orderstatus":{"type":"AString"}},{"o_totalprice":{"type":"ADouble"}},{"o_orderdate":{"type":"AString"}},{"o_orderpriority":{"type":"AString"}},{"o_clerk":{"type":"AString"}},{"o_shippriority":{"type":"AInt64"}},{"o_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/Orders/2/Orders"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/Orders/2/Orders"},{"ip":"127.0.0.1","path":"storage/partition_2/tpch/Orders/2/Orders"},{"ip":"127.0.0.1","path":"storage/partition_3/tpch/Orders/2/Orders"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.5.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.5.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.5.adm
index 8ff99de..374e1af 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.5.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.5.adm
@@ -1 +1 @@
-{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/1/LineItem_idx_LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/1/LineItem_idx_LineItem"}]}
+{"keys":"l_orderkey,l_linenumber","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"LineItemType","open":false,"fields":[{"l_orderkey":{"type":"AInt64"}},{"l_partkey":{"type":"AInt64"}},{"l_suppkey":{"type":"AInt64"}},{"l_linenumber":{"type":"AInt64"}},{"l_quantity":{"type":"ADouble"}},{"l_extendedprice":{"type":"ADouble"}},{"l_discount":{"type":"ADouble"}},{"l_tax":{"type":"ADouble"}},{"l_returnflag":{"type":"AString"}},{"l_linestatus":{"type":"AString"}},{"l_shipdate":{"type":"AString"}},{"l_commitdate":{"type":"AString"}},{"l_receiptdate":{"type":"AString"}},{"l_shipinstruct":{"type":"AString"}},{"l_shipmode":{"type":"AString"}},{"l_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/LineItem/1/LineItem"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/LineItem/1/LineItem"}]}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.6.adm
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.6.adm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.6.adm
index 6889a70..bca061b 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.6.adm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/rebalance/single_dataverse/single_dataverse.6.adm
@@ -1 +1 @@
-{"keys":"o_orderkey","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"OrderType","open":false,"fields":[{"o_orderkey":{"type":"AInt64"}},{"o_custkey":{"type":"AInt64"}},{"o_orderstatus":{"type":"AString"}},{"o_totalprice":{"type":"ADouble"}},{"o_orderdate":{"type":"AString"}},{"o_orderpriority":{"type":"AString"}},{"o_clerk":{"type":"AString"}},{"o_shippriority":{"type":"AInt64"}},{"o_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/1/Orders_idx_Orders"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/1/Orders_idx_Orders"}]}
\ No newline at end of file
+{"keys":"o_orderkey","type":{"type":"org.apache.asterix.om.types.ARecordType","name":"OrderType","open":false,"fields":[{"o_orderkey":{"type":"AInt64"}},{"o_custkey":{"type":"AInt64"}},{"o_orderstatus":{"type":"AString"}},{"o_totalprice":{"type":"ADouble"}},{"o_orderdate":{"type":"AString"}},{"o_orderpriority":{"type":"AString"}},{"o_clerk":{"type":"AString"}},{"o_shippriority":{"type":"AInt64"}},{"o_comment":{"type":"AString"}}]},"splits":[{"ip":"127.0.0.1","path":"storage/partition_0/tpch/Orders/1/Orders"},{"ip":"127.0.0.1","path":"storage/partition_1/tpch/Orders/1/Orders"}]}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java
index 78b5cb2..48bbf00 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/dataflow/DatasetLocalResource.java
@@ -59,6 +59,11 @@ public class DatasetLocalResource implements IResource {
}
@Override
+ public void setPath(String path) {
+ resource.setPath(path);
+ }
+
+ @Override
public IIndex createInstance(INCServiceContext ncServiceCtx) throws
HyracksDataException {
return resource.createInstance(ncServiceCtx);
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
index 6ffa095..1c3f030 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/replication/IReplicaResourcesManager.java
@@ -20,11 +20,14 @@ package org.apache.asterix.common.replication;
import java.util.Set;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
public interface IReplicaResourcesManager {
/**
* @param partitions
* @return the minimum LSN of all indexes that belong to {@code
partitions}.
+ * @throws HyracksDataException
*/
- public long getPartitionsMinLSN(Set<Integer> partitions);
+ long getPartitionsMinLSN(Set<Integer> partitions) throws
HyracksDataException;
}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetResourceReference.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetResourceReference.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetResourceReference.java
new file mode 100644
index 0000000..d05321e
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/DatasetResourceReference.java
@@ -0,0 +1,77 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.storage;
+
+import java.nio.file.Paths;
+
+import org.apache.asterix.common.dataflow.DatasetLocalResource;
+import org.apache.asterix.common.utils.StorageConstants;
+import org.apache.hyracks.storage.common.LocalResource;
+
+public class DatasetResourceReference extends ResourceReference {
+
+ private int datasetId;
+ private int partitionId;
+
+ private DatasetResourceReference() {
+ super();
+ }
+
+ public static DatasetResourceReference of(LocalResource localResource) {
+ return of(localResource, StorageConstants.VERSION);
+ }
+
+ public static DatasetResourceReference of(LocalResource localResource, int
version) {
+ if (version < StorageConstants.REBALANCE_STORAGE_VERSION) {
+ // to support legacy storage migration
+ return parseLegacyPath(localResource);
+ }
+ return parse(localResource);
+ }
+
+ public int getDatasetId() {
+ return datasetId;
+ }
+
+ public int getPartitionId() {
+ return partitionId;
+ }
+
+ private static DatasetResourceReference parse(LocalResource localResource)
{
+ final DatasetResourceReference datasetResourceReference = new
DatasetResourceReference();
+ final String filePath = Paths.get(localResource.getPath(),
StorageConstants.METADATA_FILE_NAME).toString();
+ parse(datasetResourceReference, filePath);
+ assignIds(localResource, datasetResourceReference);
+ return datasetResourceReference;
+ }
+
+ private static DatasetResourceReference parseLegacyPath(LocalResource
localResource) {
+ final DatasetResourceReference datasetResourceReference = new
DatasetResourceReference();
+ final String filePath = Paths.get(localResource.getPath(),
StorageConstants.METADATA_FILE_NAME).toString();
+ parseLegacyPath(datasetResourceReference, filePath);
+ assignIds(localResource, datasetResourceReference);
+ return datasetResourceReference;
+ }
+
+ private static void assignIds(LocalResource localResource,
DatasetResourceReference lrr) {
+ final DatasetLocalResource dsResource = (DatasetLocalResource)
localResource.getResource();
+ lrr.datasetId = dsResource.getDatasetId();
+ lrr.partitionId = dsResource.getPartition();
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexFileProperties.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexFileProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexFileProperties.java
deleted file mode 100644
index ca6968f..0000000
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexFileProperties.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.common.storage;
-
-import java.io.File;
-
-import org.apache.asterix.common.utils.StoragePathUtil;
-
-/**
- * A holder class for an index file properties.
- */
-public class IndexFileProperties {
-
- private final String fileName;
- private final String idxName;
- private final String dataverseName;
- private final int partitionId;
- private final int datasetId;
-
- public IndexFileProperties(int partitionId, String dataverseName, String
idxName, String fileName, int datasetId) {
- this.partitionId = partitionId;
- this.dataverseName = dataverseName;
- this.idxName = idxName;
- this.fileName = fileName;
- this.datasetId = datasetId;
- }
-
- public String getFileName() {
- return fileName;
- }
-
- public String getIdxName() {
- return idxName;
- }
-
- public String getDataverseName() {
- return dataverseName;
- }
-
- public int getPartitionId() {
- return partitionId;
- }
-
- public int getDatasetId() {
- return datasetId;
- }
-
- @Override
- public String toString() {
- StringBuilder sb = new StringBuilder();
- sb.append(StoragePathUtil.PARTITION_DIR_PREFIX + partitionId +
File.separator);
- sb.append(dataverseName + File.separator);
- sb.append(idxName + File.separator);
- sb.append(fileName);
- sb.append(" [Dataset ID: " + datasetId + "]");
- return sb.toString();
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexPathElements.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexPathElements.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexPathElements.java
new file mode 100644
index 0000000..4d0f3dd
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/IndexPathElements.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.storage;
+
+public class IndexPathElements {
+
+ private final String datasetName;
+ private final String indexName;
+ private final String rebalanceCount;
+
+ public IndexPathElements(String datasetName, String indexName, String
rebalanceCount) {
+ this.datasetName = datasetName;
+ this.indexName = indexName;
+ this.rebalanceCount = rebalanceCount;
+ }
+
+ public String getDatasetName() {
+ return datasetName;
+ }
+
+ public String getIndexName() {
+ return indexName;
+ }
+
+ public String getRebalanceCount() {
+ return rebalanceCount;
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
new file mode 100644
index 0000000..0d65067
--- /dev/null
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/storage/ResourceReference.java
@@ -0,0 +1,110 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.common.storage;
+
+import java.io.File;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+
+import org.apache.asterix.common.utils.StoragePathUtil;
+
+public class ResourceReference {
+
+ protected String root;
+ protected String partition;
+ protected String dataverse;
+ protected String dataset;
+ protected String rebalance;
+ protected String index;
+ protected String name;
+
+ protected ResourceReference() {
+ }
+
+ public static ResourceReference of(String localResourcePath) {
+ ResourceReference lrr = new ResourceReference();
+ parse(lrr, localResourcePath);
+ return lrr;
+ }
+
+ public String getPartition() {
+ return partition;
+ }
+
+ public String getDataverse() {
+ return dataverse;
+ }
+
+ public String getDataset() {
+ return dataset;
+ }
+
+ public String getRebalance() {
+ return rebalance;
+ }
+
+ public String getIndex() {
+ return index;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public Path getRelativePath() {
+ return Paths.get(root, partition, dataverse, dataset, rebalance,
index);
+ }
+
+ protected static void parse(ResourceReference ref, String path) {
+ // format:
root/partition/dataverse/dataset/rebalanceCount/index/fileName
+ final String[] tokens = path.split(File.separator);
+ if (tokens.length < 6) {
+ throw new IllegalStateException("Unrecognized path structure: " +
path);
+ }
+ int offset = tokens.length;
+ ref.name = tokens[--offset];
+ ref.index = tokens[--offset];
+ ref.rebalance = tokens[--offset];
+ ref.dataset = tokens[--offset];
+ ref.dataverse = tokens[--offset];
+ ref.partition = tokens[--offset];
+ ref.root = tokens[--offset];
+ }
+
+ protected static void parseLegacyPath(ResourceReference ref, String path) {
+ // old format:
root/partition/dataverse/datasetName_idx_IndexName/fileName
+ final String[] tokens = path.split(File.separator);
+ if (tokens.length < 4) {
+ throw new IllegalStateException("Unrecognized legacy path
structure: " + path);
+ }
+ int offset = tokens.length;
+ ref.name = tokens[--offset];
+ // split combined dataset/index name
+ final String[] indexTokens =
tokens[--offset].split(StoragePathUtil.DATASET_INDEX_NAME_SEPARATOR);
+ if (indexTokens.length != 2) {
+ throw new IllegalStateException("Unrecognized legacy path
structure: " + path);
+ }
+ ref.dataset = indexTokens[0];
+ ref.index = indexTokens[1];
+ ref.dataverse = tokens[--offset];
+ ref.partition = tokens[--offset];
+ ref.root = tokens[--offset];
+ ref.rebalance = String.valueOf(0);
+ }
+}
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/f9e6bae9/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
----------------------------------------------------------------------
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
index 49d64d6..48769d4 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
@@ -24,13 +24,25 @@ import
org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
* A static class that stores storage constants
*/
public class StorageConstants {
+
public static final String METADATA_ROOT = "root_metadata";
- /** The storage version of AsterixDB related artifacts (e.g. log files,
checkpoint files, etc..). */
- private static final int LOCAL_STORAGE_VERSION = 1;
+ public static final String METADATA_FILE_NAME = ".metadata";
+
+ /**
+ * The storage version of AsterixDB related artifacts (e.g. log files,
checkpoint files, etc..).
+ */
+ private static final int LOCAL_STORAGE_VERSION = 2;
- /** The storage version of AsterixDB stack. */
+ /**
+ * The storage version of AsterixDB stack.
+ */
public static final int VERSION = LOCAL_STORAGE_VERSION +
ITreeIndexFrame.Constants.VERSION;
+ /**
+ * The storage version in which the rebalance storage structure was
introduced
+ */
+ public static final int REBALANCE_STORAGE_VERSION = 8;
+
private StorageConstants() {
}
}
\ No newline at end of file
