This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new ab99ed9a96 [ASTERIXDB-3222][COMP] Align compute-to-storage map with
constraints
ab99ed9a96 is described below
commit ab99ed9a96a2926e1590b018740f29d74fc43cb6
Author: Ali Alsuliman <[email protected]>
AuthorDate: Sun Jul 9 01:38:33 2023 -0700
[ASTERIXDB-3222][COMP] Align compute-to-storage map with constraints
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
When constructing the compute-to-storage map that is used for the
PartitioningProperties, the map should be constructed such that it is
aligned with the AlgebricksPartitionConstraint
Change-Id: I0050538ab25134acacf74775909d83bbe050c6f0
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17634
Integration-Tests: Jenkins <[email protected]>
Reviewed-by: Murtadha Al Hubail <[email protected]>
Tested-by: Murtadha Al Hubail <[email protected]>
---
.../asterix/common/cluster/StorageComputePartitionsMap.java | 11 ++++++-----
.../metadata/utils/StaticDataPartitioningProvider.java | 13 +++++++------
2 files changed, 13 insertions(+), 11 deletions(-)
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
index a16204da20..dc62cac667 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
@@ -22,6 +22,7 @@ import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
+import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
@@ -41,7 +42,7 @@ public class StorageComputePartitionsMap {
}
public int[][] getComputeToStorageMap(boolean metadataDataset) {
- Map<Integer, List<Integer>> computeToStoragePartitions = new
HashMap<>();
+ Map<Integer, List<Integer>> computeToStoragePartitions = new
LinkedHashMap<>();
if (metadataDataset) {
final int computePartitionIdForMetadata = 0;
computeToStoragePartitions.put(computePartitionIdForMetadata,
@@ -55,13 +56,13 @@ public class StorageComputePartitionsMap {
storagePartitions.add(i);
}
}
- int[][] computerToStoArray = new
int[computeToStoragePartitions.size()][];
+ int[][] computeToStoArray = new
int[computeToStoragePartitions.size()][];
int partitionIdx = 0;
- for (Map.Entry<Integer, List<Integer>> integerListEntry :
computeToStoragePartitions.entrySet()) {
- computerToStoArray[partitionIdx] =
integerListEntry.getValue().stream().mapToInt(i -> i).toArray();
+ for (Map.Entry<Integer, List<Integer>> computeToStoPartitions :
computeToStoragePartitions.entrySet()) {
+ computeToStoArray[partitionIdx] =
computeToStoPartitions.getValue().stream().mapToInt(i -> i).toArray();
partitionIdx++;
}
- return computerToStoArray;
+ return computeToStoArray;
}
public int getStoragePartitionsCount() {
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
index dbdcdc2d05..ffcdc57929 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
@@ -60,11 +60,11 @@ public class StaticDataPartitioningProvider extends
DataPartitioningProvider {
@Override
public PartitioningProperties
getPartitioningProperties(MetadataTransactionContext mdTxnCtx, Dataset ds,
String indexName) throws AlgebricksException {
- SplitComputeLocations dataverseSplits = getDatasetSplits(ds,
indexName);
+ SplitComputeLocations datasetSplits = getDatasetSplits(ds, indexName);
StorageComputePartitionsMap partitionMap =
clusterStateManager.getStorageComputeMap();
int[][] partitionsMap = partitionMap
.getComputeToStorageMap(MetadataIndexImmutableProperties.isMetadataDataset(ds.getDatasetId()));
- return PartitioningProperties.of(dataverseSplits.getSplitsProvider(),
dataverseSplits.getConstraints(),
+ return PartitioningProperties.of(datasetSplits.getSplitsProvider(),
datasetSplits.getConstraints(),
partitionsMap);
}
@@ -94,19 +94,20 @@ public class StaticDataPartitioningProvider extends
DataPartitioningProvider {
List<String> locations = new ArrayList<>();
Set<Integer> uniqueLocations = new HashSet<>();
StorageComputePartitionsMap partitionMap =
clusterStateManager.getStorageComputeMap();
- final int datasetPartitons = getNumberOfPartitions(dataset);
+ final int datasetPartitions = getNumberOfPartitions(dataset);
boolean metadataDataset =
MetadataIndexImmutableProperties.isMetadataDataset(dataset.getDatasetId());
- for (int i = 0; i < datasetPartitons; i++) {
+ for (int i = 0; i < datasetPartitions; i++) {
int storagePartition = metadataDataset ?
StorageConstants.METADATA_PARTITION : i;
final String relPath =
StoragePathUtil.prepareDataverseIndexName(dataset.getDataverseName(),
dataset.getDatasetName(), indexName,
dataset.getRebalanceCount());
File f = new
File(StoragePathUtil.prepareStoragePartitionPath(storagePartition), relPath);
ComputePartition computePartition =
partitionMap.getComputePartition(storagePartition);
splits.add(new MappedFileSplit(computePartition.getNodeId(),
f.getPath(), 0));
- if (!uniqueLocations.contains(computePartition.getId())) {
+ int computePartitionId = computePartition.getId();
+ if (!uniqueLocations.contains(computePartitionId)) {
locations.add(computePartition.getNodeId());
}
- uniqueLocations.add(computePartition.getId());
+ uniqueLocations.add(computePartitionId);
}
IFileSplitProvider splitProvider =
StoragePathUtil.splitProvider(splits.toArray(new FileSplit[0]));
AlgebricksPartitionConstraint constraints =
