>From Ali Alsuliman <[email protected]>: Ali Alsuliman has submitted this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17634 )
Change subject: [ASTERIXDB-3222][COMP] Align compute-to-storage map with constraints ...................................................................... [ASTERIXDB-3222][COMP] Align compute-to-storage map with constraints - user model changes: no - storage format changes: no - interface changes: no Details: When constructing the compute-to-storage map that is used for the PartitioningProperties, the map should be constructed such that it is aligned with the AlgebricksPartitionConstraint Change-Id: I0050538ab25134acacf74775909d83bbe050c6f0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17634 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Murtadha Al Hubail <[email protected]> Tested-by: Murtadha Al Hubail <[email protected]> --- M asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java M asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java 2 files changed, 35 insertions(+), 11 deletions(-) Approvals: Murtadha Al Hubail: Looks good to me, approved; Verified Jenkins: Verified Objections: Anon. E. Moose #1000171: Violations found diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java index a16204d..dc62cac 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java @@ -22,6 +22,7 @@ import java.util.Collections; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; @@ -41,7 +42,7 @@ } public int[][] getComputeToStorageMap(boolean metadataDataset) { - Map<Integer, List<Integer>> computeToStoragePartitions = new HashMap<>(); + Map<Integer, List<Integer>> computeToStoragePartitions = new LinkedHashMap<>(); if (metadataDataset) { final int computePartitionIdForMetadata = 0; computeToStoragePartitions.put(computePartitionIdForMetadata, @@ -55,13 +56,13 @@ storagePartitions.add(i); } } - int[][] computerToStoArray = new int[computeToStoragePartitions.size()][]; + int[][] computeToStoArray = new int[computeToStoragePartitions.size()][]; int partitionIdx = 0; - for (Map.Entry<Integer, List<Integer>> integerListEntry : computeToStoragePartitions.entrySet()) { - computerToStoArray[partitionIdx] = integerListEntry.getValue().stream().mapToInt(i -> i).toArray(); + for (Map.Entry<Integer, List<Integer>> computeToStoPartitions : computeToStoragePartitions.entrySet()) { + computeToStoArray[partitionIdx] = computeToStoPartitions.getValue().stream().mapToInt(i -> i).toArray(); partitionIdx++; } - return computerToStoArray; + return computeToStoArray; } public int getStoragePartitionsCount() { diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java index dbdcdc2..ffcdc57 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java @@ -60,11 +60,11 @@ @Override public PartitioningProperties getPartitioningProperties(MetadataTransactionContext mdTxnCtx, Dataset ds, String indexName) throws AlgebricksException { - SplitComputeLocations dataverseSplits = getDatasetSplits(ds, indexName); + SplitComputeLocations datasetSplits = getDatasetSplits(ds, indexName); StorageComputePartitionsMap partitionMap = clusterStateManager.getStorageComputeMap(); int[][] partitionsMap = partitionMap .getComputeToStorageMap(MetadataIndexImmutableProperties.isMetadataDataset(ds.getDatasetId())); - return PartitioningProperties.of(dataverseSplits.getSplitsProvider(), dataverseSplits.getConstraints(), + return PartitioningProperties.of(datasetSplits.getSplitsProvider(), datasetSplits.getConstraints(), partitionsMap); } @@ -94,19 +94,20 @@ List<String> locations = new ArrayList<>(); Set<Integer> uniqueLocations = new HashSet<>(); StorageComputePartitionsMap partitionMap = clusterStateManager.getStorageComputeMap(); - final int datasetPartitons = getNumberOfPartitions(dataset); + final int datasetPartitions = getNumberOfPartitions(dataset); boolean metadataDataset = MetadataIndexImmutableProperties.isMetadataDataset(dataset.getDatasetId()); - for (int i = 0; i < datasetPartitons; i++) { + for (int i = 0; i < datasetPartitions; i++) { int storagePartition = metadataDataset ? StorageConstants.METADATA_PARTITION : i; final String relPath = StoragePathUtil.prepareDataverseIndexName(dataset.getDataverseName(), dataset.getDatasetName(), indexName, dataset.getRebalanceCount()); File f = new File(StoragePathUtil.prepareStoragePartitionPath(storagePartition), relPath); ComputePartition computePartition = partitionMap.getComputePartition(storagePartition); splits.add(new MappedFileSplit(computePartition.getNodeId(), f.getPath(), 0)); - if (!uniqueLocations.contains(computePartition.getId())) { + int computePartitionId = computePartition.getId(); + if (!uniqueLocations.contains(computePartitionId)) { locations.add(computePartition.getNodeId()); } - uniqueLocations.add(computePartition.getId()); + uniqueLocations.add(computePartitionId); } IFileSplitProvider splitProvider = StoragePathUtil.splitProvider(splits.toArray(new FileSplit[0])); AlgebricksPartitionConstraint constraints = -- To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17634 To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Change-Id: I0050538ab25134acacf74775909d83bbe050c6f0 Gerrit-Change-Number: 17634 Gerrit-PatchSet: 3 Gerrit-Owner: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Ali Alsuliman <[email protected]> Gerrit-Reviewer: Anon. E. Moose #1000171 Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Murtadha Al Hubail <[email protected]> Gerrit-MessageType: merged
