This is an automated email from the ASF dual-hosted git repository. mhubail pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push: new a69202d649 [ASTERIXDB-3144][CONF] Configurable storage partitions count a69202d649 is described below commit a69202d64926facc239a52b9348299910e5f7740 Author: Murtadha Hubail <mhub...@apache.org> AuthorDate: Tue May 9 17:06:45 2023 +0300 [ASTERIXDB-3144][CONF] Configurable storage partitions count - user model changes: no - storage format changes: no - interface changes: yes Details: - Make number of storage partitions used for static partitioning configurable. Change-Id: I1549b408562910c32bbbb6764f366c95af345da9 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17519 Integration-Tests: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Tested-by: Jenkins <jenk...@fulliautomatix.ics.uci.edu> Reviewed-by: Ali Alsuliman <ali.al.solai...@gmail.com> --- .../api/cluster_state_1/cluster_state_1.1.regexadm | 1 + .../cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 + .../cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 + .../asterix/common/cluster/IClusterStateManager.java | 6 ++++++ .../common/cluster/StorageComputePartitionsMap.java | 16 ++++++++++------ .../apache/asterix/common/config/StorageProperties.java | 16 +++++++++++++--- .../apache/asterix/common/utils/StorageConstants.java | 1 - .../asterix/metadata/utils/DataPartitioningProvider.java | 8 ++++---- .../metadata/utils/StaticDataPartitioningProvider.java | 2 +- .../asterix/runtime/utils/ClusterStateManager.java | 5 +++++ 10 files changed, 42 insertions(+), 15 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 801cfb09bc..f28c30311f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -55,6 +55,7 @@ "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", + "storage.partitions.count" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.dataset\.checkpoint\.interval" : 3600, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index e54bd70ba9..99001d865f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -55,6 +55,7 @@ "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", + "storage.partitions.count" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.dataset\.checkpoint\.interval" : 3600, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index c5496b26ee..f62aee38c9 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -55,6 +55,7 @@ "storage.global.cleanup.timeout" : 600, "storage.lsm.bloomfilter.falsepositiverate" : 0.01, "storage.partitioning" : "dynamic", + "storage.partitions.count" : 8, "txn\.commitprofiler\.enabled" : false, "txn\.commitprofiler\.reportinterval" : 5, "txn\.dataset\.checkpoint\.interval" : 3600, diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java index 76802d9a1b..3d0fee8e22 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java @@ -280,4 +280,10 @@ public interface IClusterStateManager { * @return true if any of the nodes is currently inactive, otherwise false */ boolean nodesFailed(Set<String> nodeIds); + + /** + * Gets the count of storage partitions + * @return the count of storage partitions + */ + int getStoragePartitionsCount(); } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java index 874371e09a..48b2ea1a93 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java @@ -24,11 +24,14 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import org.apache.asterix.common.utils.StorageConstants; - public class StorageComputePartitionsMap { private final Map<Integer, ComputePartition> stoToComputeLocation = new HashMap<>(); + private final int storagePartitionsCount; + + public StorageComputePartitionsMap(int storagePartitionsCount) { + this.storagePartitionsCount = storagePartitionsCount; + } public void addStoragePartition(int stoPart, ComputePartition compute) { stoToComputeLocation.put(stoPart, compute); @@ -41,7 +44,7 @@ public class StorageComputePartitionsMap { computeToStoragePartitions.put(computePartitionIdForMetadata, Collections.singletonList(computePartitionIdForMetadata)); } else { - for (int i = 0; i < StorageConstants.NUM_STORAGE_PARTITIONS; i++) { + for (int i = 0; i < storagePartitionsCount; i++) { ComputePartition computePartition = getComputePartition(i); int computeId = computePartition.getId(); List<Integer> storagePartitions = @@ -64,13 +67,14 @@ public class StorageComputePartitionsMap { public static StorageComputePartitionsMap computePartitionsMap(IClusterStateManager clusterStateManager) { ClusterPartition metadataPartition = clusterStateManager.getMetadataPartition(); Map<Integer, ClusterPartition> clusterPartitions = clusterStateManager.getClusterPartitions(); - StorageComputePartitionsMap newMap = new StorageComputePartitionsMap(); + final int storagePartitionsCount = clusterStateManager.getStoragePartitionsCount(); + StorageComputePartitionsMap newMap = new StorageComputePartitionsMap(storagePartitionsCount); newMap.addStoragePartition(metadataPartition.getPartitionId(), new ComputePartition(metadataPartition.getPartitionId(), metadataPartition.getActiveNodeId())); - int storagePartitionsPerComputePartition = StorageConstants.NUM_STORAGE_PARTITIONS / clusterPartitions.size(); + int storagePartitionsPerComputePartition = storagePartitionsCount / clusterPartitions.size(); int storagePartitionId = 0; int lastComputePartition = 1; - int remainingStoragePartition = StorageConstants.NUM_STORAGE_PARTITIONS % clusterPartitions.size(); + int remainingStoragePartition = storagePartitionsCount % clusterPartitions.size(); for (Map.Entry<Integer, ClusterPartition> cp : clusterPartitions.entrySet()) { ClusterPartition clusterPartition = cp.getValue(); for (int i = 0; i < storagePartitionsPerComputePartition; i++) { diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java index 40bcfb01c7..d61ee14273 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java @@ -20,6 +20,7 @@ package org.apache.asterix.common.config; import static org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN; import static org.apache.hyracks.control.common.config.OptionTypes.DOUBLE; +import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER; import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT; import static org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT; import static org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER; @@ -66,7 +67,8 @@ public class StorageProperties extends AbstractProperties { STORAGE_COLUMN_MAX_TUPLE_COUNT(NONNEGATIVE_INTEGER, 15000), STORAGE_COLUMN_FREE_SPACE_TOLERANCE(DOUBLE, 0.15), STORAGE_FORMAT(STRING, "row"), - STORAGE_PARTITIONING(STRING, "dynamic"); + STORAGE_PARTITIONING(STRING, "dynamic"), + STORAGE_PARTITIONS_COUNT(INTEGER, 8); private final IOptionType interpreter; private final Object defaultValue; @@ -84,6 +86,7 @@ public class StorageProperties extends AbstractProperties { case STORAGE_GLOBAL_CLEANUP: case STORAGE_GLOBAL_CLEANUP_TIMEOUT: case STORAGE_PARTITIONING: + case STORAGE_PARTITIONS_COUNT: return Section.COMMON; default: return Section.NC; @@ -143,8 +146,11 @@ public class StorageProperties extends AbstractProperties { case STORAGE_FORMAT: return "The default storage format (either row or column)"; case STORAGE_PARTITIONING: - return "The storage partitioning scheme (either dynamic or static). This value should not be changed" - + " after any dataset have been created"; + return "The storage partitioning scheme (either dynamic or static). This value should not be" + + " changed after any dataset has been created"; + case STORAGE_PARTITIONS_COUNT: + return "The number of storage partitions to use for static partitioning. This value should not be" + + " changed after any dataset has been created"; default: throw new IllegalStateException("NYI: " + this); } @@ -297,4 +303,8 @@ public class StorageProperties extends AbstractProperties { public PartitioningScheme getPartitioningScheme() { return PartitioningScheme.fromName(accessor.getString(Option.STORAGE_PARTITIONING)); } + + public int getStoragePartitionsCount() { + return accessor.getInt(Option.STORAGE_PARTITIONS_COUNT); + } } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java index 5d6322b0bd..1321c96306 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java @@ -48,7 +48,6 @@ public class StorageConstants { public static final String DEFAULT_FILTERED_DATASET_COMPACTION_POLICY_NAME = "correlated-prefix"; public static final Map<String, String> DEFAULT_COMPACTION_POLICY_PROPERTIES; public static final int METADATA_PARTITION = -1; - public static final int NUM_STORAGE_PARTITIONS = 8; /** * The storage version of AsterixDB related artifacts (e.g. log files, checkpoint files, etc..). diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java index 63af664a1b..d763430336 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java @@ -30,7 +30,6 @@ import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.common.metadata.DataverseName; import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties; import org.apache.asterix.common.utils.PartitioningScheme; -import org.apache.asterix.common.utils.StorageConstants; import org.apache.asterix.common.utils.StoragePathUtil; import org.apache.asterix.external.util.FeedUtils; import org.apache.asterix.metadata.MetadataTransactionContext; @@ -49,10 +48,12 @@ public abstract class DataPartitioningProvider implements IDataPartitioningProvi protected final ICcApplicationContext appCtx; protected final ClusterStateManager clusterStateManager; + protected final int storagePartitionsCounts; DataPartitioningProvider(ICcApplicationContext appCtx) { this.appCtx = appCtx; this.clusterStateManager = (ClusterStateManager) appCtx.getClusterStateManager(); + this.storagePartitionsCounts = clusterStateManager.getStoragePartitionsCount(); } public static DataPartitioningProvider create(ICcApplicationContext appCtx) { @@ -86,10 +87,9 @@ public abstract class DataPartitioningProvider implements IDataPartitioningProvi return PartitioningProperties.of(spC.first, spC.second, partitionsMap); } - protected static int getNumberOfPartitions(Dataset ds) { + protected int getNumberOfPartitions(Dataset ds) { return MetadataIndexImmutableProperties.isMetadataDataset(ds.getDatasetId()) - ? MetadataIndexImmutableProperties.METADATA_DATASETS_PARTITIONS - : StorageConstants.NUM_STORAGE_PARTITIONS; + ? MetadataIndexImmutableProperties.METADATA_DATASETS_PARTITIONS : storagePartitionsCounts; } protected static int getLocationsCount(AlgebricksPartitionConstraint constraint) { diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java index eaafc6ce57..dbdcdc2d05 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java @@ -73,7 +73,7 @@ public class StaticDataPartitioningProvider extends DataPartitioningProvider { List<String> locations = new ArrayList<>(); Set<Integer> uniqueLocations = new HashSet<>(); StorageComputePartitionsMap partitionMap = clusterStateManager.getStorageComputeMap(); - for (int i = 0; i < StorageConstants.NUM_STORAGE_PARTITIONS; i++) { + for (int i = 0; i < storagePartitionsCounts; i++) { File f = new File(StoragePathUtil.prepareStoragePartitionPath(i), StoragePathUtil.prepareDataverseName(dataverseName)); ComputePartition computePartition = partitionMap.getComputePartition(i); diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java index 984b3ce22e..92ea173fb4 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java @@ -503,6 +503,11 @@ public class ClusterStateManager implements IClusterStateManager { return nodeIds.stream().anyMatch(failedNodes::contains); } + @Override + public int getStoragePartitionsCount() { + return appCtx.getStorageProperties().getStoragePartitionsCount(); + } + public synchronized StorageComputePartitionsMap getStorageComputeMap() { return storageComputePartitionsMap; }