This is an automated email from the ASF dual-hosted git repository.
mhubail pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new a69202d649 [ASTERIXDB-3144][CONF] Configurable storage partitions count
a69202d649 is described below
commit a69202d64926facc239a52b9348299910e5f7740
Author: Murtadha Hubail <[email protected]>
AuthorDate: Tue May 9 17:06:45 2023 +0300
[ASTERIXDB-3144][CONF] Configurable storage partitions count
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
- Make number of storage partitions used for static partitioning
configurable.
Change-Id: I1549b408562910c32bbbb6764f366c95af345da9
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17519
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Ali Alsuliman <[email protected]>
---
.../api/cluster_state_1/cluster_state_1.1.regexadm | 1 +
.../cluster_state_1_full/cluster_state_1_full.1.regexadm | 1 +
.../cluster_state_1_less/cluster_state_1_less.1.regexadm | 1 +
.../asterix/common/cluster/IClusterStateManager.java | 6 ++++++
.../common/cluster/StorageComputePartitionsMap.java | 16 ++++++++++------
.../apache/asterix/common/config/StorageProperties.java | 16 +++++++++++++---
.../apache/asterix/common/utils/StorageConstants.java | 1 -
.../asterix/metadata/utils/DataPartitioningProvider.java | 8 ++++----
.../metadata/utils/StaticDataPartitioningProvider.java | 2 +-
.../asterix/runtime/utils/ClusterStateManager.java | 5 +++++
10 files changed, 42 insertions(+), 15 deletions(-)
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
index 801cfb09bc..f28c30311f 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm
@@ -55,6 +55,7 @@
"storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"storage.partitioning" : "dynamic",
+ "storage.partitions.count" : 8,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
"txn\.dataset\.checkpoint\.interval" : 3600,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
index e54bd70ba9..99001d865f 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm
@@ -55,6 +55,7 @@
"storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"storage.partitioning" : "dynamic",
+ "storage.partitions.count" : 8,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
"txn\.dataset\.checkpoint\.interval" : 3600,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
index c5496b26ee..f62aee38c9 100644
---
a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
+++
b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm
@@ -55,6 +55,7 @@
"storage.global.cleanup.timeout" : 600,
"storage.lsm.bloomfilter.falsepositiverate" : 0.01,
"storage.partitioning" : "dynamic",
+ "storage.partitions.count" : 8,
"txn\.commitprofiler\.enabled" : false,
"txn\.commitprofiler\.reportinterval" : 5,
"txn\.dataset\.checkpoint\.interval" : 3600,
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
index 76802d9a1b..3d0fee8e22 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/IClusterStateManager.java
@@ -280,4 +280,10 @@ public interface IClusterStateManager {
* @return true if any of the nodes is currently inactive, otherwise false
*/
boolean nodesFailed(Set<String> nodeIds);
+
+ /**
+ * Gets the count of storage partitions
+ * @return the count of storage partitions
+ */
+ int getStoragePartitionsCount();
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
index 874371e09a..48b2ea1a93 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/cluster/StorageComputePartitionsMap.java
@@ -24,11 +24,14 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.asterix.common.utils.StorageConstants;
-
public class StorageComputePartitionsMap {
private final Map<Integer, ComputePartition> stoToComputeLocation = new
HashMap<>();
+ private final int storagePartitionsCount;
+
+ public StorageComputePartitionsMap(int storagePartitionsCount) {
+ this.storagePartitionsCount = storagePartitionsCount;
+ }
public void addStoragePartition(int stoPart, ComputePartition compute) {
stoToComputeLocation.put(stoPart, compute);
@@ -41,7 +44,7 @@ public class StorageComputePartitionsMap {
computeToStoragePartitions.put(computePartitionIdForMetadata,
Collections.singletonList(computePartitionIdForMetadata));
} else {
- for (int i = 0; i < StorageConstants.NUM_STORAGE_PARTITIONS; i++) {
+ for (int i = 0; i < storagePartitionsCount; i++) {
ComputePartition computePartition = getComputePartition(i);
int computeId = computePartition.getId();
List<Integer> storagePartitions =
@@ -64,13 +67,14 @@ public class StorageComputePartitionsMap {
public static StorageComputePartitionsMap
computePartitionsMap(IClusterStateManager clusterStateManager) {
ClusterPartition metadataPartition =
clusterStateManager.getMetadataPartition();
Map<Integer, ClusterPartition> clusterPartitions =
clusterStateManager.getClusterPartitions();
- StorageComputePartitionsMap newMap = new StorageComputePartitionsMap();
+ final int storagePartitionsCount =
clusterStateManager.getStoragePartitionsCount();
+ StorageComputePartitionsMap newMap = new
StorageComputePartitionsMap(storagePartitionsCount);
newMap.addStoragePartition(metadataPartition.getPartitionId(),
new ComputePartition(metadataPartition.getPartitionId(),
metadataPartition.getActiveNodeId()));
- int storagePartitionsPerComputePartition =
StorageConstants.NUM_STORAGE_PARTITIONS / clusterPartitions.size();
+ int storagePartitionsPerComputePartition = storagePartitionsCount /
clusterPartitions.size();
int storagePartitionId = 0;
int lastComputePartition = 1;
- int remainingStoragePartition =
StorageConstants.NUM_STORAGE_PARTITIONS % clusterPartitions.size();
+ int remainingStoragePartition = storagePartitionsCount %
clusterPartitions.size();
for (Map.Entry<Integer, ClusterPartition> cp :
clusterPartitions.entrySet()) {
ClusterPartition clusterPartition = cp.getValue();
for (int i = 0; i < storagePartitionsPerComputePartition; i++) {
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
index 40bcfb01c7..d61ee14273 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/StorageProperties.java
@@ -20,6 +20,7 @@ package org.apache.asterix.common.config;
import static org.apache.hyracks.control.common.config.OptionTypes.BOOLEAN;
import static org.apache.hyracks.control.common.config.OptionTypes.DOUBLE;
+import static org.apache.hyracks.control.common.config.OptionTypes.INTEGER;
import static
org.apache.hyracks.control.common.config.OptionTypes.INTEGER_BYTE_UNIT;
import static
org.apache.hyracks.control.common.config.OptionTypes.LONG_BYTE_UNIT;
import static
org.apache.hyracks.control.common.config.OptionTypes.NONNEGATIVE_INTEGER;
@@ -66,7 +67,8 @@ public class StorageProperties extends AbstractProperties {
STORAGE_COLUMN_MAX_TUPLE_COUNT(NONNEGATIVE_INTEGER, 15000),
STORAGE_COLUMN_FREE_SPACE_TOLERANCE(DOUBLE, 0.15),
STORAGE_FORMAT(STRING, "row"),
- STORAGE_PARTITIONING(STRING, "dynamic");
+ STORAGE_PARTITIONING(STRING, "dynamic"),
+ STORAGE_PARTITIONS_COUNT(INTEGER, 8);
private final IOptionType interpreter;
private final Object defaultValue;
@@ -84,6 +86,7 @@ public class StorageProperties extends AbstractProperties {
case STORAGE_GLOBAL_CLEANUP:
case STORAGE_GLOBAL_CLEANUP_TIMEOUT:
case STORAGE_PARTITIONING:
+ case STORAGE_PARTITIONS_COUNT:
return Section.COMMON;
default:
return Section.NC;
@@ -143,8 +146,11 @@ public class StorageProperties extends AbstractProperties {
case STORAGE_FORMAT:
return "The default storage format (either row or column)";
case STORAGE_PARTITIONING:
- return "The storage partitioning scheme (either dynamic or
static). This value should not be changed"
- + " after any dataset have been created";
+ return "The storage partitioning scheme (either dynamic or
static). This value should not be"
+ + " changed after any dataset has been created";
+ case STORAGE_PARTITIONS_COUNT:
+ return "The number of storage partitions to use for static
partitioning. This value should not be"
+ + " changed after any dataset has been created";
default:
throw new IllegalStateException("NYI: " + this);
}
@@ -297,4 +303,8 @@ public class StorageProperties extends AbstractProperties {
public PartitioningScheme getPartitioningScheme() {
return
PartitioningScheme.fromName(accessor.getString(Option.STORAGE_PARTITIONING));
}
+
+ public int getStoragePartitionsCount() {
+ return accessor.getInt(Option.STORAGE_PARTITIONS_COUNT);
+ }
}
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
index 5d6322b0bd..1321c96306 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
@@ -48,7 +48,6 @@ public class StorageConstants {
public static final String DEFAULT_FILTERED_DATASET_COMPACTION_POLICY_NAME
= "correlated-prefix";
public static final Map<String, String>
DEFAULT_COMPACTION_POLICY_PROPERTIES;
public static final int METADATA_PARTITION = -1;
- public static final int NUM_STORAGE_PARTITIONS = 8;
/**
* The storage version of AsterixDB related artifacts (e.g. log files,
checkpoint files, etc..).
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
index 63af664a1b..d763430336 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DataPartitioningProvider.java
@@ -30,7 +30,6 @@ import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.metadata.MetadataIndexImmutableProperties;
import org.apache.asterix.common.utils.PartitioningScheme;
-import org.apache.asterix.common.utils.StorageConstants;
import org.apache.asterix.common.utils.StoragePathUtil;
import org.apache.asterix.external.util.FeedUtils;
import org.apache.asterix.metadata.MetadataTransactionContext;
@@ -49,10 +48,12 @@ public abstract class DataPartitioningProvider implements
IDataPartitioningProvi
protected final ICcApplicationContext appCtx;
protected final ClusterStateManager clusterStateManager;
+ protected final int storagePartitionsCounts;
DataPartitioningProvider(ICcApplicationContext appCtx) {
this.appCtx = appCtx;
this.clusterStateManager = (ClusterStateManager)
appCtx.getClusterStateManager();
+ this.storagePartitionsCounts =
clusterStateManager.getStoragePartitionsCount();
}
public static DataPartitioningProvider create(ICcApplicationContext
appCtx) {
@@ -86,10 +87,9 @@ public abstract class DataPartitioningProvider implements
IDataPartitioningProvi
return PartitioningProperties.of(spC.first, spC.second, partitionsMap);
}
- protected static int getNumberOfPartitions(Dataset ds) {
+ protected int getNumberOfPartitions(Dataset ds) {
return
MetadataIndexImmutableProperties.isMetadataDataset(ds.getDatasetId())
- ? MetadataIndexImmutableProperties.METADATA_DATASETS_PARTITIONS
- : StorageConstants.NUM_STORAGE_PARTITIONS;
+ ?
MetadataIndexImmutableProperties.METADATA_DATASETS_PARTITIONS :
storagePartitionsCounts;
}
protected static int getLocationsCount(AlgebricksPartitionConstraint
constraint) {
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
index eaafc6ce57..dbdcdc2d05 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/StaticDataPartitioningProvider.java
@@ -73,7 +73,7 @@ public class StaticDataPartitioningProvider extends
DataPartitioningProvider {
List<String> locations = new ArrayList<>();
Set<Integer> uniqueLocations = new HashSet<>();
StorageComputePartitionsMap partitionMap =
clusterStateManager.getStorageComputeMap();
- for (int i = 0; i < StorageConstants.NUM_STORAGE_PARTITIONS; i++) {
+ for (int i = 0; i < storagePartitionsCounts; i++) {
File f = new File(StoragePathUtil.prepareStoragePartitionPath(i),
StoragePathUtil.prepareDataverseName(dataverseName));
ComputePartition computePartition =
partitionMap.getComputePartition(i);
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
index 984b3ce22e..92ea173fb4 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/utils/ClusterStateManager.java
@@ -503,6 +503,11 @@ public class ClusterStateManager implements
IClusterStateManager {
return nodeIds.stream().anyMatch(failedNodes::contains);
}
+ @Override
+ public int getStoragePartitionsCount() {
+ return appCtx.getStorageProperties().getStoragePartitionsCount();
+ }
+
public synchronized StorageComputePartitionsMap getStorageComputeMap() {
return storageComputePartitionsMap;
}
