Michael Blow has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/3467
Change subject: DO NOT MERGE: combo patch of forced greedy, concurrent merge
policy
......................................................................
DO NOT MERGE: combo patch of forced greedy, concurrent merge policy
Change-Id: I5908f953a18dc98b92bc38cc498b20357f288a72
---
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
M
asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
M
asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
M
asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
5 files changed, 20 insertions(+), 9 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/67/3467/1
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
index 894dc18..a606cd8 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/io/PersistedResourceRegistry.java
@@ -102,6 +102,7 @@
import
org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeLocalResource;
import
org.apache.hyracks.storage.am.lsm.btree.dataflow.ExternalBTreeWithBuddyLocalResource;
import org.apache.hyracks.storage.am.lsm.btree.dataflow.LSMBTreeLocalResource;
+import
org.apache.hyracks.storage.am.lsm.common.impls.ConcurrentMergePolicyFactory;
import
org.apache.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
import
org.apache.hyracks.storage.am.lsm.common.impls.NoOpIOOperationCallbackFactory;
@@ -158,6 +159,7 @@
// ILSMMergePolicyFactory
REGISTERED_CLASSES.put("NoMergePolicyFactory",
NoMergePolicyFactory.class);
REGISTERED_CLASSES.put("PrefixMergePolicyFactory",
PrefixMergePolicyFactory.class);
+ REGISTERED_CLASSES.put("ConcurrentMergePolicyFactory",
ConcurrentMergePolicyFactory.class);
REGISTERED_CLASSES.put("ConstantMergePolicyFactory",
ConstantMergePolicyFactory.class);
REGISTERED_CLASSES.put("CorrelatedPrefixMergePolicyFactory",
CorrelatedPrefixMergePolicyFactory.class);
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
index 8cf114c..988751e 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/nc/NCAppRuntimeContext.java
@@ -92,8 +92,8 @@
import org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationScheduler;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.AsynchronousScheduler;
+import
org.apache.hyracks.storage.am.lsm.common.impls.ConcurrentMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.GreedyScheduler;
-import org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicyFactory;
import org.apache.hyracks.storage.common.ILocalResourceRepository;
import org.apache.hyracks.storage.common.buffercache.BufferCache;
import
org.apache.hyracks.storage.common.buffercache.ClockPageReplacementStrategy;
@@ -196,7 +196,7 @@
throw new IllegalStateException("Unknown storage I/O scheduler: "
+ schedulerName + ", options are "
+ AsynchronousScheduler.FACTORY.getName() + ", " +
GreedyScheduler.FACTORY.getName());
}
- metadataMergePolicyFactory = new PrefixMergePolicyFactory();
+ metadataMergePolicyFactory = new ConcurrentMergePolicyFactory();
indexCheckpointManagerProvider = new
IndexCheckpointManagerProvider(ioManager);
ILocalResourceRepositoryFactory
persistentLocalResourceRepositoryFactory =
new PersistentLocalResourceRepositoryFactory(ioManager,
indexCheckpointManagerProvider,
diff --git
a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 282c03a..028a313 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -3594,6 +3594,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="dml">
+ <compilation-unit name="using-concurrent-merge-policy">
+ <output-dir compare="Text">using-concurrent-merge-policy</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="dml">
<compilation-unit name="using-correlated-prefix-merge-policy">
<output-dir
compare="Text">using-correlated-prefix-merge-policy</output-dir>
</compilation-unit>
diff --git
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
index d2443e1..0207e21 100644
---
a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
+++
b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/utils/StorageConstants.java
@@ -23,6 +23,7 @@
import org.apache.hyracks.storage.am.common.api.ITreeIndexFrame;
import
org.apache.hyracks.storage.am.lsm.common.impls.AbstractLSMIndexFileManager;
+import
org.apache.hyracks.storage.am.lsm.common.impls.ConcurrentMergePolicyFactory;
/**
* A static class that stores storage constants
@@ -41,7 +42,7 @@
public static final String MASK_FILE_PREFIX = ".mask_";
public static final String COMPONENT_MASK_FILE_PREFIX = MASK_FILE_PREFIX +
"C_";
public static final float DEFAULT_TREE_FILL_FACTOR = 1.00f;
- public static final String DEFAULT_COMPACTION_POLICY_NAME = "prefix";
+ public static final String DEFAULT_COMPACTION_POLICY_NAME = "concurrent";
public static final String DEFAULT_FILTERED_DATASET_COMPACTION_POLICY_NAME
= "correlated-prefix";
public static final Map<String, String>
DEFAULT_COMPACTION_POLICY_PROPERTIES;
@@ -57,10 +58,12 @@
static {
DEFAULT_COMPACTION_POLICY_PROPERTIES = new LinkedHashMap<>();
-
DEFAULT_COMPACTION_POLICY_PROPERTIES.put("max-mergable-component-size",
"1073741824"); // 1GB
-
DEFAULT_COMPACTION_POLICY_PROPERTIES.put("max-tolerance-component-count", "5");
// 5 components
+
DEFAULT_COMPACTION_POLICY_PROPERTIES.put(ConcurrentMergePolicyFactory.MAX_COMPONENT_COUNT,
"30");
+
DEFAULT_COMPACTION_POLICY_PROPERTIES.put(ConcurrentMergePolicyFactory.MIN_MERGE_COMPONENT_COUNT,
"3");
+
DEFAULT_COMPACTION_POLICY_PROPERTIES.put(ConcurrentMergePolicyFactory.MAX_MERGE_COMPONENT_COUNT,
"10");
+
DEFAULT_COMPACTION_POLICY_PROPERTIES.put(ConcurrentMergePolicyFactory.SIZE_RATIO,
"1.2");
}
private StorageConstants() {
}
-}
\ No newline at end of file
+}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
index 1ce9002..f0d0efe 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/bootstrap/MetadataBootstrap.java
@@ -82,6 +82,7 @@
import
org.apache.hyracks.storage.am.lsm.common.api.ILSMIOOperationCallbackFactory;
import org.apache.hyracks.storage.am.lsm.common.api.ILSMMergePolicyFactory;
import
org.apache.hyracks.storage.am.lsm.common.api.ILSMOperationTrackerFactory;
+import
org.apache.hyracks.storage.am.lsm.common.impls.ConcurrentMergePolicyFactory;
import
org.apache.hyracks.storage.am.lsm.common.impls.ConstantMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.NoMergePolicyFactory;
import org.apache.hyracks.storage.am.lsm.common.impls.PrefixMergePolicyFactory;
@@ -263,9 +264,9 @@
private static void
insertInitialCompactionPolicies(MetadataTransactionContext mdTxnCtx)
throws AlgebricksException {
- String[] builtInCompactionPolicyClassNames =
- new String[] { ConstantMergePolicyFactory.class.getName(),
PrefixMergePolicyFactory.class.getName(),
- NoMergePolicyFactory.class.getName(),
CorrelatedPrefixMergePolicyFactory.class.getName() };
+ String[] builtInCompactionPolicyClassNames = new String[] {
ConstantMergePolicyFactory.class.getName(),
+ PrefixMergePolicyFactory.class.getName(),
ConcurrentMergePolicyFactory.class.getName(),
+ NoMergePolicyFactory.class.getName(),
CorrelatedPrefixMergePolicyFactory.class.getName() };
for (String policyClassName : builtInCompactionPolicyClassNames) {
CompactionPolicy compactionPolicy =
getCompactionPolicyEntity(policyClassName);
MetadataManager.INSTANCE.addCompactionPolicy(mdTxnCtx,
compactionPolicy);
--
To view, visit https://asterix-gerrit.ics.uci.edu/3467
To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-MessageType: newchange
Gerrit-Change-Id: I5908f953a18dc98b92bc38cc498b20357f288a72
Gerrit-Change-Number: 3467
Gerrit-PatchSet: 1
Gerrit-Owner: Michael Blow <[email protected]>
