This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit be592d91555389a389827bcfb5a243d0c3c4c804 Author: Ali Alsuliman <[email protected]> AuthorDate: Sun Feb 7 11:11:40 2021 -0800 [NO ISSUE][COMP] Add compiler property to skip assiging low budget - user model changes: no - storage format changes: no - interface changes: no Details: Add a compiler property to allow users to instruct the compiler not to assign low budget for queries such metadata queries. Change-Id: Ibba711d76dbdc73c1072c999c05c43dbce30debc Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/9848 Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> --- .../rules/SetAsterixMemoryRequirementsRule.java | 3 ++ .../apache/asterix/api/common/APIFramework.java | 3 +- .../apache/asterix/test/common/TestExecutor.java | 32 +++++++++++++++++++++- .../metadata_only_01.2.plans.sqlpp | 31 +++++++++++---------- .../api/cluster_state_1/cluster_state_1.1.regexadm | 1 + .../cluster_state_1_full.1.regexadm | 1 + .../cluster_state_1_less.1.regexadm | 1 + .../misc/metadata_only_01/metadata_only_01.2.regex | 1 + .../asterix/common/config/CompilerProperties.java | 12 +++++++- .../common/config/OptimizationConfUtil.java | 3 ++ .../algebricks/core/config/AlgebricksConfig.java | 1 + .../rewriter/base/PhysicalOptimizationConfig.java | 11 +++++++- 12 files changed, 82 insertions(+), 18 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java index abe4b71..8de1fb5 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java @@ -52,6 +52,9 @@ public final class SetAsterixMemoryRequirementsRule extends SetMemoryRequirement } private boolean forceMinMemoryBudget(AsterixOptimizationContext context) { + if (!context.getPhysicalOptimizationConfig().getMinMemoryAllocation()) { + return false; + } Int2ObjectMap<Set<DataSource>> dataSourceMap = context.getDataSourceMap(); if (dataSourceMap.isEmpty()) { return false; diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index baf083d..9088db6 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -135,7 +135,8 @@ public class APIFramework { CompilerProperties.COMPILER_SORT_PARALLEL_KEY, CompilerProperties.COMPILER_SORT_SAMPLES_KEY, CompilerProperties.COMPILER_INDEXONLY_KEY, CompilerProperties.COMPILER_INTERNAL_SANITYCHECK_KEY, CompilerProperties.COMPILER_EXTERNAL_FIELD_PUSHDOWN_KEY, CompilerProperties.COMPILER_SUBPLAN_MERGE_KEY, - CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, + CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, + CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, FunctionUtil.IMPORT_PRIVATE_FUNCTIONS, FuzzyUtils.SIM_FUNCTION_PROP_NAME, FuzzyUtils.SIM_THRESHOLD_PROP_NAME, StartFeedStatement.WAIT_FOR_COMPLETION, FeedActivityDetails.FEED_POLICY_NAME, FeedActivityDetails.COLLECT_LOCATIONS, SqlppQueryRewriter.INLINE_WITH_OPTION, diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java index e202ddf..44038cc 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java @@ -42,6 +42,8 @@ import java.net.URI; import java.net.URISyntaxException; import java.nio.CharBuffer; import java.nio.charset.Charset; +import java.nio.file.Files; +import java.nio.file.Paths; import java.text.MessageFormat; import java.util.ArrayDeque; import java.util.ArrayList; @@ -161,6 +163,7 @@ public class TestExecutor { private static final Pattern HANDLE_VARIABLE_PATTERN = Pattern.compile("handlevariable=(\\w+)"); private static final Pattern RESULT_VARIABLE_PATTERN = Pattern.compile("resultvariable=(\\w+)"); private static final Pattern COMPARE_UNORDERED_ARRAY_PATTERN = Pattern.compile("compareunorderedarray=(\\w+)"); + private static final Pattern BODY_REF_PATTERN = Pattern.compile("bodyref=(.*)", Pattern.MULTILINE); private static final Pattern MACRO_PARAM_PATTERN = Pattern.compile("macro (?<name>[\\w-$]+)=(?<value>.*)", Pattern.MULTILINE); @@ -1329,12 +1332,16 @@ public class TestExecutor { final String trimmedPathAndQuery = stripAllComments(statement).trim(); final String variablesReplaced = replaceVarRef(trimmedPathAndQuery, variableCtx); final List<Parameter> params = extractParameters(statement); - final Optional<String> body = extractBody(statement); + Optional<String> body = extractBody(statement); final Predicate<Integer> statusCodePredicate = extractStatusCodePredicate(statement); final boolean extracResult = isExtracResult(statement); final boolean extractStatus = isExtractStatus(statement); final String mimeReqType = extractHttpRequestType(statement); + final String saveResponseVar = getResultVariable(statement); ContentType contentType = mimeReqType != null ? ContentType.create(mimeReqType, UTF_8) : TEXT_PLAIN_UTF8; + if (!body.isPresent()) { + body = getBodyFromReference(statement, variableCtx); + } InputStream resultStream; if ("http".equals(extension)) { resultStream = executeHttp(reqType, variablesReplaced, fmt, params, statusCodePredicate, body, contentType); @@ -1356,6 +1363,8 @@ public class TestExecutor { } else { throw new Exception("no handle for test " + testFile.toString()); } + } else if (saveResponseVar != null) { + variableCtx.put(saveResponseVar, IOUtils.toString(resultStream, UTF_8)); } else { if (expectedResultFile == null) { if (testFile.getName().startsWith(DIAGNOSE)) { @@ -1689,6 +1698,27 @@ public class TestExecutor { return resultVariableMatcher.find() ? resultVariableMatcher.group(1) : null; } + protected static Optional<String> getBodyFromReference(String statement, Map<String, Object> varMap) + throws IOException { + Optional<String> bodyRef = findPattern(statement, BODY_REF_PATTERN); + if (!bodyRef.isPresent()) { + return Optional.empty(); + } + String bodyReference = bodyRef.get(); + String body = (String) varMap.get(bodyReference); + if (body != null) { + return Optional.of(body); + } + try (Stream<String> stream = Files.lines(Paths.get(bodyReference), UTF_8)) { + return Optional.of(stream.collect(Collectors.joining())); + } + } + + protected static Optional<String> findPattern(String statement, Pattern pattern) { + final Matcher matcher = pattern.matcher(statement); + return matcher.find() ? Optional.of(matcher.group(1)) : Optional.empty(); + } + protected static boolean getCompareUnorderedArray(String statement) { final Matcher matcher = COMPARE_UNORDERED_ARRAY_PATTERN.matcher(statement); return matcher.find() && Boolean.parseBoolean(matcher.group(1)); diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp similarity index 50% copy from hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp index 2ed40fd..0b6fdbf 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/metadata_only_01/metadata_only_01.2.plans.sqlpp @@ -16,20 +16,23 @@ * specific language governing permissions and limitations * under the License. */ -package org.apache.hyracks.algebricks.core.config; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; +/* Test that a metadata-only query gets a normal memory budget instead of the compiler-allocated memory */ -public class AlgebricksConfig { +-- param job:string=true - public static final String ALGEBRICKS_LOGGER_NAME = "org.apache.hyracks.algebricks"; - public static final Logger ALGEBRICKS_LOGGER = LogManager.getLogger(ALGEBRICKS_LOGGER_NAME); - public static final int SORT_SAMPLES_DEFAULT = 100; - public static final boolean SORT_PARALLEL_DEFAULT = true; - public static final boolean INDEX_ONLY_DEFAULT = true; - public static final boolean SANITYCHECK_DEFAULT = false; - public static final boolean EXTERNAL_FIELD_PUSHDOWN_DEFAULT = false; - public static final boolean SUBPLAN_MERGE_DEFAULT = true; - public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true; -} +SET `compiler.parallelism` "1"; +SET `compiler.min.memory.allocation` "false"; +SET `compiler.sortmemory` "320KB"; +SET `compiler.groupmemory` "160KB"; +SET `compiler.joinmemory` "256KB"; +SET `compiler.textsearchmemory` "160KB"; + +from Metadata.`Dataset` ds +join Metadata.`Index` idx +on ds.DataverseName = idx.DataverseName and ds.DatasetName = idx.DatasetName +where contains(ds.DataverseName, "Metadata") + and ds.DatasetName in ["Dataverse", "Dataset", "Index"] + and idx.IsPrimary +group by ds.DataverseName +select value count(*); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm index 378fc83..b6a49a6 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1/cluster_state_1.1.regexadm @@ -14,6 +14,7 @@ "compiler\.indexonly" : true, "compiler\.internal\.sanitycheck" : true, "compiler\.joinmemory" : 262144, + "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : 0, "compiler\.sort\.parallel" : false, "compiler\.sort\.samples" : 100, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm index 5ff1b7a..dedb40f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_full/cluster_state_1_full.1.regexadm @@ -14,6 +14,7 @@ "compiler\.indexonly" : true, "compiler\.internal\.sanitycheck" : false, "compiler\.joinmemory" : 262144, + "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : -1, "compiler\.sort\.parallel" : true, "compiler\.sort\.samples" : 100, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm index 2e76d04..985d3bd 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/api/cluster_state_1_less/cluster_state_1_less.1.regexadm @@ -14,6 +14,7 @@ "compiler\.indexonly" : true, "compiler\.internal\.sanitycheck" : false, "compiler\.joinmemory" : 262144, + "compiler\.min\.memory\.allocation" : true, "compiler\.parallelism" : 3, "compiler\.sort\.parallel" : true, "compiler\.sort\.samples" : 100, diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex new file mode 100644 index 0000000..6af6770 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex @@ -0,0 +1 @@ +/memory\D+1146880/ \ No newline at end of file diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java index 9340110..39142e5 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/CompilerProperties.java @@ -91,7 +91,11 @@ public class CompilerProperties extends AbstractProperties { COMPILER_SUBPLAN_NESTEDPUSHDOWN( BOOLEAN, AlgebricksConfig.SUBPLAN_NESTEDPUSHDOWN_DEFAULT, - "When merging subplans into groupby/suplan allow nesting of subplans"); + "When merging subplans into groupby/suplan allow nesting of subplans"), + COMPILER_MIN_MEMORY_ALLOCATION( + BOOLEAN, + AlgebricksConfig.MIN_MEMORY_ALLOCATION_DEFAULT, + "Enable/disable allocating minimum budget for certain queries"); private final IOptionType type; private final Object defaultValue; @@ -150,6 +154,8 @@ public class CompilerProperties extends AbstractProperties { public static final String COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY = Option.COMPILER_SUBPLAN_NESTEDPUSHDOWN.ini(); + public static final String COMPILER_MIN_MEMORY_ALLOCATION_KEY = Option.COMPILER_MIN_MEMORY_ALLOCATION.ini(); + public static final int COMPILER_PARALLELISM_AS_STORAGE = 0; public CompilerProperties(PropertiesAccessor accessor) { @@ -211,4 +217,8 @@ public class CompilerProperties extends AbstractProperties { public boolean getSubplanNestedPushdown() { return accessor.getBoolean(Option.COMPILER_SUBPLAN_NESTEDPUSHDOWN); } + + public boolean getMinMemoryAllocation() { + return accessor.getBoolean(Option.COMPILER_MIN_MEMORY_ALLOCATION); + } } diff --git a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java index 5079b25..8832054 100644 --- a/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java +++ b/asterixdb/asterix-common/src/main/java/org/apache/asterix/common/config/OptimizationConfUtil.java @@ -70,6 +70,8 @@ public class OptimizationConfUtil { compilerProperties.getSubplanMerge()); boolean subplanNestedPushdown = getBoolean(querySpecificConfig, CompilerProperties.COMPILER_SUBPLAN_NESTEDPUSHDOWN_KEY, compilerProperties.getSubplanNestedPushdown()); + boolean minMemoryAllocation = getBoolean(querySpecificConfig, + CompilerProperties.COMPILER_MIN_MEMORY_ALLOCATION_KEY, compilerProperties.getMinMemoryAllocation()); PhysicalOptimizationConfig physOptConf = new PhysicalOptimizationConfig(); physOptConf.setFrameSize(frameSize); @@ -85,6 +87,7 @@ public class OptimizationConfUtil { physOptConf.setExternalFieldPushdown(externalFieldPushdown); physOptConf.setSubplanMerge(subplanMerge); physOptConf.setSubplanNestedPushdown(subplanNestedPushdown); + physOptConf.setMinMemoryAllocation(minMemoryAllocation); return physOptConf; } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java index 2ed40fd..c99117b 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/config/AlgebricksConfig.java @@ -32,4 +32,5 @@ public class AlgebricksConfig { public static final boolean EXTERNAL_FIELD_PUSHDOWN_DEFAULT = false; public static final boolean SUBPLAN_MERGE_DEFAULT = true; public static final boolean SUBPLAN_NESTEDPUSHDOWN_DEFAULT = true; + public static final boolean MIN_MEMORY_ALLOCATION_DEFAULT = true; } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java index bfa4298..92f579c 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/rewriter/base/PhysicalOptimizationConfig.java @@ -44,8 +44,9 @@ public class PhysicalOptimizationConfig { private static final String EXTERNAL_FIELD_PUSHDOWN = "EXTERNAL_FIELD_PUSHDOWN"; private static final String SUBPLAN_MERGE = "SUBPLAN_MERGE"; private static final String SUBPLAN_NESTEDPUSHDOWN = "SUBPLAN_NESTEDPUSHDOWN"; + private static final String MIN_MEMORY_ALLOCATION = "MIN_MEMORY_ALLOCATION"; - private Properties properties = new Properties(); + private final Properties properties = new Properties(); public PhysicalOptimizationConfig() { int frameSize = 32768; @@ -217,6 +218,14 @@ public class PhysicalOptimizationConfig { setBoolean(SUBPLAN_NESTEDPUSHDOWN, value); } + public boolean getMinMemoryAllocation() { + return getBoolean(MIN_MEMORY_ALLOCATION, AlgebricksConfig.MIN_MEMORY_ALLOCATION_DEFAULT); + } + + public void setMinMemoryAllocation(boolean value) { + setBoolean(MIN_MEMORY_ALLOCATION, value); + } + private void setInt(String property, int value) { properties.setProperty(property, Integer.toString(value)); }
