This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit a700ae3420698a6b3e5a2941e98efc90cb78f2e3 Author: Ali Alsuliman <[email protected]> AuthorDate: Mon Oct 30 22:43:37 2023 -0700 [ASTERIXDB-3293][COMP] Do not require job capacity for metadata queries - user model changes: no - storage format changes: no - interface changes: yes Details: Metadata queries can sometimes fail to run because they are forced to use the minimum budget for the operators. This patch is to lift this minimum budget and let the operators be assigned the default budget like regular queries. In addition, do not assign a job capacity for metadata queries. A better solution for metadata queries should consider a combination of solutions (e.g. calculate job capacity precisely given that metadata queries are run in a single partition, don't make the default budget of operators for regular and metadata queries the same, make minimum budget configurable, ... etc) Change-Id: I2a6721e2b6182aa0e12a1b0173709bc139fc77e0 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17894 Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ali Alsuliman <[email protected]> Reviewed-by: Murtadha Hubail <[email protected]> Tested-by: Michael Blow <[email protected]> --- .../optimizer/base/AsterixOptimizationContext.java | 24 ++++++++++++++++++++++ .../rules/SetAsterixMemoryRequirementsRule.java | 3 --- .../apache/asterix/api/common/APIFramework.java | 22 +++++++++++--------- .../misc/metadata_only_01/metadata_only_01.1.regex | 2 +- .../misc/metadata_only_01/metadata_only_01.2.regex | 2 +- .../api/HeuristicCompilerFactoryBuilder.java | 5 +++++ .../hyracks/algebricks/compiler/api/ICompiler.java | 2 ++ .../core/algebra/base/IOptimizationContext.java | 4 ++++ 8 files changed, 49 insertions(+), 15 deletions(-) diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java index 7b2dd7a9b1..05e7f7b8a6 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/AsterixOptimizationContext.java @@ -21,8 +21,10 @@ package org.apache.asterix.optimizer.base; import java.util.HashSet; import java.util.Set; +import java.util.function.Predicate; import org.apache.asterix.metadata.declared.DataSource; +import org.apache.asterix.metadata.utils.MetadataConstants; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver; import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer; @@ -73,4 +75,26 @@ public final class AsterixOptimizationContext extends AlgebricksOptimizationCont public Int2ObjectMap<Set<DataSource>> getDataSourceMap() { return dataSourceMap; } + + @Override + public boolean skipJobCapacityAssignment() { + if (dataSourceMap.isEmpty()) { + return false; + } + for (Int2ObjectMap.Entry<Set<DataSource>> me : dataSourceMap.int2ObjectEntrySet()) { + int dataSourceType = me.getIntKey(); + if (dataSourceType != DataSource.Type.INTERNAL_DATASET) { + return false; + } + Predicate<DataSource> dataSourceTest = AsterixOptimizationContext::isMetadata; + if (!me.getValue().stream().allMatch(dataSourceTest)) { + return false; + } + } + return true; + } + + private static boolean isMetadata(DataSource ds) { + return MetadataConstants.METADATA_DATAVERSE_NAME.equals(ds.getId().getDataverseName()); + } } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java index e9de35392d..9467555234 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/SetAsterixMemoryRequirementsRule.java @@ -63,9 +63,6 @@ public final class SetAsterixMemoryRequirementsRule extends SetMemoryRequirement int dataSourceType = me.getIntKey(); Predicate<DataSource> dataSourceTest; switch (dataSourceType) { - case DataSource.Type.INTERNAL_DATASET: - dataSourceTest = SetAsterixMemoryRequirementsRule::isMinMemoryBudgetDataset; - break; case DataSource.Type.FUNCTION: dataSourceTest = SetAsterixMemoryRequirementsRule::isMinMemoryBudgetFunction; break; diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index be21673b33..99678dabc4 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -323,16 +323,18 @@ public class APIFramework { JobSpecification spec = compiler.createJob(ccAppContext, jobEventListenerFactory); if (isQuery) { - if (requestParameters == null || !requestParameters.isSkipAdmissionPolicy()) { - // Sets a required capacity, only for read-only queries. - // DDLs and DMLs are considered not that frequent. - // limit the computation locations to the locations that will be used in the query - final INodeJobTracker nodeJobTracker = ccAppContext.getNodeJobTracker(); - final AlgebricksAbsolutePartitionConstraint jobLocations = - getJobLocations(spec, nodeJobTracker, computationLocations); - final IClusterCapacity jobRequiredCapacity = - ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf); - spec.setRequiredClusterCapacity(jobRequiredCapacity); + if (!compiler.skipJobCapacityAssignment()) { + if (requestParameters == null || !requestParameters.isSkipAdmissionPolicy()) { + // Sets a required capacity, only for read-only queries. + // DDLs and DMLs are considered not that frequent. + // limit the computation locations to the locations that will be used in the query + final INodeJobTracker nodeJobTracker = ccAppContext.getNodeJobTracker(); + final AlgebricksAbsolutePartitionConstraint jobLocations = + getJobLocations(spec, nodeJobTracker, computationLocations); + final IClusterCapacity jobRequiredCapacity = + ResourceUtils.getRequiredCapacity(plan, jobLocations, physOptConf); + spec.setRequiredClusterCapacity(jobRequiredCapacity); + } } } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.1.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.1.regex index 7bc4c426f4..959bc0d0c3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.1.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.1.regex @@ -1 +1 @@ -/memory\D+917504/ +/memory\D+0/ diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex index acf84056d4..959bc0d0c3 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/metadata_only_01/metadata_only_01.2.regex @@ -1 +1 @@ -/memory\D+1146880/ +/memory\D+0/ diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java index e35a539952..85910aa1c4 100644 --- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java +++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/HeuristicCompilerFactoryBuilder.java @@ -170,5 +170,10 @@ public class HeuristicCompilerFactoryBuilder extends AbstractCompilerFactoryBuil PlanCompiler pc = factory.createPlanCompiler(oc, appContext, writerFactory); return pc.compilePlan(plan, jobEventListenerFactory); } + + @Override + public boolean skipJobCapacityAssignment() { + return oc.skipJobCapacityAssignment(); + } } } diff --git a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java index 7dac3ebb2c..0f5798e578 100644 --- a/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java +++ b/hyracks-fullstack/algebricks/algebricks-compiler/src/main/java/org/apache/hyracks/algebricks/compiler/api/ICompiler.java @@ -27,4 +27,6 @@ public interface ICompiler { public JobSpecification createJob(Object appContext, IJobletEventListenerFactory jobEventListenerFactory) throws AlgebricksException; + + boolean skipJobCapacityAssignment(); } diff --git a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java index 69ec210362..10b04516f1 100644 --- a/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java +++ b/hyracks-fullstack/algebricks/algebricks-core/src/main/java/org/apache/hyracks/algebricks/core/algebra/base/IOptimizationContext.java @@ -100,4 +100,8 @@ public interface IOptimizationContext extends ITypingContext, IVariableContext { Object getCompilerFactory(); IOptimizationContextFactory getOptimizationContextFactory(); + + default boolean skipJobCapacityAssignment() { + return false; + } }
