Till Westmann has submitted this change and it was merged. Change subject: ASTERIXDB-1573 Allow Extension of Rewrite Rules ......................................................................
ASTERIXDB-1573 Allow Extension of Rewrite Rules Add IRuleSetFactory which builds rewrite Rule Set Can be extended by Asterix Extensions Removed some obsolete extension code Change-Id: I7f59dea86b0ef4ee9d31b56766a97bd2b34ec02c Reviewed-on: https://asterix-gerrit.ics.uci.edu/1304 Tested-by: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Till Westmann <[email protected]> --- M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java D asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java A asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/DefaultRuleSetFactory.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java A asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/IRuleSetFactory.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java M asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java M asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java 22 files changed, 274 insertions(+), 343 deletions(-) Approvals: Till Westmann: Looks good to me, approved Jenkins: Verified; Verified Objections: Jenkins: Violations found diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java index ac3bc03..4803691 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java @@ -21,13 +21,6 @@ import org.apache.asterix.common.api.IExtension; import org.apache.asterix.compiler.provider.ILangCompilationProvider; -import org.apache.commons.lang3.mutable.Mutable; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; -import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; -import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; /** * An interface for language extensions @@ -45,20 +38,4 @@ } ILangCompilationProvider getLangCompilationProvider(Language lang); - - //TODO(amoudi/yingyi) this is not a good way to extend re-write rules. introduce rewrite-rule-provider. - /** - * Called by the compiler when the unnest function is an extension function. - * Provides a way to add additional types of datasources - * - * @param opRef - * @param context - * @param unnestOp - * @param unnestExpr - * @param functionCallExpr - * @return true if transformation was successful, false otherwise - * @throws AlgebricksException - */ - boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnestOp, - ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr) throws AlgebricksException; } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java deleted file mode 100644 index ae67337..0000000 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/ExtensionFunctionIdentifier.java +++ /dev/null @@ -1,78 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ -package org.apache.asterix.algebra.extension; - -import org.apache.asterix.common.api.ExtensionId; -import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; - -/** - * an Identifier for an extension function - */ -public class ExtensionFunctionIdentifier extends FunctionIdentifier { - //TODO (till): remove this classs - - private static final long serialVersionUID = 1L; - private final ExtensionId extensionId; - - /** - * Create an identifier for an external function - * - * @param namespace - * @param name - * @param extensionId - */ - public ExtensionFunctionIdentifier(String namespace, String name, ExtensionId extensionId) { - super(namespace, name); - this.extensionId = extensionId; - } - - /** - * Create an identifier for an external function - * - * @param namespace - * @param name - * @param arity - * @param extensionId - */ - public ExtensionFunctionIdentifier(String namespace, String name, int arity, ExtensionId extensionId) { - super(namespace, name, arity); - this.extensionId = extensionId; - } - - public ExtensionId getExtensionId() { - return extensionId; - } - - @Override - public int hashCode() { - return super.hashCode() + extensionId.hashCode(); - } - - @Override - public boolean equals(Object o) { - if (this == o) { - return true; - } - if (o instanceof ExtensionFunctionIdentifier) { - ExtensionFunctionIdentifier oId = (ExtensionFunctionIdentifier) o; - return super.equals(o) && extensionId.equals(oId.getExtensionId()); - } - return false; - } -} diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java index 0a8402c..d9641df 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/extension/IAlgebraExtensionManager.java @@ -18,17 +18,6 @@ */ package org.apache.asterix.algebra.extension; -import org.apache.commons.lang3.mutable.Mutable; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; -import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; -import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; - public interface IAlgebraExtensionManager { - - boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context, UnnestOperator unnestOp, - ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr) throws AlgebricksException; } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java index 00affa8..af0ba73 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/AqlCompilationProvider.java @@ -49,4 +49,9 @@ return new AqlExpressionToPlanTranslatorFactory(); } + @Override + public IRuleSetFactory getRuleSetFactory() { + return new DefaultRuleSetFactory(); + } + } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/DefaultRuleSetFactory.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/DefaultRuleSetFactory.java new file mode 100644 index 0000000..f6526c4 --- /dev/null +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/DefaultRuleSetFactory.java @@ -0,0 +1,86 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.compiler.provider; + +import java.util.ArrayList; +import java.util.List; + +import org.apache.asterix.optimizer.base.RuleCollections; +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialFixpointRuleController; +import org.apache.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialOnceRuleController; +import org.apache.hyracks.algebricks.core.rewriter.base.AbstractRuleController; +import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; + +public class DefaultRuleSetFactory implements IRuleSetFactory { + + @Override + public List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> getLogicalRewrites() + throws AlgebricksException { + return buildLogical(); + } + + @Override + public List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> getPhysicalRewrites() { + return buildPhysical(); + } + + public static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildLogical() { + List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<>(); + SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false); + SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true); + SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true); + defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildInitialTranslationRuleCollection())); + defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildTypeInferenceRuleCollection())); + defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildAutogenerateIDRuleCollection())); + defaultLogicalRewrites + .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection())); + defaultLogicalRewrites + .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection())); + defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection())); + // fj + defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildFuzzyJoinRuleCollection())); + // + defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection())); + defaultLogicalRewrites + .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection())); + defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection())); + defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildDataExchangeRuleCollection())); + defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildConsolidationRuleCollection())); + defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildAccessMethodRuleCollection())); + defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildPlanCleanupRuleCollection())); + + //put TXnRuleCollection! + return defaultLogicalRewrites; + } + + public static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildPhysical() { + List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<>(); + SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true); + SequentialOnceRuleController seqOnceTopLevel = new SequentialOnceRuleController(false); + defaultPhysicalRewrites + .add(new Pair<>(seqOnceCtrl, RuleCollections.buildPhysicalRewritesAllLevelsRuleCollection())); + defaultPhysicalRewrites + .add(new Pair<>(seqOnceTopLevel, RuleCollections.buildPhysicalRewritesTopLevelRuleCollection())); + defaultPhysicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.prepareForJobGenRuleCollection())); + return defaultPhysicalRewrites; + } + +} diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java index fbb55e9..f658be9 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/ILangCompilationProvider.java @@ -45,4 +45,9 @@ */ public ILangExpressionToPlanTranslatorFactory getExpressionToPlanTranslatorFactory(); + /** + * @return the rule set factory of a language implementation + */ + public IRuleSetFactory getRuleSetFactory(); + } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/IRuleSetFactory.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/IRuleSetFactory.java new file mode 100644 index 0000000..af07355 --- /dev/null +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/IRuleSetFactory.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.compiler.provider; + +import java.util.List; + +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.common.utils.Pair; +import org.apache.hyracks.algebricks.core.rewriter.base.AbstractRuleController; +import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; + +public interface IRuleSetFactory { + + /** + * @return the logical rewrites + * @throws AlgebricksException + */ + public List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> getLogicalRewrites() + throws AlgebricksException; + + /** + * @return the physical rewrites + */ + public List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> getPhysicalRewrites() + throws AlgebricksException; + +} diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java index 2b270df..70f8f92 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/compiler/provider/SqlppCompilationProvider.java @@ -49,4 +49,9 @@ return new SqlppExpressionToPlanTranslatorFactory(); } + @Override + public IRuleSetFactory getRuleSetFactory() { + return new DefaultRuleSetFactory(); + } + } diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java index 56a3bd0..3efc4dc 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/base/RuleCollections.java @@ -22,7 +22,6 @@ import java.util.LinkedList; import java.util.List; -import org.apache.asterix.algebra.extension.IAlgebraExtensionManager; import org.apache.asterix.optimizer.rules.AddEquivalenceClassForRecordConstructorRule; import org.apache.asterix.optimizer.rules.AsterixExtractFunctionsFromJoinConditionRule; import org.apache.asterix.optimizer.rules.AsterixInlineVariablesRule; @@ -66,10 +65,10 @@ import org.apache.asterix.optimizer.rules.RemoveRedundantSelectRule; import org.apache.asterix.optimizer.rules.RemoveSortInFeedIngestionRule; import org.apache.asterix.optimizer.rules.RemoveUnusedOneToOneEquiJoinRule; -import org.apache.asterix.optimizer.rules.SetupCommitExtensionOpRule; import org.apache.asterix.optimizer.rules.ResolveVariableRule; import org.apache.asterix.optimizer.rules.SetAsterixPhysicalOperatorsRule; import org.apache.asterix.optimizer.rules.SetClosedRecordConstructorsRule; +import org.apache.asterix.optimizer.rules.SetupCommitExtensionOpRule; import org.apache.asterix.optimizer.rules.SimilarityCheckRule; import org.apache.asterix.optimizer.rules.SweepIllegalNonfunctionalFunctions; import org.apache.asterix.optimizer.rules.UnnestToDataScanRule; @@ -155,9 +154,7 @@ return autogen; } - //TODO(amoudi/yingyi): refactor this to use a provider instead of passing the extensionManager - public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection( - IAlgebraExtensionManager algebraExtensionManager) { + public static final List<IAlgebraicRewriteRule> buildNormalizationRuleCollection() { List<IAlgebraicRewriteRule> normalization = new LinkedList<>(); normalization.add(new ResolveVariableRule()); normalization.add(new IntroduceUnnestForCollectionToSequenceRule()); @@ -183,7 +180,7 @@ normalization.add(new ListifyUnnestingFunctionRule()); normalization.add(new ConstantFoldingRule()); normalization.add(new RemoveRedundantSelectRule()); - normalization.add(new UnnestToDataScanRule(algebraExtensionManager)); + normalization.add(new UnnestToDataScanRule()); normalization.add(new MetaFunctionToMetaVariableRule()); normalization.add(new FuzzyEqRule()); normalization.add(new SimilarityCheckRule()); diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java index c033214..3503f56 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/UnnestToDataScanRule.java @@ -21,7 +21,6 @@ import java.util.ArrayList; import java.util.List; -import org.apache.asterix.algebra.extension.IAlgebraExtensionManager; import org.apache.asterix.common.config.DatasetConfig.DatasetType; import org.apache.asterix.external.feed.watch.FeedActivityDetails; import org.apache.asterix.external.util.ExternalDataUtils; @@ -68,11 +67,6 @@ import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; public class UnnestToDataScanRule implements IAlgebraicRewriteRule { - private final IAlgebraExtensionManager algebraExtensionManager; - - public UnnestToDataScanRule(IAlgebraExtensionManager algebraExtensionManager) { - this.algebraExtensionManager = algebraExtensionManager; - } @Override public boolean rewritePre(Mutable<ILogicalOperator> opRef, IOptimizationContext context) @@ -89,124 +83,114 @@ } UnnestOperator unnest = (UnnestOperator) op; ILogicalExpression unnestExpr = unnest.getExpressionRef().getValue(); + if (unnestExpr.getExpressionTag() != LogicalExpressionTag.FUNCTION_CALL) { + return false; + } + return handleFunction(opRef, context, unnest, (AbstractFunctionCallExpression) unnestExpr); + } - if (unnestExpr.getExpressionTag() == LogicalExpressionTag.FUNCTION_CALL) { - AbstractFunctionCallExpression f = (AbstractFunctionCallExpression) unnestExpr; - FunctionIdentifier fid = f.getFunctionIdentifier(); - - if (fid.equals(AsterixBuiltinFunctions.DATASET)) { - if (unnest.getPositionalVariable() != null) { - // TODO remove this after enabling the support of positional variables in data scan - throw new AlgebricksException("No positional variables are allowed over datasets."); - } - ILogicalExpression expr = f.getArguments().get(0).getValue(); - if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) { - return false; - } - ConstantExpression ce = (ConstantExpression) expr; - IAlgebricksConstantValue acv = ce.getValue(); - if (!(acv instanceof AsterixConstantValue)) { - return false; - } - AsterixConstantValue acv2 = (AsterixConstantValue) acv; - if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) { - return false; - } - String datasetArg = ((AString) acv2.getObject()).getStringValue(); - - AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider(); - Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg); - String dataverseName = datasetReference.first; - String datasetName = datasetReference.second; - Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); - if (dataset == null) { - throw new AlgebricksException( - "Could not find dataset " + datasetName + " in dataverse " + dataverseName); - } - - AqlSourceId asid = new AqlSourceId(dataverseName, datasetName); - List<LogicalVariable> variables = new ArrayList<LogicalVariable>(); - - if (dataset.getDatasetType() == DatasetType.INTERNAL) { - int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size(); - for (int i = 0; i < numPrimaryKeys; i++) { - variables.add(context.newVar()); - } - } - variables.add(unnest.getVariable()); - AqlDataSource dataSource = metadataProvider.findDataSource(asid); - boolean hasMeta = dataSource.hasMeta(); - if (hasMeta) { + protected boolean handleFunction(Mutable<ILogicalOperator> opRef, IOptimizationContext context, + UnnestOperator unnest, AbstractFunctionCallExpression f) throws AlgebricksException { + FunctionIdentifier fid = f.getFunctionIdentifier(); + if (fid.equals(AsterixBuiltinFunctions.DATASET)) { + if (unnest.getPositionalVariable() != null) { + // TODO remove this after enabling the support of positional variables in data scan + throw new AlgebricksException("No positional variables are allowed over datasets."); + } + ILogicalExpression expr = f.getArguments().get(0).getValue(); + if (expr.getExpressionTag() != LogicalExpressionTag.CONSTANT) { + return false; + } + ConstantExpression ce = (ConstantExpression) expr; + IAlgebricksConstantValue acv = ce.getValue(); + if (!(acv instanceof AsterixConstantValue)) { + return false; + } + AsterixConstantValue acv2 = (AsterixConstantValue) acv; + if (acv2.getObject().getType().getTypeTag() != ATypeTag.STRING) { + return false; + } + String datasetArg = ((AString) acv2.getObject()).getStringValue(); + AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider(); + Pair<String, String> datasetReference = parseDatasetReference(metadataProvider, datasetArg); + String dataverseName = datasetReference.first; + String datasetName = datasetReference.second; + Dataset dataset = metadataProvider.findDataset(dataverseName, datasetName); + if (dataset == null) { + throw new AlgebricksException( + "Could not find dataset " + datasetName + " in dataverse " + dataverseName); + } + AqlSourceId asid = new AqlSourceId(dataverseName, datasetName); + List<LogicalVariable> variables = new ArrayList<>(); + if (dataset.getDatasetType() == DatasetType.INTERNAL) { + int numPrimaryKeys = DatasetUtils.getPartitioningKeys(dataset).size(); + for (int i = 0; i < numPrimaryKeys; i++) { variables.add(context.newVar()); } - DataSourceScanOperator scan = new DataSourceScanOperator(variables, dataSource); - List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs(); - scanInpList.addAll(unnest.getInputs()); - opRef.setValue(scan); - addPrimaryKey(variables, dataSource, context); - context.computeAndSetTypeEnvironmentForOperator(scan); - - // Adds equivalence classes --- one equivalent class between a primary key - // variable and a record field-access expression. - IAType[] schemaTypes = dataSource.getSchemaTypes(); - ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] - : schemaTypes[schemaTypes.length - 1]); - ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null); - EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType, - metaRecordType, dataset, context); - return true; - } else if (fid.equals(AsterixBuiltinFunctions.FEED_COLLECT)) { - if (unnest.getPositionalVariable() != null) { - throw new AlgebricksException("No positional variables are allowed over feeds."); - } - - String dataverse = ConstantExpressionUtil.getStringArgument(f, 0); - String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1); - String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2); - String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3); - String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4); - String outputType = ConstantExpressionUtil.getStringArgument(f, 5); - - AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider(); - - AqlSourceId asid = new AqlSourceId(dataverse, getTargetFeed); - String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME); - FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName); - if (policy == null) { - policy = BuiltinFeedPolicies.getFeedPolicy(policyName); - if (policy == null) { - throw new AlgebricksException("Unknown feed policy:" + policyName); - } - } - - ArrayList<LogicalVariable> feedDataScanOutputVariables = new ArrayList<LogicalVariable>(); - - String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS); - List<LogicalVariable> pkVars = new ArrayList<>(); - FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation, - metadataProvider, policy, outputType, csLocations, unnest.getVariable(), context, pkVars); - // The order for feeds is <Record-Meta-PK> - feedDataScanOutputVariables.add(unnest.getVariable()); - // Does it produce meta? - if (ds.hasMeta()) { - feedDataScanOutputVariables.add(context.newVar()); - } - // Does it produce pk? - if (ds.isChange()) { - feedDataScanOutputVariables.addAll(pkVars); - } - - DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds); - List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs(); - scanInpList.addAll(unnest.getInputs()); - opRef.setValue(scan); - context.computeAndSetTypeEnvironmentForOperator(scan); - return true; - } else if (algebraExtensionManager != null) { - return algebraExtensionManager.unnestToDataScan(opRef, context, unnest, unnestExpr, f); } + variables.add(unnest.getVariable()); + AqlDataSource dataSource = metadataProvider.findDataSource(asid); + boolean hasMeta = dataSource.hasMeta(); + if (hasMeta) { + variables.add(context.newVar()); + } + DataSourceScanOperator scan = new DataSourceScanOperator(variables, dataSource); + List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs(); + scanInpList.addAll(unnest.getInputs()); + opRef.setValue(scan); + addPrimaryKey(variables, dataSource, context); + context.computeAndSetTypeEnvironmentForOperator(scan); + // Adds equivalence classes --- one equivalent class between a primary key + // variable and a record field-access expression. + IAType[] schemaTypes = dataSource.getSchemaTypes(); + ARecordType recordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 2] + : schemaTypes[schemaTypes.length - 1]); + ARecordType metaRecordType = (ARecordType) (hasMeta ? schemaTypes[schemaTypes.length - 1] : null); + EquivalenceClassUtils.addEquivalenceClassesForPrimaryIndexAccess(scan, variables, recordType, + metaRecordType, dataset, context); + return true; + } else if (fid.equals(AsterixBuiltinFunctions.FEED_COLLECT)) { + if (unnest.getPositionalVariable() != null) { + throw new AlgebricksException("No positional variables are allowed over feeds."); + } + String dataverse = ConstantExpressionUtil.getStringArgument(f, 0); + String sourceFeedName = ConstantExpressionUtil.getStringArgument(f, 1); + String getTargetFeed = ConstantExpressionUtil.getStringArgument(f, 2); + String subscriptionLocation = ConstantExpressionUtil.getStringArgument(f, 3); + String targetDataset = ConstantExpressionUtil.getStringArgument(f, 4); + String outputType = ConstantExpressionUtil.getStringArgument(f, 5); + AqlMetadataProvider metadataProvider = (AqlMetadataProvider) context.getMetadataProvider(); + AqlSourceId asid = new AqlSourceId(dataverse, getTargetFeed); + String policyName = metadataProvider.getConfig().get(FeedActivityDetails.FEED_POLICY_NAME); + FeedPolicyEntity policy = metadataProvider.findFeedPolicy(dataverse, policyName); + if (policy == null) { + policy = BuiltinFeedPolicies.getFeedPolicy(policyName); + if (policy == null) { + throw new AlgebricksException("Unknown feed policy:" + policyName); + } + } + ArrayList<LogicalVariable> feedDataScanOutputVariables = new ArrayList<>(); + String csLocations = metadataProvider.getConfig().get(FeedActivityDetails.COLLECT_LOCATIONS); + List<LogicalVariable> pkVars = new ArrayList<>(); + FeedDataSource ds = createFeedDataSource(asid, targetDataset, sourceFeedName, subscriptionLocation, + metadataProvider, policy, outputType, csLocations, unnest.getVariable(), context, pkVars); + // The order for feeds is <Record-Meta-PK> + feedDataScanOutputVariables.add(unnest.getVariable()); + // Does it produce meta? + if (ds.hasMeta()) { + feedDataScanOutputVariables.add(context.newVar()); + } + // Does it produce pk? + if (ds.isChange()) { + feedDataScanOutputVariables.addAll(pkVars); + } + DataSourceScanOperator scan = new DataSourceScanOperator(feedDataScanOutputVariables, ds); + List<Mutable<ILogicalOperator>> scanInpList = scan.getInputs(); + scanInpList.addAll(unnest.getInputs()); + opRef.setValue(scan); + context.computeAndSetTypeEnvironmentForOperator(scan); + return true; } - return false; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index eb1764c..31ef778 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -21,13 +21,11 @@ import java.io.IOException; import java.io.PrintWriter; import java.rmi.RemoteException; -import java.util.ArrayList; import java.util.List; import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslator; import org.apache.asterix.algebra.base.ILangExpressionToPlanTranslatorFactory; import org.apache.asterix.api.common.Job.SubmissionMode; -import org.apache.asterix.app.cc.CompilerExtensionManager; import org.apache.asterix.app.result.ResultUtil; import org.apache.asterix.common.config.AsterixCompilerProperties; import org.apache.asterix.common.config.AsterixExternalProperties; @@ -35,6 +33,7 @@ import org.apache.asterix.common.exceptions.ACIDException; import org.apache.asterix.common.exceptions.AsterixException; import org.apache.asterix.compiler.provider.ILangCompilationProvider; +import org.apache.asterix.compiler.provider.IRuleSetFactory; import org.apache.asterix.dataflow.data.common.AqlExpressionTypeComputer; import org.apache.asterix.dataflow.data.common.AqlMergeAggregationExpressionFactory; import org.apache.asterix.dataflow.data.common.AqlMissableTypeComputer; @@ -50,7 +49,6 @@ import org.apache.asterix.lang.common.statement.FunctionDecl; import org.apache.asterix.lang.common.statement.Query; import org.apache.asterix.metadata.declared.AqlMetadataProvider; -import org.apache.asterix.optimizer.base.RuleCollections; import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.asterix.runtime.util.AsterixAppContextInfo; import org.apache.asterix.transaction.management.service.transaction.JobIdFactory; @@ -63,8 +61,6 @@ import org.apache.hyracks.algebricks.compiler.api.HeuristicCompilerFactoryBuilder; import org.apache.hyracks.algebricks.compiler.api.ICompiler; import org.apache.hyracks.algebricks.compiler.api.ICompilerFactory; -import org.apache.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialFixpointRuleController; -import org.apache.hyracks.algebricks.compiler.rewriter.rulecontrollers.SequentialOnceRuleController; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalPlan; import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; import org.apache.hyracks.algebricks.core.algebra.expressions.IConflictingTypeResolver; @@ -77,9 +73,7 @@ import org.apache.hyracks.algebricks.core.algebra.prettyprint.LogicalOperatorPrettyPrintVisitor; import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPlotter; import org.apache.hyracks.algebricks.core.algebra.prettyprint.PlanPrettyPrinter; -import org.apache.hyracks.algebricks.core.rewriter.base.AbstractRuleController; import org.apache.hyracks.algebricks.core.rewriter.base.AlgebricksOptimizationContext; -import org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule; import org.apache.hyracks.algebricks.core.rewriter.base.IOptimizationContextFactory; import org.apache.hyracks.algebricks.core.rewriter.base.PhysicalOptimizationConfig; import org.apache.hyracks.api.client.IHyracksClientConnection; @@ -96,56 +90,13 @@ private final IRewriterFactory rewriterFactory; private final IAstPrintVisitorFactory astPrintVisitorFactory; private final ILangExpressionToPlanTranslatorFactory translatorFactory; - private final CompilerExtensionManager cExtensionManager; + private final IRuleSetFactory ruleSetFactory; - public APIFramework(ILangCompilationProvider compilationProvider, CompilerExtensionManager cExtensionManager) { + public APIFramework(ILangCompilationProvider compilationProvider) { this.rewriterFactory = compilationProvider.getRewriterFactory(); this.astPrintVisitorFactory = compilationProvider.getAstPrintVisitorFactory(); this.translatorFactory = compilationProvider.getExpressionToPlanTranslatorFactory(); - this.cExtensionManager = cExtensionManager; - } - - private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> - buildDefaultLogicalRewrites(CompilerExtensionManager ccExtensionManager) { - List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultLogicalRewrites = new ArrayList<>(); - SequentialFixpointRuleController seqCtrlNoDfs = new SequentialFixpointRuleController(false); - SequentialFixpointRuleController seqCtrlFullDfs = new SequentialFixpointRuleController(true); - SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true); - defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildInitialTranslationRuleCollection())); - defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildTypeInferenceRuleCollection())); - defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildAutogenerateIDRuleCollection())); - defaultLogicalRewrites - .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection(ccExtensionManager))); - defaultLogicalRewrites - .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection())); - defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection())); - // fj - defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildFuzzyJoinRuleCollection())); - // - defaultLogicalRewrites - .add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildNormalizationRuleCollection(ccExtensionManager))); - defaultLogicalRewrites - .add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildCondPushDownAndJoinInferenceRuleCollection())); - defaultLogicalRewrites.add(new Pair<>(seqCtrlFullDfs, RuleCollections.buildLoadFieldsRuleCollection())); - defaultLogicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.buildDataExchangeRuleCollection())); - defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildConsolidationRuleCollection())); - defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildAccessMethodRuleCollection())); - defaultLogicalRewrites.add(new Pair<>(seqCtrlNoDfs, RuleCollections.buildPlanCleanupRuleCollection())); - - //put TXnRuleCollection! - return defaultLogicalRewrites; - } - - private static List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> buildDefaultPhysicalRewrites() { - List<Pair<AbstractRuleController, List<IAlgebraicRewriteRule>>> defaultPhysicalRewrites = new ArrayList<>(); - SequentialOnceRuleController seqOnceCtrl = new SequentialOnceRuleController(true); - SequentialOnceRuleController seqOnceTopLevel = new SequentialOnceRuleController(false); - defaultPhysicalRewrites - .add(new Pair<>(seqOnceCtrl, RuleCollections.buildPhysicalRewritesAllLevelsRuleCollection())); - defaultPhysicalRewrites - .add(new Pair<>(seqOnceTopLevel, RuleCollections.buildPhysicalRewritesTopLevelRuleCollection())); - defaultPhysicalRewrites.add(new Pair<>(seqOnceCtrl, RuleCollections.prepareForJobGenRuleCollection())); - return defaultPhysicalRewrites; + this.ruleSetFactory = compilationProvider.getRuleSetFactory(); } private static class AqlOptimizationContextFactory implements IOptimizationContextFactory { @@ -258,8 +209,8 @@ HeuristicCompilerFactoryBuilder builder = new HeuristicCompilerFactoryBuilder(AqlOptimizationContextFactory.INSTANCE); builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig()); - builder.setLogicalRewrites(buildDefaultLogicalRewrites(cExtensionManager)); - builder.setPhysicalRewrites(buildDefaultPhysicalRewrites()); + builder.setLogicalRewrites(ruleSetFactory.getLogicalRewrites()); + builder.setPhysicalRewrites(ruleSetFactory.getPhysicalRewrites()); IDataFormat format = queryMetadataProvider.getFormat(); ICompilerFactory compilerFactory = builder.create(); builder.setExpressionEvalSizeComputer(format.getExpressionEvalSizeComputer()); diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java index b269ecf..fd9c6cd 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/java/AsterixJavaClient.java @@ -56,7 +56,7 @@ this.queryText = queryText; this.writer = writer; this.compilationProvider = compilationProvider; - this.apiFramework = new APIFramework(compilationProvider, null); + this.apiFramework = new APIFramework(compilationProvider); this.statementExecutorFactory = statementExecutorFactory; parserFactory = compilationProvider.getParserFactory(); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java index af10394..4734841 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CompilerExtensionManager.java @@ -24,7 +24,6 @@ import org.apache.asterix.algebra.base.ILangExtension; import org.apache.asterix.algebra.base.ILangExtension.Language; -import org.apache.asterix.algebra.extension.ExtensionFunctionIdentifier; import org.apache.asterix.algebra.extension.IAlgebraExtensionManager; import org.apache.asterix.app.translator.DefaultStatementExecutorFactory; import org.apache.asterix.common.api.ExtensionId; @@ -35,15 +34,7 @@ import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.compiler.provider.SqlppCompilationProvider; import org.apache.asterix.translator.IStatementExecutorFactory; -import org.apache.commons.lang3.mutable.Mutable; -import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalOperator; -import org.apache.hyracks.algebricks.core.algebra.base.IOptimizationContext; -import org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression; -import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; -import org.apache.hyracks.algebricks.core.algebra.operators.logical.UnnestOperator; import org.apache.hyracks.api.exceptions.HyracksDataException; /** @@ -77,7 +68,7 @@ Pair<ExtensionId, ILangCompilationProvider> aqlcp = null; Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null; IStatementExecutorExtension see = null; - defaultQueryTranslatorFactory = new DefaultStatementExecutorFactory(this); + defaultQueryTranslatorFactory = new DefaultStatementExecutorFactory(); if (list != null) { for (AsterixExtension extensionConf : list) { @@ -138,19 +129,5 @@ public ILangCompilationProvider getSqlppCompilationProvider() { return sqlppCompilationProvider; - } - - // TODO(amoudi/yingyi): this is not a good way to extend rewrite rules. introduce re-write rule provider - @Override - public boolean unnestToDataScan(Mutable<ILogicalOperator> opRef, IOptimizationContext context, - UnnestOperator unnestOp, ILogicalExpression unnestExpr, AbstractFunctionCallExpression functionCallExpr) - throws AlgebricksException { - FunctionIdentifier functionId = functionCallExpr.getFunctionIdentifier(); - if (functionId instanceof ExtensionFunctionIdentifier) { - ILangExtension extension = - (ILangExtension) extensions.get(((ExtensionFunctionIdentifier) functionId).getExtensionId()); - return extension.unnestToDataScan(opRef, context, unnestOp, unnestExpr, functionCallExpr); - } - return false; } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java index 62f11df..414585a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/external/FeedWorkCollection.java @@ -70,7 +70,7 @@ private static class SubscribeFeedWorkRunnable implements Runnable { - private static final DefaultStatementExecutorFactory qtFactory = new DefaultStatementExecutorFactory(null); + private static final DefaultStatementExecutorFactory qtFactory = new DefaultStatementExecutorFactory(); private final FeedConnectionRequest request; private final String[] locations; diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java index 1b10f3a..6cdf329 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/DefaultStatementExecutorFactory.java @@ -20,7 +20,6 @@ import java.util.List; -import org.apache.asterix.app.cc.CompilerExtensionManager; import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.lang.common.base.Statement; import org.apache.asterix.translator.IStatementExecutorFactory; @@ -28,16 +27,10 @@ public class DefaultStatementExecutorFactory implements IStatementExecutorFactory { - protected final CompilerExtensionManager cExtensionManager; - - public DefaultStatementExecutorFactory(CompilerExtensionManager cExtensionManager) { - this.cExtensionManager = cExtensionManager; - } - @Override public QueryTranslator create(List<Statement> aqlStatements, SessionConfig conf, ILangCompilationProvider compilationProvider) { - return new QueryTranslator(aqlStatements, conf, compilationProvider, cExtensionManager); + return new QueryTranslator(aqlStatements, conf, compilationProvider); } } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java index d5b1415..0b6d806 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java @@ -47,7 +47,6 @@ import org.apache.asterix.algebra.extension.IExtensionStatement; import org.apache.asterix.api.common.APIFramework; import org.apache.asterix.api.http.servlet.APIServlet; -import org.apache.asterix.app.cc.CompilerExtensionManager; import org.apache.asterix.app.external.ExternalIndexingOperations; import org.apache.asterix.app.external.FeedJoint; import org.apache.asterix.app.external.FeedOperations; @@ -230,12 +229,12 @@ protected final APIFramework apiFramework; protected final IRewriterFactory rewriterFactory; - public QueryTranslator(List<Statement> aqlStatements, SessionConfig conf, - ILangCompilationProvider compliationProvider, CompilerExtensionManager ccExtensionManager) { - this.statements = aqlStatements; + public QueryTranslator(List<Statement> statements, SessionConfig conf, + ILangCompilationProvider compliationProvider) { + this.statements = statements; this.sessionConfig = conf; - this.declaredFunctions = getDeclaredFunctions(aqlStatements); - this.apiFramework = new APIFramework(compliationProvider, ccExtensionManager); + this.declaredFunctions = getDeclaredFunctions(statements); + this.apiFramework = new APIFramework(compliationProvider); this.rewriterFactory = compliationProvider.getRewriterFactory(); activeDefaultDataverse = MetadataBuiltinEntities.DEFAULT_DATAVERSE; } @@ -3123,7 +3122,7 @@ return (dataverse != null) ? dataverse : activeDefaultDataverse.getDataverseName(); } - protected String getActiveDataverse(Identifier dataverse) { + public String getActiveDataverse(Identifier dataverse) { return getActiveDataverseName(dataverse != null ? dataverse.getValue() : null); } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java index 486b4ab..9b9ab4a 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixCLI.java @@ -58,7 +58,7 @@ for (String queryFile : options.args) { Reader in = new FileReader(queryFile); AsterixJavaClient ajc = new AsterixJavaClient(integrationUtil.getHyracksClientConnection(), in, - compilationProvider, new DefaultStatementExecutorFactory(null)); + compilationProvider, new DefaultStatementExecutorFactory()); try { ajc.compile(true, false, false, false, false, true, false); } finally { diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java index e0a60c6..f05b67d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixClientDriver.java @@ -64,7 +64,7 @@ ILangCompilationProvider compilationProvider = new AqlCompilationProvider(); FileReader reader = new FileReader(filename); AsterixJavaClient q = new AsterixJavaClient(hcc, reader, compilationProvider, - new DefaultStatementExecutorFactory(null)); + new DefaultStatementExecutorFactory()); q.compile(optimize, true, true, true, onlyPhysical, createBinaryRuntime, createBinaryRuntime); return q; } diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java index 8bcceca..f0eac39 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/drivers/AsterixWebServer.java @@ -34,7 +34,7 @@ server.setHandler(context); context.addServlet(new ServletHolder(new APIServlet(new AqlCompilationProvider(), - new SqlppCompilationProvider(), new DefaultStatementExecutorFactory(null))), "/*"); + new SqlppCompilationProvider(), new DefaultStatementExecutorFactory())), "/*"); server.start(); server.join(); } diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java index 56d4191..53b8602 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/aql/translator/QueryTranslatorTest.java @@ -69,7 +69,7 @@ when(mockCluster.getMasterNode()).thenReturn(mockMasterNode); when(mockMasterNode.getClientIp()).thenReturn("127.0.0.1"); - IStatementExecutor aqlTranslator = new DefaultStatementExecutorFactory(null).create(statements, mockSessionConfig, + IStatementExecutor aqlTranslator = new DefaultStatementExecutorFactory().create(statements, mockSessionConfig, new AqlCompilationProvider()); List<String> parameters = new ArrayList<String>(); parameters.add("examples/pregelix-example-jar-with-dependencies.jar"); diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java index 34dab28..d5501c3 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/dml/DmlTest.java @@ -61,7 +61,7 @@ Reader loadReader = new BufferedReader( new InputStreamReader(new FileInputStream(LOAD_FOR_ENLIST_FILE), "UTF-8")); AsterixJavaClient asterixLoad = new AsterixJavaClient(integrationUtil.getHyracksClientConnection(), loadReader, - ERR, new AqlCompilationProvider(), new DefaultStatementExecutorFactory(null)); + ERR, new AqlCompilationProvider(), new DefaultStatementExecutorFactory()); try { asterixLoad.compile(true, false, false, false, false, true, false); } catch (AsterixException e) { diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java index 2716859..8db2bc0 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/optimizer/OptimizerTest.java @@ -30,7 +30,6 @@ import org.apache.asterix.api.common.AsterixHyracksIntegrationUtil; import org.apache.asterix.api.java.AsterixJavaClient; -import org.apache.asterix.app.cc.CompilerExtensionManager; import org.apache.asterix.app.translator.DefaultStatementExecutorFactory; import org.apache.asterix.common.config.GlobalConfig; import org.apache.asterix.common.exceptions.AsterixException; @@ -39,10 +38,10 @@ import org.apache.asterix.compiler.provider.SqlppCompilationProvider; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.IdentitiyResolverFactory; -import org.apache.asterix.runtime.util.AsterixAppContextInfo; import org.apache.asterix.test.base.AsterixTestHelper; import org.apache.asterix.test.common.TestHelper; import org.apache.asterix.test.runtime.HDFSCluster; +import org.apache.asterix.translator.IStatementExecutorFactory; import org.apache.hyracks.api.client.IHyracksClientConnection; import org.junit.AfterClass; import org.junit.Assume; @@ -76,6 +75,7 @@ private static final ILangCompilationProvider aqlCompilationProvider = new AqlCompilationProvider(); private static final ILangCompilationProvider sqlppCompilationProvider = new SqlppCompilationProvider(); protected static ILangCompilationProvider extensionLangCompilationProvider = null; + protected static IStatementExecutorFactory statementExecutorFactory = new DefaultStatementExecutorFactory(); protected static AsterixHyracksIntegrationUtil integrationUtil = new AsterixHyracksIntegrationUtil(); @@ -178,9 +178,7 @@ provider = extensionLangCompilationProvider; } IHyracksClientConnection hcc = integrationUtil.getHyracksClientConnection(); - AsterixJavaClient asterix = new AsterixJavaClient(hcc, query, plan, provider, - new DefaultStatementExecutorFactory( - (CompilerExtensionManager) AsterixAppContextInfo.INSTANCE.getExtensionManager())); + AsterixJavaClient asterix = new AsterixJavaClient(hcc, query, plan, provider, statementExecutorFactory); try { asterix.compile(true, false, false, true, true, false, false); } catch (AsterixException e) { -- To view, visit https://asterix-gerrit.ics.uci.edu/1304 To unsubscribe, visit https://asterix-gerrit.ics.uci.edu/settings Gerrit-MessageType: merged Gerrit-Change-Id: I7f59dea86b0ef4ee9d31b56766a97bd2b34ec02c Gerrit-PatchSet: 8 Gerrit-Project: asterixdb Gerrit-Branch: master Gerrit-Owner: Steven Jacobs <[email protected]> Gerrit-Reviewer: Jenkins <[email protected]> Gerrit-Reviewer: Steven Jacobs <[email protected]> Gerrit-Reviewer: Till Westmann <[email protected]> Gerrit-Reviewer: Yingyi Bu <[email protected]> Gerrit-Reviewer: abdullah alamoudi <[email protected]>
