[ASTERIXDB-2154][COMP] Extensible built-in function management - user model changes: no - storage format changes: no - interface changes: yes
Details: - Support for augmenting built-in functions in ILangExtension - Move function type inferers from NonTaggedDataFormat to FunctionTypeInferers, exposed through IFunctionDescriptorFactory and IFunctionManager - Cleanup: obtain IFunctionManager/IDataFormat from MetadataProvider Change-Id: Iafbf7e4e298fb7af6cb84b402cb00cfa2e90b5ce Reviewed-on: https://asterix-gerrit.ics.uci.edu/2118 Tested-by: Jenkins <[email protected]> Contrib: Jenkins <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Ian Maxon <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/asterixdb/repo Commit: http://git-wip-us.apache.org/repos/asf/asterixdb/commit/76d24d49 Tree: http://git-wip-us.apache.org/repos/asf/asterixdb/tree/76d24d49 Diff: http://git-wip-us.apache.org/repos/asf/asterixdb/diff/76d24d49 Branch: refs/heads/master Commit: 76d24d49055d764fc03b19bce441e97a563622b4 Parents: 52e5134 Author: Dmitry Lychagin <[email protected]> Authored: Fri Nov 3 13:28:21 2017 -0700 Committer: Dmitry Lychagin <[email protected]> Committed: Sat Nov 4 11:56:19 2017 -0700 ---------------------------------------------------------------------- .../asterix/algebra/base/ILangExtension.java | 5 + .../jobgen/QueryLogicalExpressionJobGen.java | 62 +- .../optimizer/rules/ConstantFoldingRule.java | 9 +- .../apache/asterix/api/common/APIFramework.java | 5 +- .../asterix/app/cc/CCExtensionManager.java | 30 +- .../org/apache/asterix/utils/ExtensionUtil.java | 36 +- .../org/apache/asterix/runtime/ExceptionIT.java | 3 +- .../apache/asterix/runtime/NullMissingTest.java | 3 +- .../test/active/ActiveEventsListenerTest.java | 12 + .../metadata/declared/DatasetDataSource.java | 4 +- .../metadata/declared/FeedDataSource.java | 3 +- .../metadata/declared/MetadataProvider.java | 17 +- .../asterix/metadata/entities/Dataset.java | 2 +- .../asterix/metadata/utils/DatasetUtil.java | 18 +- .../utils/SecondaryBTreeOperationsHelper.java | 17 +- ...econdaryCorrelatedBTreeOperationsHelper.java | 17 +- ...CorrelatedInvertedIndexOperationsHelper.java | 16 +- ...econdaryCorrelatedRTreeOperationsHelper.java | 6 +- ...daryCorrelatedTreeIndexOperationsHelper.java | 7 +- .../utils/SecondaryIndexOperationsHelper.java | 17 +- .../SecondaryInvertedIndexOperationsHelper.java | 16 +- .../utils/SecondaryRTreeOperationsHelper.java | 6 +- .../asterix/formats/base/IDataFormat.java | 19 +- .../functions/IFunctionDescriptorFactory.java | 10 +- .../om/functions/IFunctionExtensionManager.java | 32 + .../asterix/om/functions/IFunctionManager.java | 12 +- .../om/functions/IFunctionTypeInferer.java | 29 + .../collections/ListifyAggregateDescriptor.java | 10 +- .../AbstractScalarAggregateDescriptor.java | 14 +- .../scalar/ScalarAvgAggregateDescriptor.java | 10 +- .../scalar/ScalarCountAggregateDescriptor.java | 10 +- .../scalar/ScalarMaxAggregateDescriptor.java | 10 +- .../scalar/ScalarMinAggregateDescriptor.java | 10 +- .../scalar/ScalarSqlAvgAggregateDescriptor.java | 10 +- .../ScalarSqlCountAggregateDescriptor.java | 11 +- .../scalar/ScalarSqlMaxAggregateDescriptor.java | 10 +- .../scalar/ScalarSqlMinAggregateDescriptor.java | 10 +- .../scalar/ScalarSqlSumAggregateDescriptor.java | 10 +- .../scalar/ScalarSumAggregateDescriptor.java | 10 +- .../ClosedRecordConstructorDescriptor.java | 10 +- .../OpenRecordConstructorDescriptor.java | 7 + .../OrderedListConstructorDescriptor.java | 12 +- .../UnorderedListConstructorDescriptor.java | 12 +- .../functions/CastTypeDescriptor.java | 9 +- .../functions/CastTypeLaxDescriptor.java | 7 + .../functions/DeepEqualityDescriptor.java | 9 +- .../records/FieldAccessByIndexDescriptor.java | 11 +- .../records/FieldAccessNestedDescriptor.java | 11 +- .../records/GetRecordFieldValueDescriptor.java | 11 +- .../records/GetRecordFieldsDescriptor.java | 11 +- .../records/RecordAddFieldsDescriptor.java | 8 + .../records/RecordMergeDescriptor.java | 8 + .../records/RecordPairsDescriptor.java | 7 + .../records/RecordRemoveFieldsDescriptor.java | 18 +- .../runtime/formats/NonTaggedDataFormat.java | 350 +--------- .../runtime/functions/FunctionCollection.java | 641 ++++++++++--------- .../runtime/functions/FunctionManager.java | 73 +++ .../functions/FunctionManagerHolder.java | 41 -- .../runtime/functions/FunctionManagerImpl.java | 68 -- .../runtime/functions/FunctionTypeInferers.java | 271 ++++++++ 60 files changed, 1189 insertions(+), 944 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java index 4803691..fb74475 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/algebra/base/ILangExtension.java @@ -21,6 +21,7 @@ package org.apache.asterix.algebra.base; import org.apache.asterix.common.api.IExtension; import org.apache.asterix.compiler.provider.ILangCompilationProvider; +import org.apache.asterix.om.functions.IFunctionManager; /** * An interface for language extensions @@ -38,4 +39,8 @@ public interface ILangExtension extends IExtension { } ILangCompilationProvider getLangCompilationProvider(Language lang); + + default IFunctionManager getFunctionManager() { + return null; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java index 9f88bb6..812f3c9 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java @@ -23,11 +23,12 @@ import java.util.List; import org.apache.asterix.common.dataflow.ICcApplicationContext; import org.apache.asterix.common.functions.FunctionDescriptorTag; import org.apache.asterix.external.library.ExternalFunctionDescriptorProvider; -import org.apache.asterix.formats.base.IDataFormat; +import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.functions.IExternalFunctionInfo; import org.apache.asterix.om.functions.IFunctionDescriptor; -import org.apache.asterix.runtime.formats.FormatUtils; +import org.apache.asterix.om.functions.IFunctionManager; +import org.apache.asterix.om.functions.IFunctionTypeInferer; import org.apache.commons.lang3.mutable.Mutable; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; @@ -40,6 +41,7 @@ import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvir import org.apache.hyracks.algebricks.core.algebra.expressions.StatefulFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.UnnestingFunctionCallExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.VariableReferenceExpression; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.core.algebra.operators.logical.IOperatorSchema; import org.apache.hyracks.algebricks.core.jobgen.impl.JobGenContext; import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory; @@ -51,9 +53,10 @@ import org.apache.hyracks.algebricks.runtime.evaluators.ColumnAccessEvalFactory; public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { - public static final QueryLogicalExpressionJobGen INSTANCE = new QueryLogicalExpressionJobGen(); + private final IFunctionManager functionManager; - private QueryLogicalExpressionJobGen() { + public QueryLogicalExpressionJobGen(IFunctionManager functionManager) { + this.functionManager = functionManager; } @Override @@ -61,7 +64,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); - IFunctionDescriptor fd = getFunctionDescriptor(expr, env, context); + IFunctionDescriptor fd = resolveFunction(expr, env); switch (fd.getFunctionDescriptorTag()) { case SERIALAGGREGATE: return null; @@ -79,7 +82,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); - return getFunctionDescriptor(expr, env, context).createRunningAggregateEvaluatorFactory(args); + return resolveFunction(expr, env).createRunningAggregateEvaluatorFactory(args); } @Override @@ -87,22 +90,22 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); - return getFunctionDescriptor(expr, env, context).createUnnestingEvaluatorFactory(args); + return resolveFunction(expr, env).createUnnestingEvaluatorFactory(args); } @Override public IScalarEvaluatorFactory createEvaluatorFactory(ILogicalExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { - IScalarEvaluatorFactory copyEvaluatorFactory = null; + IScalarEvaluatorFactory copyEvaluatorFactory; switch (expr.getExpressionTag()) { case VARIABLE: { VariableReferenceExpression v = (VariableReferenceExpression) expr; - copyEvaluatorFactory = createVariableEvaluatorFactory(v, inputSchemas, context); + copyEvaluatorFactory = createVariableEvaluatorFactory(v, inputSchemas); return copyEvaluatorFactory; } case CONSTANT: { ConstantExpression c = (ConstantExpression) expr; - copyEvaluatorFactory = createConstantEvaluatorFactory(c, inputSchemas, context); + copyEvaluatorFactory = createConstantEvaluatorFactory(c, context); return copyEvaluatorFactory; } case FUNCTION_CALL: { @@ -117,7 +120,7 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { } private IScalarEvaluatorFactory createVariableEvaluatorFactory(VariableReferenceExpression expr, - IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { + IOperatorSchema[] inputSchemas) throws AlgebricksException { LogicalVariable variable = expr.getVariableReference(); for (IOperatorSchema scm : inputSchemas) { int pos = scm.findVariable(variable); @@ -132,22 +135,17 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); - IFunctionDescriptor fd = null; - if (!(expr.getFunctionInfo() instanceof IExternalFunctionInfo)) { - IDataFormat format = FormatUtils.getDefaultFormat(); - fd = format.resolveFunction(expr, env); - } else { - ICcApplicationContext appCtx = (ICcApplicationContext) context.getAppContext(); - fd = ExternalFunctionDescriptorProvider - .getExternalFunctionDescriptor((IExternalFunctionInfo) expr.getFunctionInfo(), appCtx); - } + IFunctionDescriptor fd = expr.getFunctionInfo() instanceof IExternalFunctionInfo + ? ExternalFunctionDescriptorProvider.getExternalFunctionDescriptor( + (IExternalFunctionInfo) expr.getFunctionInfo(), (ICcApplicationContext) context.getAppContext()) + : resolveFunction(expr, env); return fd.createEvaluatorFactory(args); } - private IScalarEvaluatorFactory createConstantEvaluatorFactory(ConstantExpression expr, - IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { - IDataFormat format = FormatUtils.getDefaultFormat(); - return format.getConstantEvalFactory(expr.getValue()); + private IScalarEvaluatorFactory createConstantEvaluatorFactory(ConstantExpression expr, JobGenContext context) + throws AlgebricksException { + MetadataProvider metadataProvider = (MetadataProvider) context.getMetadataProvider(); + return metadataProvider.getDataFormat().getConstantEvalFactory(expr.getValue()); } private IScalarEvaluatorFactory[] codegenArguments(AbstractFunctionCallExpression expr, @@ -168,14 +166,14 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { AggregateFunctionCallExpression expr, IVariableTypeEnvironment env, IOperatorSchema[] inputSchemas, JobGenContext context) throws AlgebricksException { IScalarEvaluatorFactory[] args = codegenArguments(expr, env, inputSchemas, context); - IFunctionDescriptor fd = getFunctionDescriptor(expr, env, context); + IFunctionDescriptor fd = resolveFunction(expr, env); switch (fd.getFunctionDescriptorTag()) { case AGGREGATE: { if (BuiltinFunctions.isAggregateFunctionSerializable(fd.getIdentifier())) { AggregateFunctionCallExpression serialAggExpr = BuiltinFunctions .makeSerializableAggregateFunctionExpression(fd.getIdentifier(), expr.getArguments()); - IFunctionDescriptor afdd = getFunctionDescriptor(serialAggExpr, env, context); + IFunctionDescriptor afdd = resolveFunction(serialAggExpr, env); return afdd.createSerializableAggregateEvaluatorFactory(args); } else { throw new AlgebricksException( @@ -194,10 +192,14 @@ public class QueryLogicalExpressionJobGen implements ILogicalExpressionJobGen { } } - private IFunctionDescriptor getFunctionDescriptor(AbstractFunctionCallExpression expr, IVariableTypeEnvironment env, - JobGenContext context) throws AlgebricksException { - IFunctionDescriptor fd = FormatUtils.getDefaultFormat().resolveFunction(expr, env); + private IFunctionDescriptor resolveFunction(ILogicalExpression expr, IVariableTypeEnvironment env) + throws AlgebricksException { + FunctionIdentifier fnId = ((AbstractFunctionCallExpression) expr).getFunctionIdentifier(); + IFunctionDescriptor fd = functionManager.lookupFunction(fnId); + IFunctionTypeInferer fnTypeInfer = functionManager.lookupFunctionTypeInferer(fnId); + if (fnTypeInfer != null) { + fnTypeInfer.infer(expr, fd, env); + } return fd; } - } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java index 5ef41c4..f7695ea 100644 --- a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java +++ b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java @@ -36,9 +36,11 @@ import org.apache.asterix.formats.nontagged.BinaryIntegerInspector; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; import org.apache.asterix.formats.nontagged.TypeTraitProvider; import org.apache.asterix.jobgen.QueryLogicalExpressionJobGen; +import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.om.base.IAObject; import org.apache.asterix.om.constants.AsterixConstantValue; import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionExtensionManager; import org.apache.asterix.om.typecomputer.base.TypeCastUtils; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; @@ -125,12 +127,13 @@ public class ConstantFoldingRule implements IAlgebraicRewriteRule { private static final IOperatorSchema[] _emptySchemas = new IOperatorSchema[] {}; public ConstantFoldingRule(ICcApplicationContext appCtx) { - jobGenCtx = new JobGenContext(null, null, appCtx, SerializerDeserializerProvider.INSTANCE, + MetadataProvider metadataProvider = new MetadataProvider(appCtx, null); + jobGenCtx = new JobGenContext(null, metadataProvider, appCtx, SerializerDeserializerProvider.INSTANCE, BinaryHashFunctionFactoryProvider.INSTANCE, BinaryHashFunctionFamilyProvider.INSTANCE, BinaryComparatorFactoryProvider.INSTANCE, TypeTraitProvider.INSTANCE, BinaryBooleanInspector.FACTORY, BinaryIntegerInspector.FACTORY, ADMPrinterFactoryProvider.INSTANCE, MissingWriterFactory.INSTANCE, null, - new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE), ExpressionTypeComputer.INSTANCE, - null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null); + new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager())), + ExpressionTypeComputer.INSTANCE, null, null, null, null, GlobalConfig.DEFAULT_FRAME_SIZE, null); } @Override http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java index 8290446..2078288 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/api/common/APIFramework.java @@ -263,7 +263,7 @@ public class APIFramework { builder.setPhysicalOptimizationConfig(OptimizationConfUtil.getPhysicalOptimizationConfig()); builder.setLogicalRewrites(ruleSetFactory.getLogicalRewrites(metadataProvider.getApplicationContext())); builder.setPhysicalRewrites(ruleSetFactory.getPhysicalRewrites(metadataProvider.getApplicationContext())); - IDataFormat format = metadataProvider.getFormat(); + IDataFormat format = metadataProvider.getDataFormat(); ICompilerFactory compilerFactory = builder.create(); builder.setExpressionEvalSizeComputer(format.getExpressionEvalSizeComputer()); builder.setIMergeAggregationExpressionFactory(new MergeAggregationExpressionFactory()); @@ -321,7 +321,8 @@ public class APIFramework { builder.setBinaryBooleanInspectorFactory(format.getBinaryBooleanInspectorFactory()); builder.setBinaryIntegerInspectorFactory(format.getBinaryIntegerInspectorFactory()); builder.setComparatorFactoryProvider(format.getBinaryComparatorFactoryProvider()); - builder.setExpressionRuntimeProvider(new ExpressionRuntimeProvider(QueryLogicalExpressionJobGen.INSTANCE)); + builder.setExpressionRuntimeProvider( + new ExpressionRuntimeProvider(new QueryLogicalExpressionJobGen(metadataProvider.getFunctionManager()))); builder.setHashFunctionFactoryProvider(format.getBinaryHashFunctionFactoryProvider()); builder.setHashFunctionFamilyProvider(format.getBinaryHashFunctionFamilyProvider()); builder.setMissingWriterFactory(format.getMissingWriterFactory()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java index 0c6b2cc..768416d 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/cc/CCExtensionManager.java @@ -18,9 +18,9 @@ */ package org.apache.asterix.app.cc; -import java.util.HashMap; +import java.util.HashSet; import java.util.List; -import java.util.Map; +import java.util.Set; import java.util.concurrent.ExecutorService; import org.apache.asterix.algebra.base.ILangExtension; @@ -35,22 +35,25 @@ import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.compiler.provider.AqlCompilationProvider; import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.compiler.provider.SqlppCompilationProvider; +import org.apache.asterix.om.functions.IFunctionExtensionManager; +import org.apache.asterix.om.functions.IFunctionManager; +import org.apache.asterix.runtime.functions.FunctionCollection; +import org.apache.asterix.runtime.functions.FunctionManager; import org.apache.asterix.translator.IStatementExecutorFactory; import org.apache.asterix.utils.ExtensionUtil; import org.apache.hyracks.algebricks.common.utils.Pair; import org.apache.hyracks.api.exceptions.HyracksDataException; /** - * AsterixDB's implementation of {@code IAlgebraExtensionManager} which takes care of - * initializing extensions for App and Compilation purposes + * AsterixDB's implementation of {@code IAlgebraExtensionManager} and {@code IFunctionExtensionManager} + * which takes care of initializing extensions for App and Compilation purposes */ -public class CCExtensionManager implements IAlgebraExtensionManager { - - private final Map<ExtensionId, IExtension> extensions = new HashMap<>(); +public class CCExtensionManager implements IAlgebraExtensionManager, IFunctionExtensionManager { private final IStatementExecutorExtension statementExecutorExtension; private final ILangCompilationProvider aqlCompilationProvider; private final ILangCompilationProvider sqlppCompilationProvider; + private final IFunctionManager functionManager; private transient IStatementExecutorFactory statementExecutorFactory; /** @@ -67,15 +70,16 @@ public class CCExtensionManager implements IAlgebraExtensionManager { throws InstantiationException, IllegalAccessException, ClassNotFoundException, HyracksDataException { Pair<ExtensionId, ILangCompilationProvider> aqlcp = null; Pair<ExtensionId, ILangCompilationProvider> sqlppcp = null; + Pair<ExtensionId, IFunctionManager> fm = null; IStatementExecutorExtension see = null; if (list != null) { + Set<ExtensionId> extensionIds = new HashSet<>(); for (AsterixExtension extensionConf : list) { IExtension extension = (IExtension) Class.forName(extensionConf.getClassName()).newInstance(); extension.configure(extensionConf.getArgs()); - if (extensions.containsKey(extension.getId())) { + if (!extensionIds.add(extension.getId())) { throw new RuntimeDataException(ErrorCode.EXTENSION_ID_CONFLICT, extension.getId()); } - extensions.put(extension.getId(), extension); switch (extension.getExtensionKind()) { case STATEMENT_EXECUTOR: see = ExtensionUtil.extendStatementExecutor(see, (IStatementExecutorExtension) extension); @@ -84,6 +88,7 @@ public class CCExtensionManager implements IAlgebraExtensionManager { ILangExtension le = (ILangExtension) extension; aqlcp = ExtensionUtil.extendLangCompilationProvider(Language.AQL, aqlcp, le); sqlppcp = ExtensionUtil.extendLangCompilationProvider(Language.SQLPP, sqlppcp, le); + fm = ExtensionUtil.extendFunctionManager(fm, le); break; default: break; @@ -93,6 +98,8 @@ public class CCExtensionManager implements IAlgebraExtensionManager { this.statementExecutorExtension = see; this.aqlCompilationProvider = aqlcp == null ? new AqlCompilationProvider() : aqlcp.second; this.sqlppCompilationProvider = sqlppcp == null ? new SqlppCompilationProvider() : sqlppcp.second; + this.functionManager = + fm == null ? new FunctionManager(FunctionCollection.createDefaultFunctionCollection()) : fm.second; } /** @deprecated use getStatementExecutorFactory instead */ @@ -117,4 +124,9 @@ public class CCExtensionManager implements IAlgebraExtensionManager { default: throw new IllegalArgumentException(String.valueOf(lang)); } } + + @Override + public IFunctionManager getFunctionManager() { + return functionManager; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java index 07eed0d..380c5a9 100644 --- a/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java +++ b/asterixdb/asterix-app/src/main/java/org/apache/asterix/utils/ExtensionUtil.java @@ -26,6 +26,7 @@ import org.apache.asterix.common.exceptions.ErrorCode; import org.apache.asterix.common.exceptions.RuntimeDataException; import org.apache.asterix.compiler.provider.ILangCompilationProvider; import org.apache.asterix.metadata.api.IMetadataExtension; +import org.apache.asterix.om.functions.IFunctionManager; import org.apache.asterix.translator.IStatementExecutorFactory; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -52,18 +53,39 @@ public class ExtensionUtil { */ public static Pair<ExtensionId, ILangCompilationProvider> extendLangCompilationProvider(Language lang, Pair<ExtensionId, ILangCompilationProvider> cp, ILangExtension le) throws RuntimeDataException { - if (cp != null && le.getLangCompilationProvider(lang) != null) { + ILangCompilationProvider lecp = le.getLangCompilationProvider(lang); + if (cp != null && lecp != null) { throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, le.getId(), cp.first, lang.toString()); } - return (le.getLangCompilationProvider(lang) != null) - ? new Pair<>(le.getId(), le.getLangCompilationProvider(lang)) : cp; + return lecp != null ? new Pair<>(le.getId(), lecp) : cp; + } + + /** + * Validate no extension conflict and return function manager extension + * + * @param fm + * place holder for function manager extension + * @param le + * user defined extension + * @return the user defined extension + * @throws RuntimeDataException + * if extension conflict was detected + */ + public static Pair<ExtensionId, IFunctionManager> extendFunctionManager(Pair<ExtensionId, IFunctionManager> fm, + ILangExtension le) throws RuntimeDataException { + IFunctionManager lefm = le.getFunctionManager(); + if (fm != null && lefm != null) { + throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, le.getId(), fm.first, + IFunctionManager.class.getSimpleName()); + } + return lefm != null ? new Pair<>(le.getId(), lefm) : fm; } /** * Validate no extension conflict and return statement executor extension * - * @param qte + * @param see * place holder for statement executor extension * @param extension * user defined extension @@ -71,10 +93,10 @@ public class ExtensionUtil { * @throws RuntimeDataException * if extension conflict was detected */ - public static IStatementExecutorExtension extendStatementExecutor(IStatementExecutorExtension qte, + public static IStatementExecutorExtension extendStatementExecutor(IStatementExecutorExtension see, IStatementExecutorExtension extension) throws RuntimeDataException { - if (qte != null) { - throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, qte.getId(), extension.getId(), + if (see != null) { + throw new RuntimeDataException(ErrorCode.EXTENSION_COMPONENT_CONFLICT, see.getId(), extension.getId(), IStatementExecutorFactory.class.getSimpleName()); } return extension; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java index 3c54c7c..4bcee04 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/ExceptionIT.java @@ -43,7 +43,8 @@ public class ExceptionIT { @Test public void test() throws Exception { - List<IFunctionDescriptorFactory> functions = FunctionCollection.getFunctionDescriptorFactories(); + List<IFunctionDescriptorFactory> functions = + FunctionCollection.createDefaultFunctionCollection().getFunctionDescriptorFactories(); int testedFunctions = 0; for (IFunctionDescriptorFactory func : functions) { String className = func.getClass().getName(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java index f5e7402..3b33868 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/runtime/NullMissingTest.java @@ -43,7 +43,8 @@ public class NullMissingTest { @Test public void test() throws Exception { - List<IFunctionDescriptorFactory> functions = FunctionCollection.getFunctionDescriptorFactories(); + List<IFunctionDescriptorFactory> functions = + FunctionCollection.createDefaultFunctionCollection().getFunctionDescriptorFactories(); int testedFunctions = 0; for (IFunctionDescriptorFactory func : functions) { String className = func.getClass().getName(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java index b1191ec..40d4f6a 100644 --- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java +++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/active/ActiveEventsListenerTest.java @@ -45,6 +45,9 @@ import org.apache.asterix.metadata.declared.MetadataProvider; import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Feed; import org.apache.asterix.metadata.lock.MetadataLockManager; +import org.apache.asterix.om.functions.IFunctionExtensionManager; +import org.apache.asterix.runtime.functions.FunctionCollection; +import org.apache.asterix.runtime.functions.FunctionManager; import org.apache.asterix.runtime.utils.CcApplicationContext; import org.apache.asterix.test.active.TestEventsListener.Behavior; import org.apache.asterix.test.base.TestMethodTracer; @@ -82,6 +85,7 @@ public class ActiveEventsListenerTest { static CcApplicationContext appCtx; static IStatementExecutor statementExecutor; static IHyracksClientConnection hcc; + static IFunctionExtensionManager functionExtensionManager; static MetadataProvider metadataProvider; static IStorageComponentProvider componentProvider; static JobIdFactory jobIdFactory; @@ -121,6 +125,10 @@ public class ActiveEventsListenerTest { Mockito.when(ccServiceCtx.getControllerService()).thenReturn(ccService); Mockito.when(ccService.getExecutor()).thenReturn(executor); locations = new AlgebricksAbsolutePartitionConstraint(nodes); + functionExtensionManager = Mockito.mock(IFunctionExtensionManager.class); + Mockito.when(functionExtensionManager.getFunctionManager()) + .thenReturn(new FunctionManager(FunctionCollection.createDefaultFunctionCollection())); + Mockito.when(appCtx.getExtensionManager()).thenReturn(functionExtensionManager); metadataProvider = new MetadataProvider(appCtx, null); clusterController = new TestClusterControllerActor("CC", handler, allDatasets); nodeControllers = new TestNodeControllerActor[2]; @@ -1449,6 +1457,10 @@ public class ActiveEventsListenerTest { CcApplicationContext ccAppCtx = Mockito.mock(CcApplicationContext.class); IStatementExecutor statementExecutor = Mockito.mock(IStatementExecutor.class); IHyracksClientConnection hcc = Mockito.mock(IHyracksClientConnection.class); + IFunctionExtensionManager functionExtensionManager = Mockito.mock(IFunctionExtensionManager.class); + Mockito.when(functionExtensionManager.getFunctionManager()) + .thenReturn(new FunctionManager(FunctionCollection.createDefaultFunctionCollection())); + Mockito.when(ccAppCtx.getExtensionManager()).thenReturn(functionExtensionManager); Mockito.when(ccAppCtx.getActiveNotificationHandler()).thenReturn(handler); Mockito.when(ccAppCtx.getMetadataLockManager()).thenReturn(lockManager); Mockito.when(ccAppCtx.getServiceContext()).thenReturn(ccServiceCtx); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java index e2b1761..3f3a27f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/DatasetDataSource.java @@ -31,7 +31,6 @@ import org.apache.asterix.metadata.entities.InternalDatasetDetails; import org.apache.asterix.metadata.utils.KeyFieldTypeUtil; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.runtime.formats.NonTaggedDataFormat; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Pair; @@ -107,8 +106,7 @@ public class DatasetDataSource extends DataSource { ExternalDatasetDetails edd = (ExternalDatasetDetails) externalDataset.getDatasetDetails(); IAdapterFactory adapterFactory = metadataProvider.getConfiguredAdapterFactory(externalDataset, edd.getAdapter(), edd.getProperties(), (ARecordType) itemType, null); - return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory, - NonTaggedDataFormat.INSTANCE); + return metadataProvider.buildExternalDatasetDataScannerRuntime(jobSpec, itemType, adapterFactory); case INTERNAL: DataSourceId id = getId(); String dataverseName = id.getDataverseName(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java index 0b6608c..325d23b 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/FeedDataSource.java @@ -33,7 +33,6 @@ import org.apache.asterix.metadata.entities.FeedPolicyEntity; import org.apache.asterix.metadata.feeds.BuiltinFeedPolicies; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.runtime.formats.NonTaggedDataFormat; import org.apache.hyracks.algebricks.common.constraints.AlgebricksAbsolutePartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -170,7 +169,7 @@ public class FeedDataSource extends DataSource implements IMutationDataSource { try { ARecordType feedOutputType = (ARecordType) itemType; ISerializerDeserializer payloadSerde = - NonTaggedDataFormat.INSTANCE.getSerdeProvider().getSerializerDeserializer(feedOutputType); + metadataProvider.getDataFormat().getSerdeProvider().getSerializerDeserializer(feedOutputType); ArrayList<ISerializerDeserializer> serdes = new ArrayList<>(); serdes.add(payloadSerde); if (metaItemType != null) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java index 9271f33..2f94ad7 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/declared/MetadataProvider.java @@ -57,6 +57,7 @@ import org.apache.asterix.external.provider.AdapterFactoryProvider; import org.apache.asterix.external.util.ExternalDataConstants; import org.apache.asterix.external.util.FeedConstants; import org.apache.asterix.formats.base.IDataFormat; +import org.apache.asterix.om.functions.IFunctionExtensionManager; import org.apache.asterix.formats.nontagged.BinaryComparatorFactoryProvider; import org.apache.asterix.formats.nontagged.LinearizeComparatorFactoryProvider; import org.apache.asterix.formats.nontagged.TypeTraitProvider; @@ -78,6 +79,7 @@ import org.apache.asterix.metadata.utils.DatasetUtil; import org.apache.asterix.metadata.utils.MetadataConstants; import org.apache.asterix.metadata.utils.SplitsAndConstraintsUtil; import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionManager; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.ATypeTag; import org.apache.asterix.om.types.IAType; @@ -143,6 +145,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> private final ICcApplicationContext appCtx; private final IStorageComponentProvider storageComponentProvider; private final StorageProperties storageProperties; + private final IFunctionManager functionManager; private final Dataverse defaultDataverse; private final LockList locks; private final Map<String, String> config; @@ -164,6 +167,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> this.defaultDataverse = defaultDataverse == null ? MetadataBuiltinEntities.DEFAULT_DATAVERSE : defaultDataverse; this.storageComponentProvider = appCtx.getStorageComponentProvider(); storageProperties = appCtx.getStorageProperties(); + functionManager = ((IFunctionExtensionManager) appCtx.getExtensionManager()).getFunctionManager(); locks = new LockList(); config = new HashMap<>(); } @@ -259,7 +263,11 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> return isTemporaryDatasetWriteJob; } - public IDataFormat getFormat() { + public IFunctionManager getFunctionManager() { + return functionManager; + } + + public IDataFormat getDataFormat() { return FormatUtils.getDefaultFormat(); } @@ -868,13 +876,14 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> } public Pair<IOperatorDescriptor, AlgebricksPartitionConstraint> buildExternalDatasetDataScannerRuntime( - JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory, IDataFormat format) + JobSpecification jobSpec, IAType itemType, IAdapterFactory adapterFactory) throws AlgebricksException { if (itemType.getTypeTag() != ATypeTag.OBJECT) { throw new AlgebricksException("Can only scan datasets of records."); } - ISerializerDeserializer<?> payloadSerde = format.getSerdeProvider().getSerializerDeserializer(itemType); + ISerializerDeserializer<?> payloadSerde = + getDataFormat().getSerdeProvider().getSerializerDeserializer(itemType); RecordDescriptor scannerDesc = new RecordDescriptor(new ISerializerDeserializer[] { payloadSerde }); ExternalScanOperatorDescriptor dataScanner = @@ -1501,7 +1510,7 @@ public class MetadataProvider implements IMetadataProvider<DataSourceId, String> // Generate Output Record format ISerializerDeserializer<?>[] tokenKeyPairFields = new ISerializerDeserializer[numTokenKeyPairFields]; ITypeTraits[] tokenKeyPairTypeTraits = new ITypeTraits[numTokenKeyPairFields]; - ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider(); + ISerializerDeserializerProvider serdeProvider = getDataFormat().getSerdeProvider(); // The order of the output record: propagated variables (including // PK and SK), token, and number of token. http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java index 9156b0f..8e1c34d 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Dataset.java @@ -730,7 +730,7 @@ public class Dataset implements IMetadataEntity<Dataset>, IDataset { ISerializerDeserializer[] primaryRecFields = new ISerializerDeserializer[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)]; ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (hasMetaPart() ? 1 : 0)]; - ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider(); + ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); List<Integer> indicators = null; if (hasMetaPart()) { indicators = ((InternalDatasetDetails) getDatasetDetails()).getKeySourceIndicator(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java index e4a6ca8..53c3dc0 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/DatasetUtil.java @@ -41,6 +41,7 @@ import org.apache.asterix.common.exceptions.MetadataException; import org.apache.asterix.common.transactions.IRecoveryManager; import org.apache.asterix.common.transactions.JobId; import org.apache.asterix.external.indexing.IndexingConstants; +import org.apache.asterix.formats.base.IDataFormat; import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; import org.apache.asterix.formats.nontagged.TypeTraitProvider; import org.apache.asterix.metadata.MetadataManager; @@ -57,7 +58,6 @@ import org.apache.asterix.om.base.AString; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.BuiltinType; import org.apache.asterix.om.types.IAType; -import org.apache.asterix.runtime.formats.FormatUtils; import org.apache.asterix.runtime.job.listener.JobEventListenerFactory; import org.apache.asterix.runtime.operators.LSMPrimaryUpsertOperatorDescriptor; import org.apache.asterix.runtime.utils.RuntimeUtils; @@ -419,16 +419,16 @@ public class DatasetUtil { + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields]; ISerializerDeserializer<?>[] outputSerDes = new ISerializerDeserializer[inputRecordDesc.getFieldCount() + (dataset.hasMetaPart() ? 2 : 1) + numFilterFields]; + IDataFormat dataFormat = metadataProvider.getDataFormat(); // add the previous record first int f = 0; - outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(itemType); + outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(itemType); f++; // add the previous meta second if (dataset.hasMetaPart()) { - outputSerDes[f] = - FormatUtils.getDefaultFormat().getSerdeProvider().getSerializerDeserializer(metaItemType); - outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider().getTypeTrait(metaItemType); + outputSerDes[f] = dataFormat.getSerdeProvider().getSerializerDeserializer(metaItemType); + outputTypeTraits[f] = dataFormat.getTypeTraitProvider().getTypeTrait(metaItemType); f++; } // add the previous filter third @@ -443,10 +443,10 @@ public class DatasetUtil { } } fieldIdx = i; - outputTypeTraits[f] = FormatUtils.getDefaultFormat().getTypeTraitProvider() - .getTypeTrait(itemType.getFieldTypes()[fieldIdx]); - outputSerDes[f] = FormatUtils.getDefaultFormat().getSerdeProvider() - .getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]); + outputTypeTraits[f] = + dataFormat.getTypeTraitProvider().getTypeTrait(itemType.getFieldTypes()[fieldIdx]); + outputSerDes[f] = + dataFormat.getSerdeProvider().getSerializerDeserializer(itemType.getFieldTypes()[fieldIdx]); f++; } for (int j = 0; j < inputRecordDesc.getFieldCount(); j++) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java index 445fbd8..78d9c19 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryBTreeOperationsHelper.java @@ -212,10 +212,10 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields]; secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys]; - ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider(); - ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider(); + ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); + ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider(); IBinaryComparatorFactoryProvider comparatorFactoryProvider = - metadataProvider.getFormat().getBinaryComparatorFactoryProvider(); + metadataProvider.getDataFormat().getBinaryComparatorFactoryProvider(); // Record column is 0 for external datasets, numPrimaryKeys for internal ones int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0; boolean isOverridingKeyFieldTypes = index.isOverridingKeyFieldTypes(); @@ -230,9 +230,9 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations sourceType = metaType; sourceColumn = recordColumn + 1; } - secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory( - isOverridingKeyFieldTypes ? enforcedItemType : sourceType, index.getKeyFieldNames().get(i), - sourceColumn); + secondaryFieldAccessEvalFactories[i] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( + metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : sourceType, + index.getKeyFieldNames().get(i), sourceColumn); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i), sourceType); IAType keyType = keyTypePair.first; @@ -270,8 +270,9 @@ public class SecondaryBTreeOperationsHelper extends SecondaryTreeIndexOperations } if (numFilterFields > 0) { - secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getFormat() - .getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys); + secondaryFieldAccessEvalFactories[numSecondaryKeys] = + metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( + metadataProvider.getFunctionManager(), itemType, filterFieldName, numPrimaryKeys); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java index 96fdf27..8ef5f34 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedBTreeOperationsHelper.java @@ -137,10 +137,10 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys + (dataset.hasMetaPart() ? 1 : 0) + numFilterFields]; secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys]; - ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider(); - ITypeTraitProvider typeTraitProvider = metadataProvider.getFormat().getTypeTraitProvider(); + ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); + ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider(); IBinaryComparatorFactoryProvider comparatorFactoryProvider = - metadataProvider.getFormat().getBinaryComparatorFactoryProvider(); + metadataProvider.getDataFormat().getBinaryComparatorFactoryProvider(); // Record column is 0 for external datasets, numPrimaryKeys for internal ones int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys; boolean isOverridingKeyTypes = index.isOverridingKeyFieldTypes(); @@ -155,9 +155,9 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate sourceType = metaType; sourceColumn = recordColumn + 1; } - secondaryFieldAccessEvalFactories[i] = metadataProvider.getFormat().getFieldAccessEvaluatorFactory( - isOverridingKeyTypes ? enforcedItemType : sourceType, index.getKeyFieldNames().get(i), - sourceColumn); + secondaryFieldAccessEvalFactories[i] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( + metadataProvider.getFunctionManager(), isOverridingKeyTypes ? enforcedItemType : sourceType, + index.getKeyFieldNames().get(i), sourceColumn); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(i), index.getKeyFieldNames().get(i), sourceType); IAType keyType = keyTypePair.first; @@ -185,8 +185,9 @@ public class SecondaryCorrelatedBTreeOperationsHelper extends SecondaryCorrelate } if (numFilterFields > 0) { - secondaryFieldAccessEvalFactories[numSecondaryKeys] = metadataProvider.getFormat() - .getFieldAccessEvaluatorFactory(itemType, filterFieldName, recordColumn); + secondaryFieldAccessEvalFactories[numSecondaryKeys] = + metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( + metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java index 15f8a23..262b259 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedInvertedIndexOperationsHelper.java @@ -29,7 +29,6 @@ import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.asterix.om.utils.RecordUtil; -import org.apache.asterix.runtime.formats.FormatUtils; import org.apache.asterix.runtime.utils.RuntimeUtils; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -108,13 +107,13 @@ public class SecondaryCorrelatedInvertedIndexOperationsHelper extends SecondaryC ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields]; secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys]; ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys]; - ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider(); - ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider(); + ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); + ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider(); int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys; if (numSecondaryKeys > 0) { - secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory( - isOverridingKeyFieldTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0), - recordColumn); + secondaryFieldAccessEvalFactories[0] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( + metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType, + index.getKeyFieldNames().get(0), recordColumn); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), itemType); secondaryKeyType = keyTypePair.first; @@ -124,8 +123,9 @@ public class SecondaryCorrelatedInvertedIndexOperationsHelper extends SecondaryC secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType); } if (numFilterFields > 0) { - secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat() - .getFieldAccessEvaluatorFactory(itemType, filterFieldName, recordColumn); + secondaryFieldAccessEvalFactories[numSecondaryKeys] = + metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( + metadataProvider.getFunctionManager(), itemType, filterFieldName, recordColumn); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java index 8fd8a7a..b663b42 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedRTreeOperationsHelper.java @@ -89,9 +89,9 @@ public class SecondaryCorrelatedRTreeOperationsHelper extends SecondaryCorrelate int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag()); numNestedSecondaryKeyFields = numDimensions * 2; int recordColumn = NUM_TAG_FIELDS + numPrimaryKeys; - secondaryFieldAccessEvalFactories = - metadataProvider.getFormat().createMBRFactory(isOverridingKeyFieldTypes ? enforcedItemType : itemType, - secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR); + secondaryFieldAccessEvalFactories = metadataProvider.getDataFormat().createMBRFactory( + metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType, + secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR); secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields]; valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields]; ISerializerDeserializer[] secondaryRecFields = http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java index 171d72e..b99ae2f 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryCorrelatedTreeIndexOperationsHelper.java @@ -32,7 +32,6 @@ import org.apache.asterix.metadata.entities.Dataset; import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.functions.IFunctionDescriptor; -import org.apache.asterix.runtime.functions.FunctionManagerHolder; import org.apache.asterix.runtime.operators.LSMSecondaryIndexBulkLoadOperatorDescriptor; import org.apache.asterix.runtime.operators.LSMSecondaryIndexCreationTupleProcessorOperatorDescriptor; import org.apache.asterix.transaction.management.opcallbacks.PrimaryIndexInstantSearchOperationCallbackFactory; @@ -155,10 +154,6 @@ public abstract class SecondaryCorrelatedTreeIndexOperationsHelper extends Secon @Override protected AlgebricksMetaOperatorDescriptor createCastOp(JobSpecification spec, DatasetType dsType, boolean strictCast) throws AlgebricksException { - IFunctionDescriptor castFuncDesc = FunctionManagerHolder.getFunctionManager() - .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX); - castFuncDesc.setImmutableStates(enforcedItemType, itemType); - int[] outColumns = new int[1]; // tags(2) + primary keys + record + meta part(?) @@ -185,7 +180,7 @@ public abstract class SecondaryCorrelatedTreeIndexOperationsHelper extends Secon IScalarEvaluatorFactory[] castEvalFact = new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) }; IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1]; - sefs[0] = castFuncDesc.createEvaluatorFactory(castEvalFact); + sefs[0] = createCastFunction(strictCast).createEvaluatorFactory(castEvalFact); AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList); return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign }, new RecordDescriptor[] { getTaggedRecordDescriptor(enforcedRecDesc) }); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java index 98c47a2..5dac407 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryIndexOperationsHelper.java @@ -22,7 +22,6 @@ package org.apache.asterix.metadata.utils; import static org.apache.hyracks.storage.am.common.dataflow.IndexDropOperatorDescriptor.DropOption; import java.util.Collections; -import java.util.EnumSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -52,7 +51,6 @@ import org.apache.asterix.om.types.IAType; import org.apache.asterix.runtime.evaluators.functions.AndDescriptor; import org.apache.asterix.runtime.evaluators.functions.IsUnknownDescriptor; import org.apache.asterix.runtime.evaluators.functions.NotDescriptor; -import org.apache.asterix.runtime.functions.FunctionManagerHolder; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraint; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -239,7 +237,7 @@ public abstract class SecondaryIndexOperationsHelper { ITypeTraits[] primaryTypeTraits = new ITypeTraits[numPrimaryKeys + 1 + (dataset.hasMetaPart() ? 1 : 0)]; primaryComparatorFactories = new IBinaryComparatorFactory[numPrimaryKeys]; primaryBloomFilterKeyFields = new int[numPrimaryKeys]; - ISerializerDeserializerProvider serdeProvider = metadataProvider.getFormat().getSerdeProvider(); + ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); List<Integer> indicators = null; if (dataset.hasMetaPart()) { indicators = ((InternalDatasetDetails) dataset.getDatasetDetails()).getKeySourceIndicator(); @@ -298,10 +296,6 @@ public abstract class SecondaryIndexOperationsHelper { protected AlgebricksMetaOperatorDescriptor createCastOp(JobSpecification spec, DatasetType dsType, boolean strictCast) throws AlgebricksException { - IFunctionDescriptor castFuncDesc = FunctionManagerHolder.getFunctionManager() - .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX); - castFuncDesc.setImmutableStates(enforcedItemType, itemType); - int[] outColumns = new int[1]; int[] projectionList = new int[(dataset.hasMetaPart() ? 2 : 1) + numPrimaryKeys]; int recordIdx; @@ -322,12 +316,19 @@ public abstract class SecondaryIndexOperationsHelper { IScalarEvaluatorFactory[] castEvalFact = new IScalarEvaluatorFactory[] { new ColumnAccessEvalFactory(recordIdx) }; IScalarEvaluatorFactory[] sefs = new IScalarEvaluatorFactory[1]; - sefs[0] = castFuncDesc.createEvaluatorFactory(castEvalFact); + sefs[0] = createCastFunction(strictCast).createEvaluatorFactory(castEvalFact); AssignRuntimeFactory castAssign = new AssignRuntimeFactory(outColumns, sefs, projectionList); return new AlgebricksMetaOperatorDescriptor(spec, 1, 1, new IPushRuntimeFactory[] { castAssign }, new RecordDescriptor[] { enforcedRecDesc }); } + protected IFunctionDescriptor createCastFunction(boolean strictCast) throws AlgebricksException { + IFunctionDescriptor castFuncDesc = metadataProvider.getFunctionManager() + .lookupFunction(strictCast ? BuiltinFunctions.CAST_TYPE : BuiltinFunctions.CAST_TYPE_LAX); + castFuncDesc.setImmutableStates(enforcedItemType, itemType); + return castFuncDesc; + } + protected ExternalSortOperatorDescriptor createSortOp(JobSpecification spec, IBinaryComparatorFactory[] secondaryComparatorFactories, RecordDescriptor secondaryRecDesc) { int[] sortFields = new int[secondaryComparatorFactories.length]; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java index b4d8a22..077e431 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryInvertedIndexOperationsHelper.java @@ -30,7 +30,6 @@ import org.apache.asterix.metadata.entities.Index; import org.apache.asterix.om.types.IAType; import org.apache.asterix.om.utils.NonTaggedFormatUtil; import org.apache.asterix.om.utils.RecordUtil; -import org.apache.asterix.runtime.formats.FormatUtils; import org.apache.asterix.runtime.utils.RuntimeUtils; import org.apache.hyracks.algebricks.common.constraints.AlgebricksPartitionConstraintHelper; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; @@ -111,12 +110,12 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryTreeIndexOp ISerializerDeserializer[] enforcedRecFields = new ISerializerDeserializer[1 + numPrimaryKeys + numFilterFields]; secondaryTypeTraits = new ITypeTraits[numSecondaryKeys + numPrimaryKeys]; ITypeTraits[] enforcedTypeTraits = new ITypeTraits[1 + numPrimaryKeys]; - ISerializerDeserializerProvider serdeProvider = FormatUtils.getDefaultFormat().getSerdeProvider(); - ITypeTraitProvider typeTraitProvider = FormatUtils.getDefaultFormat().getTypeTraitProvider(); + ISerializerDeserializerProvider serdeProvider = metadataProvider.getDataFormat().getSerdeProvider(); + ITypeTraitProvider typeTraitProvider = metadataProvider.getDataFormat().getTypeTraitProvider(); if (numSecondaryKeys > 0) { - secondaryFieldAccessEvalFactories[0] = FormatUtils.getDefaultFormat().getFieldAccessEvaluatorFactory( - isOverridingKeyFieldTypes ? enforcedItemType : itemType, index.getKeyFieldNames().get(0), - numPrimaryKeys); + secondaryFieldAccessEvalFactories[0] = metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( + metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType, + index.getKeyFieldNames().get(0), numPrimaryKeys); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableOpenFieldType(index.getKeyFieldTypes().get(0), index.getKeyFieldNames().get(0), itemType); secondaryKeyType = keyTypePair.first; @@ -126,8 +125,9 @@ public class SecondaryInvertedIndexOperationsHelper extends SecondaryTreeIndexOp secondaryTypeTraits[0] = typeTraitProvider.getTypeTrait(secondaryKeyType); } if (numFilterFields > 0) { - secondaryFieldAccessEvalFactories[numSecondaryKeys] = FormatUtils.getDefaultFormat() - .getFieldAccessEvaluatorFactory(itemType, filterFieldName, numPrimaryKeys); + secondaryFieldAccessEvalFactories[numSecondaryKeys] = + metadataProvider.getDataFormat().getFieldAccessEvaluatorFactory( + metadataProvider.getFunctionManager(), itemType, filterFieldName, numPrimaryKeys); Pair<IAType, Boolean> keyTypePair = Index.getNonNullableKeyFieldType(filterFieldName, itemType); IAType type = keyTypePair.first; ISerializerDeserializer serde = serdeProvider.getSerializerDeserializer(type); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java index 08ee0aa..1726470 100644 --- a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java +++ b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/utils/SecondaryRTreeOperationsHelper.java @@ -99,9 +99,9 @@ public class SecondaryRTreeOperationsHelper extends SecondaryTreeIndexOperations int numDimensions = NonTaggedFormatUtil.getNumDimensions(spatialType.getTypeTag()); numNestedSecondaryKeyFields = numDimensions * 2; int recordColumn = dataset.getDatasetType() == DatasetType.INTERNAL ? numPrimaryKeys : 0; - secondaryFieldAccessEvalFactories = - metadataProvider.getFormat().createMBRFactory(isOverridingKeyFieldTypes ? enforcedItemType : itemType, - secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR); + secondaryFieldAccessEvalFactories = metadataProvider.getDataFormat().createMBRFactory( + metadataProvider.getFunctionManager(), isOverridingKeyFieldTypes ? enforcedItemType : itemType, + secondaryKeyFields.get(0), recordColumn, numDimensions, filterFieldName, isPointMBR); secondaryComparatorFactories = new IBinaryComparatorFactory[numNestedSecondaryKeyFields]; valueProviderFactories = new IPrimitiveValueProviderFactory[numNestedSecondaryKeyFields]; ISerializerDeserializer[] secondaryRecFields = http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java index 7624a8a..678a864 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/formats/base/IDataFormat.java @@ -20,16 +20,13 @@ package org.apache.asterix.formats.base; import java.util.List; -import org.apache.asterix.om.functions.IFunctionDescriptor; -import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.om.functions.IFunctionManager; import org.apache.asterix.om.types.ARecordType; import org.apache.asterix.om.types.IAType; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.common.utils.Triple; -import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; import org.apache.hyracks.algebricks.core.algebra.expressions.IAlgebricksConstantValue; import org.apache.hyracks.algebricks.core.algebra.expressions.IExpressionEvalSizeComputer; -import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; import org.apache.hyracks.algebricks.core.algebra.expressions.ScalarFunctionCallExpression; import org.apache.hyracks.algebricks.data.IBinaryBooleanInspectorFactory; import org.apache.hyracks.algebricks.data.IBinaryComparatorFactoryProvider; @@ -69,18 +66,16 @@ public interface IDataFormat { public IMissingWriterFactory getMissingWriterFactory(); public Triple<IScalarEvaluatorFactory, ScalarFunctionCallExpression, IAType> partitioningEvaluatorFactory( - ARecordType recType, List<String> fldName) throws AlgebricksException; + IFunctionManager functionManager, ARecordType recType, List<String> fldName) throws AlgebricksException; - public IScalarEvaluatorFactory getFieldAccessEvaluatorFactory(ARecordType recType, List<String> fldName, - int recordColumn) throws AlgebricksException; - - public IFunctionDescriptor resolveFunction(ILogicalExpression expr, IVariableTypeEnvironment typeEnvironment) - throws AlgebricksException; + public IScalarEvaluatorFactory getFieldAccessEvaluatorFactory(IFunctionManager functionManager, ARecordType recType, + List<String> fldName, int recordColumn) throws AlgebricksException; public IScalarEvaluatorFactory getConstantEvalFactory(IAlgebricksConstantValue value) throws AlgebricksException; - public IScalarEvaluatorFactory[] createMBRFactory(ARecordType recType, List<String> fldName, int recordColumn, - int dimension, List<String> filterFieldName, boolean isPointMBR) throws AlgebricksException; + public IScalarEvaluatorFactory[] createMBRFactory(IFunctionManager functionManager, ARecordType recType, + List<String> fldName, int recordColumn, int dimension, List<String> filterFieldName, boolean isPointMBR) + throws AlgebricksException; public IExpressionEvalSizeComputer getExpressionEvalSizeComputer(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java index 526eab6..0e746f9 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionDescriptorFactory.java @@ -26,5 +26,13 @@ public interface IFunctionDescriptorFactory { * * @return a new IFunctionDescriptor instance */ - public IFunctionDescriptor createFunctionDescriptor(); + IFunctionDescriptor createFunctionDescriptor(); + + /** + * Creates type inferer for this function + * @return an new instance of the type inferer or {@code null} if not provided + */ + default IFunctionTypeInferer createFunctionTypeInferer() { + return null; + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java new file mode 100644 index 0000000..fb8155a --- /dev/null +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionExtensionManager.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.om.functions; + +/** + * An extension point for built-in functions + */ +public interface IFunctionExtensionManager { + /** + * Returns function manager + * + * @return a new {@link IFunctionManager} instance + */ + IFunctionManager getFunctionManager(); +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java index f7c11ff..e1657ff 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionManager.java @@ -21,11 +21,13 @@ package org.apache.asterix.om.functions; import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; -public interface IFunctionManager extends Iterable<IFunctionDescriptorFactory> { - - public void registerFunction(IFunctionDescriptorFactory descriptorFactory); +/** + * A registry of {@link IFunctionDescriptor} and {@link IFunctionTypeInferer} + * for built-in functions + */ +public interface IFunctionManager { - public void unregisterFunction(IFunctionDescriptorFactory descriptorFactory); + IFunctionDescriptor lookupFunction(FunctionIdentifier fid) throws AlgebricksException; - public IFunctionDescriptor lookupFunction(FunctionIdentifier fid) throws AlgebricksException; + IFunctionTypeInferer lookupFunctionTypeInferer(FunctionIdentifier fid); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java new file mode 100644 index 0000000..b8e27e1 --- /dev/null +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IFunctionTypeInferer.java @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.om.functions; + +import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException; +import org.apache.hyracks.algebricks.core.algebra.base.ILogicalExpression; +import org.apache.hyracks.algebricks.core.algebra.expressions.IVariableTypeEnvironment; + +public interface IFunctionTypeInferer { + void infer(ILogicalExpression expr, IFunctionDescriptor fd, IVariableTypeEnvironment context) + throws AlgebricksException; +} http://git-wip-us.apache.org/repos/asf/asterixdb/blob/76d24d49/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java ---------------------------------------------------------------------- diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java index f117d14..4e35950 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/aggregates/collections/ListifyAggregateDescriptor.java @@ -21,22 +21,29 @@ package org.apache.asterix.runtime.aggregates.collections; import org.apache.asterix.om.functions.BuiltinFunctions; import org.apache.asterix.om.functions.IFunctionDescriptor; import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.om.functions.IFunctionTypeInferer; import org.apache.asterix.om.types.AOrderedListType; import org.apache.asterix.runtime.aggregates.base.AbstractAggregateFunctionDynamicDescriptor; +import org.apache.asterix.runtime.functions.FunctionTypeInferers; import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; import org.apache.hyracks.algebricks.runtime.base.IAggregateEvaluatorFactory; import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; public class ListifyAggregateDescriptor extends AbstractAggregateFunctionDynamicDescriptor { - private static final long serialVersionUID = 1L; public static final IFunctionDescriptorFactory FACTORY = new IFunctionDescriptorFactory() { @Override public IFunctionDescriptor createFunctionDescriptor() { return new ListifyAggregateDescriptor(); } + + @Override + public IFunctionTypeInferer createFunctionTypeInferer() { + return FunctionTypeInferers.SET_EXPRESSION_TYPE; + } }; + private static final long serialVersionUID = 1L; private AOrderedListType oltype; @Override @@ -53,5 +60,4 @@ public class ListifyAggregateDescriptor extends AbstractAggregateFunctionDynamic public IAggregateEvaluatorFactory createAggregateEvaluatorFactory(final IScalarEvaluatorFactory[] args) { return new ListifyAggregateFunctionEvalFactory(args, oltype); } - }
