This is an automated email from the ASF dual-hosted git repository.
dlych pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new 74fac08 [NO ISSUE][COMP][RT] External function improvements
74fac08 is described below
commit 74fac0841f8839e466b0791646dfbdeef78591c5
Author: Dmitry Lychagin <[email protected]>
AuthorDate: Thu Mar 5 11:28:18 2020 -0800
[NO ISSUE][COMP][RT] External function improvements
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Support multipart external identifiers in CREATE FUNCTION
- Add preliminary code for Python UDFs
- Some refactoring in external function framework
- Add method to get IServiceContext from IEvaluatorContext
Change-Id: I7ec91f5be2efa8409cda3a3c13f5e8b4de3e75e8
Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/5244
Integration-Tests: Jenkins <[email protected]>
Tested-by: Jenkins <[email protected]>
Reviewed-by: Ian Maxon <[email protected]>
---
.../jobgen/QueryLogicalExpressionJobGen.java | 5 +-
.../optimizer/rules/ConstantFoldingRule.java | 32 ++++--
.../translator/AqlExpressionToPlanTranslator.java | 6 --
.../translator/LangExpressionToPlanTranslator.java | 9 +-
.../SqlppExpressionToPlanTranslator.java | 6 --
.../asterix/app/translator/QueryTranslator.java | 30 ++----
.../ExternalFunctionDescriptorProvider.java | 56 ++--------
.../external/library/ExternalFunctionProvider.java | 90 ----------------
....java => ExternalScalarFunctionDescriptor.java} | 42 ++------
...y.java => ExternalScalarFunctionEvaluator.java} | 36 +++----
.../ExternalScalarFunctionEvaluatorFactory.java | 20 ++--
...va => ExternalScalarJavaFunctionEvaluator.java} | 79 +++++++-------
.../ExternalScalarPythonFunctionEvaluator.java | 114 +++++++++++++++++++++
.../asterix/lang/aql/parser/AQLParserFactory.java | 7 ++
.../lang/aql/rewrites/AqlQueryRewriter.java | 3 +-
.../asterix/lang/common/base/IParserFactory.java | 5 +-
.../asterix/lang/common/parser/FunctionParser.java | 13 +--
.../common/statement/CreateFunctionStatement.java | 6 +-
.../asterix/lang/common/util/FunctionUtil.java | 2 +-
.../lang/sqlpp/parser/SqlppParserFactory.java | 8 ++
.../lang/sqlpp/rewrites/SqlppQueryRewriter.java | 3 +-
.../asterix-lang-sqlpp/src/main/javacc/SQLPP.jj | 7 +-
.../apache/asterix/metadata/entities/Function.java | 18 ++--
.../FunctionTupleTranslator.java | 13 +--
.../functions/ExternalFunctionCompilerUtil.java | 95 ++++++++++++++++-
.../functions/ExternalScalarFunctionInfo.java | 20 ++--
.../ExternalFunctionCompilerUtilTest.java | 3 +-
.../asterix/om/functions/ExternalFunctionInfo.java | 33 +++---
.../om/functions/ExternalFunctionLanguage.java} | 8 +-
.../om/functions/IExternalFunctionInfo.java | 4 +-
.../algebricks/runtime/base/IEvaluatorContext.java | 7 ++
.../runtime/evaluators/EvaluatorContext.java | 19 +++-
32 files changed, 429 insertions(+), 370 deletions(-)
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
index 3a847a5..74656e4 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/jobgen/QueryLogicalExpressionJobGen.java
@@ -22,7 +22,6 @@ import java.util.List;
import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.config.CompilerProperties;
-import org.apache.asterix.common.dataflow.ICcApplicationContext;
import org.apache.asterix.common.functions.FunctionDescriptorTag;
import org.apache.asterix.external.library.ExternalFunctionDescriptorProvider;
import org.apache.asterix.metadata.declared.MetadataProvider;
@@ -141,8 +140,8 @@ public class QueryLogicalExpressionJobGen implements
ILogicalExpressionJobGen {
IScalarEvaluatorFactory[] args = codegenArguments(expr, env,
inputSchemas, context);
IFunctionDescriptor fd = null;
if (expr.getFunctionInfo() instanceof IExternalFunctionInfo) {
- fd =
ExternalFunctionDescriptorProvider.getExternalFunctionDescriptor(
- (IExternalFunctionInfo) expr.getFunctionInfo(),
(ICcApplicationContext) context.getAppContext());
+ fd = ExternalFunctionDescriptorProvider
+ .getExternalFunctionDescriptor((IExternalFunctionInfo)
expr.getFunctionInfo());
CompilerProperties props = ((IApplicationContext)
context.getAppContext()).getCompilerProperties();
FunctionTypeInferers.SET_ARGUMENTS_TYPE.infer(expr, fd, env,
props);
} else {
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
index d3093e7..a34a991 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/optimizer/rules/ConstantFoldingRule.java
@@ -43,11 +43,11 @@ import
org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.formats.nontagged.TypeTraitProvider;
import org.apache.asterix.jobgen.QueryLogicalExpressionJobGen;
import org.apache.asterix.metadata.declared.MetadataProvider;
-import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.base.ADouble;
import org.apache.asterix.om.base.IAObject;
import org.apache.asterix.om.constants.AsterixConstantValue;
import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.typecomputer.impl.TypeComputeUtils;
import org.apache.asterix.om.types.ARecordType;
@@ -87,7 +87,8 @@ import
org.apache.hyracks.algebricks.core.rewriter.base.IAlgebraicRewriteRule;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.algebricks.runtime.evaluators.EvaluatorContext;
+import org.apache.hyracks.api.application.IServiceContext;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.api.exceptions.IWarningCollector;
@@ -167,17 +168,19 @@ public class ConstantFoldingRule implements
IAlgebraicRewriteRule {
}
private class ConstantFoldingVisitor implements
ILogicalExpressionVisitor<Pair<Boolean, ILogicalExpression>, Void>,
- ILogicalExpressionReferenceTransform {
+ ILogicalExpressionReferenceTransform, IEvaluatorContext {
private final IPointable p = VoidPointable.FACTORY.createPointable();
private final ByteBufferInputStream bbis = new ByteBufferInputStream();
private final DataInputStream dis = new DataInputStream(bbis);
private final WarningCollector warningCollector = new
WarningCollector();
- private final IEvaluatorContext evalContext = new
EvaluatorContext(warningCollector);
private IOptimizationContext optContext;
+ private IServiceContext serviceContext;
private void reset(IOptimizationContext context) {
optContext = context;
+ serviceContext =
+ ((MetadataProvider)
context.getMetadataProvider()).getApplicationContext().getServiceContext();
}
@Override
@@ -230,7 +233,7 @@ public class ConstantFoldingRule implements
IAlgebraicRewriteRule {
_emptyTypeEnv, _emptySchemas, jobGenCtx);
warningCollector.clear();
- IScalarEvaluator eval =
fact.createScalarEvaluator(evalContext);
+ IScalarEvaluator eval = fact.createScalarEvaluator(this);
eval.evaluate(null, p);
IAType returnType = (IAType) _emptyTypeEnv.getType(expr);
ATypeTag runtimeType = PointableHelper.getTypeTag(p);
@@ -362,7 +365,7 @@ public class ConstantFoldingRule implements
IAlgebraicRewriteRule {
// skip external functions that are not implemented in Java
IFunctionInfo fi = function.getFunctionInfo();
if (fi instanceof IExternalFunctionInfo
- &&
!Function.FunctionLanguage.JAVA.name().equals(((IExternalFunctionInfo)
fi).getLanguage())) {
+ &&
!ExternalFunctionLanguage.JAVA.equals(((IExternalFunctionInfo)
fi).getLanguage())) {
return false;
}
// skip all functions that would produce records/arrays/multisets
(derived types) in their open format
@@ -398,5 +401,22 @@ public class ConstantFoldingRule implements
IAlgebraicRewriteRule {
}
return true;
}
+
+ // IEvaluatorContext
+
+ @Override
+ public IServiceContext getServiceContext() {
+ return serviceContext;
+ }
+
+ @Override
+ public IHyracksTaskContext getTaskContext() {
+ return null;
+ }
+
+ @Override
+ public IWarningCollector getWarningCollector() {
+ return warningCollector;
+ }
}
}
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
index b93335b..c0d6f82 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/AqlExpressionToPlanTranslator.java
@@ -35,7 +35,6 @@ import org.apache.asterix.lang.common.base.ILangExpression;
import org.apache.asterix.lang.common.expression.VariableExpr;
import org.apache.asterix.lang.common.statement.Query;
import org.apache.asterix.metadata.declared.MetadataProvider;
-import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.commons.lang3.mutable.Mutable;
import org.apache.commons.lang3.mutable.MutableObject;
@@ -78,11 +77,6 @@ class AqlExpressionToPlanTranslator extends
LangExpressionToPlanTranslator imple
}
@Override
- protected Function.FunctionLanguage getFunctionLanguage() {
- return Function.FunctionLanguage.AQL;
- }
-
- @Override
public Pair<ILogicalOperator, LogicalVariable> visit(ForClause fc,
Mutable<ILogicalOperator> tupSource)
throws CompilationException {
LogicalVariable v = context.newVarFromExpression(fc.getVarExpr());
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
index f3480d5..6550bb4 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/LangExpressionToPlanTranslator.java
@@ -884,8 +884,6 @@ abstract class LangExpressionToPlanTranslator
return f;
}
- protected abstract Function.FunctionLanguage getFunctionLanguage();
-
private AbstractFunctionCallExpression
lookupUserDefinedFunction(FunctionSignature signature,
List<Mutable<ILogicalExpression>> args, SourceLocation sourceLoc)
throws CompilationException {
try {
@@ -895,9 +893,10 @@ abstract class LangExpressionToPlanTranslator
return null;
}
IFunctionInfo finfo =
- getFunctionLanguage().equals(function.getLanguage()) ?
FunctionUtil.getFunctionInfo(signature)
- : ExternalFunctionCompilerUtil
-
.getExternalFunctionInfo(metadataProvider.getMetadataTxnContext(), function);
+ function.isExternal()
+ ? ExternalFunctionCompilerUtil
+
.getExternalFunctionInfo(metadataProvider.getMetadataTxnContext(), function)
+ : FunctionUtil.getFunctionInfo(signature);
AbstractFunctionCallExpression f = new
ScalarFunctionCallExpression(finfo, args);
f.setSourceLocation(sourceLoc);
return f;
diff --git
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
index 3023305..82dc344 100644
---
a/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
+++
b/asterixdb/asterix-algebra/src/main/java/org/apache/asterix/translator/SqlppExpressionToPlanTranslator.java
@@ -82,7 +82,6 @@ import org.apache.asterix.lang.sqlpp.util.SqlppRewriteUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
import org.apache.asterix.metadata.declared.MetadataProvider;
-import org.apache.asterix.metadata.entities.Function;
import org.apache.asterix.om.base.ABoolean;
import org.apache.asterix.om.base.AInt32;
import org.apache.asterix.om.base.AString;
@@ -162,11 +161,6 @@ public class SqlppExpressionToPlanTranslator extends
LangExpressionToPlanTransla
}
@Override
- protected Function.FunctionLanguage getFunctionLanguage() {
- return Function.FunctionLanguage.SQLPP;
- }
-
- @Override
public Pair<ILogicalOperator, LogicalVariable> visit(Query q,
Mutable<ILogicalOperator> tupSource)
throws CompilationException {
Expression queryBody = q.getBody();
diff --git
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
index dcf2f2d..16ca37f 100644
---
a/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
+++
b/asterixdb/asterix-app/src/main/java/org/apache/asterix/app/translator/QueryTranslator.java
@@ -158,6 +158,7 @@ import org.apache.asterix.metadata.entities.Library;
import org.apache.asterix.metadata.entities.NodeGroup;
import org.apache.asterix.metadata.entities.Synonym;
import org.apache.asterix.metadata.feeds.FeedMetadataUtil;
+import org.apache.asterix.metadata.functions.ExternalFunctionCompilerUtil;
import org.apache.asterix.metadata.lock.ExternalDatasetsRegistry;
import org.apache.asterix.metadata.utils.DatasetUtil;
import org.apache.asterix.metadata.utils.ExternalIndexingOperations;
@@ -166,6 +167,7 @@ import org.apache.asterix.metadata.utils.KeyFieldTypeUtil;
import org.apache.asterix.metadata.utils.MetadataConstants;
import org.apache.asterix.metadata.utils.MetadataUtil;
import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.types.ARecordType;
import org.apache.asterix.om.types.ATypeTag;
import org.apache.asterix.om.types.AUnionType;
@@ -1843,26 +1845,24 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
if (lang == null) {
throw new
CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE,
sourceLoc, "");
}
- Function.FunctionLanguage functionLang;
+ ExternalFunctionLanguage functionLang;
try {
- functionLang =
Function.FunctionLanguage.valueOf(lang.toUpperCase(Locale.ROOT));
+ functionLang =
ExternalFunctionLanguage.valueOf(lang.toUpperCase(Locale.ROOT));
} catch (IllegalArgumentException e) {
throw new
CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE,
sourceLoc,
lang);
}
- if (functionLang.equals(getFunctionLanguage())) {
- throw new
CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE,
sourceLoc,
- lang);
- }
Library libraryInMetadata =
MetadataManager.INSTANCE.getLibrary(mdTxnCtx, dataverseName, libraryName);
if (libraryInMetadata == null) {
throw new CompilationException(ErrorCode.UNKNOWN_LIBRARY,
sourceLoc, libraryName);
}
// Add functions
+ String body =
ExternalFunctionCompilerUtil.encodeExternalIdentifier(signature, functionLang,
+ cfs.getExternalIdentifier());
List<List<Triple<DataverseName, String, String>>> dependencies
=
FunctionUtil.getExternalFunctionDependencies(dependentTypes);
- Function f = new Function(signature, argNames, argTypes,
returnType, cfs.getExternalIdentifier(),
- FunctionKind.SCALAR.toString(), functionLang,
libraryName, cfs.getNullCall(),
+ Function f = new Function(signature, argNames, argTypes,
returnType, body,
+ FunctionKind.SCALAR.toString(), functionLang.name(),
libraryName, cfs.getNullCall(),
cfs.getDeterministic(), cfs.getResources(),
dependencies);
MetadataManager.INSTANCE.addFunction(mdTxnCtx, f);
if (LOGGER.isInfoEnabled()) {
@@ -1882,7 +1882,8 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
FunctionUtil.getFunctionDependencies(rewriterFactory.createQueryRewriter(),
cfs.getFunctionBodyExpression(),
metadataProvider, dependentTypes);
Function function = new Function(signature, argNames,
argTypes, returnType, cfs.getFunctionBody(),
- FunctionKind.SCALAR.toString(), getFunctionLanguage(),
null, null, null, null, dependencies);
+ FunctionKind.SCALAR.toString(),
compilationProvider.getParserFactory().getLanguage(), null,
+ null, null, null, dependencies);
MetadataManager.INSTANCE.addFunction(mdTxnCtx, function);
if (LOGGER.isInfoEnabled()) {
LOGGER.info("Installed function: " + signature);
@@ -1949,17 +1950,6 @@ public class QueryTranslator extends
AbstractLangTranslator implements IStatemen
}
}
- private Function.FunctionLanguage getFunctionLanguage() {
- switch (compilationProvider.getLanguage()) {
- case SQLPP:
- return Function.FunctionLanguage.SQLPP;
- case AQL:
- return Function.FunctionLanguage.AQL;
- default:
- throw new
IllegalStateException(String.valueOf(compilationProvider.getLanguage()));
- }
- }
-
protected boolean isFunctionUsed(MetadataTransactionContext ctx,
FunctionSignature signature,
DataverseName currentDataverse) throws AlgebricksException {
List<Dataverse> allDataverses =
MetadataManager.INSTANCE.getDataverses(ctx);
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
index 952b620..8ac507f 100755
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
@@ -18,64 +18,24 @@
*/
package org.apache.asterix.external.library;
-import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.exceptions.AsterixException;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.functions.IFunctionDescriptor;
-import org.apache.asterix.om.types.IAType;
-import
org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
-import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
public class ExternalFunctionDescriptorProvider {
- public static IFunctionDescriptor
getExternalFunctionDescriptor(IExternalFunctionInfo finfo,
- IApplicationContext appCtx) throws AlgebricksException {
+ public static IFunctionDescriptor
getExternalFunctionDescriptor(IExternalFunctionInfo finfo)
+ throws AlgebricksException {
switch (finfo.getKind()) {
case SCALAR:
- return new ExternalScalarFunctionDescriptor(finfo, appCtx);
+ return new ExternalScalarFunctionDescriptor(finfo);
case AGGREGATE:
case UNNEST:
- throw new AlgebricksException("Unsupported function kind :" +
finfo.getKind());
+ throw new
AsterixException(ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNSUPPORTED_KIND,
finfo.getKind());
default:
- break;
+ throw new
AsterixException(ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNKNOWN_KIND,
finfo.getKind());
}
- return null;
}
-
-}
-
-class ExternalScalarFunctionDescriptor extends
AbstractScalarFunctionDynamicDescriptor implements IFunctionDescriptor {
- private static final long serialVersionUID = 1L;
- private final IFunctionInfo finfo;
- private IScalarEvaluatorFactory evaluatorFactory;
- private final transient IApplicationContext appCtx;
- private IAType[] argTypes;
-
- public ExternalScalarFunctionDescriptor(IFunctionInfo finfo,
IApplicationContext appCtx) {
- this.finfo = finfo;
- this.appCtx = appCtx;
- }
-
- @Override
- public void setImmutableStates(Object... states) {
- argTypes = new IAType[states.length];
- for (int i = 0; i < states.length; i++) {
- argTypes[i] = (IAType) states[i];
- }
- }
-
- @Override
- public IScalarEvaluatorFactory
createEvaluatorFactory(IScalarEvaluatorFactory[] args) throws
AlgebricksException {
- evaluatorFactory =
- new
ExternalScalarFunctionEvaluatorFactory((IExternalFunctionInfo) finfo, args,
argTypes, appCtx);
- return evaluatorFactory;
- }
-
- @Override
- public FunctionIdentifier getIdentifier() {
- return finfo.getFunctionIdentifier();
- }
-
-}
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionProvider.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionProvider.java
deleted file mode 100755
index cfc5895..0000000
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionProvider.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.asterix.external.library;
-
-import org.apache.asterix.common.api.IApplicationContext;
-import org.apache.asterix.common.exceptions.ErrorCode;
-import org.apache.asterix.common.exceptions.RuntimeDataException;
-import org.apache.asterix.external.api.IExternalFunction;
-import org.apache.asterix.external.api.IExternalScalarFunction;
-import org.apache.asterix.external.api.IFunctionHelper;
-import org.apache.asterix.om.functions.IExternalFunctionInfo;
-import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.data.std.api.IPointable;
-import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-
-public class ExternalFunctionProvider {
-
- public static IExternalFunction
getExternalFunctionEvaluator(IExternalFunctionInfo finfo,
- IScalarEvaluatorFactory[] args, IAType[] argTypes,
IEvaluatorContext context, IApplicationContext appCtx)
- throws HyracksDataException {
- switch (finfo.getKind()) {
- case SCALAR:
- return new ExternalScalarFunction(finfo, args, argTypes,
context, appCtx);
- case AGGREGATE:
- case UNNEST:
- throw new
RuntimeDataException(ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNSUPPORTED_KIND,
finfo.getKind());
- default:
- throw new
RuntimeDataException(ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNKNOWN_KIND,
finfo.getKind());
- }
- }
-}
-
-class ExternalScalarFunction extends ExternalFunction implements
IExternalScalarFunction, IScalarEvaluator {
-
- public ExternalScalarFunction(IExternalFunctionInfo finfo,
IScalarEvaluatorFactory[] args, IAType[] argTypes,
- IEvaluatorContext context, IApplicationContext appCtx) throws
HyracksDataException {
- super(finfo, args, argTypes, context, appCtx);
- try {
- initialize(functionHelper);
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- @Override
- public void evaluate(IFrameTupleReference tuple, IPointable result) throws
HyracksDataException {
- try {
- setArguments(tuple);
- evaluate(functionHelper);
- result.set(resultBuffer.getByteArray(),
resultBuffer.getStartOffset(), resultBuffer.getLength());
- functionHelper.reset();
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
- @Override
- public void evaluate(IFunctionHelper argumentProvider) throws
HyracksDataException {
- try {
- resultBuffer.reset();
- ((IExternalScalarFunction)
externalFunctionInstance).evaluate(argumentProvider);
- if (!argumentProvider.isValidResult()) {
- throw new
RuntimeDataException(ErrorCode.EXTERNAL_UDF_RESULT_TYPE_ERROR);
- }
- } catch (Exception e) {
- throw HyracksDataException.create(e);
- }
- }
-
-}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionDescriptor.java
old mode 100755
new mode 100644
similarity index 54%
copy from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
copy to
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionDescriptor.java
index 952b620..c033cea
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunctionDescriptorProvider.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionDescriptor.java
@@ -16,46 +16,25 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.asterix.external.library;
-import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.functions.IFunctionDescriptor;
import org.apache.asterix.om.types.IAType;
import
org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
-import org.apache.hyracks.algebricks.core.algebra.functions.IFunctionInfo;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
-public class ExternalFunctionDescriptorProvider {
-
- public static IFunctionDescriptor
getExternalFunctionDescriptor(IExternalFunctionInfo finfo,
- IApplicationContext appCtx) throws AlgebricksException {
- switch (finfo.getKind()) {
- case SCALAR:
- return new ExternalScalarFunctionDescriptor(finfo, appCtx);
- case AGGREGATE:
- case UNNEST:
- throw new AlgebricksException("Unsupported function kind :" +
finfo.getKind());
- default:
- break;
- }
- return null;
- }
-
-}
+public class ExternalScalarFunctionDescriptor extends
AbstractScalarFunctionDynamicDescriptor
+ implements IFunctionDescriptor {
-class ExternalScalarFunctionDescriptor extends
AbstractScalarFunctionDynamicDescriptor implements IFunctionDescriptor {
- private static final long serialVersionUID = 1L;
- private final IFunctionInfo finfo;
- private IScalarEvaluatorFactory evaluatorFactory;
- private final transient IApplicationContext appCtx;
+ private static final long serialVersionUID = 2L;
+ private final IExternalFunctionInfo finfo;
private IAType[] argTypes;
- public ExternalScalarFunctionDescriptor(IFunctionInfo finfo,
IApplicationContext appCtx) {
+ public ExternalScalarFunctionDescriptor(IExternalFunctionInfo finfo) {
this.finfo = finfo;
- this.appCtx = appCtx;
}
@Override
@@ -67,15 +46,12 @@ class ExternalScalarFunctionDescriptor extends
AbstractScalarFunctionDynamicDesc
}
@Override
- public IScalarEvaluatorFactory
createEvaluatorFactory(IScalarEvaluatorFactory[] args) throws
AlgebricksException {
- evaluatorFactory =
- new
ExternalScalarFunctionEvaluatorFactory((IExternalFunctionInfo) finfo, args,
argTypes, appCtx);
- return evaluatorFactory;
+ public IScalarEvaluatorFactory
createEvaluatorFactory(IScalarEvaluatorFactory[] args) {
+ return new ExternalScalarFunctionEvaluatorFactory(finfo, args,
argTypes);
}
@Override
public FunctionIdentifier getIdentifier() {
return finfo.getFunctionIdentifier();
}
-
-}
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluator.java
old mode 100755
new mode 100644
similarity index 55%
copy from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
copy to
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluator.java
index 80aee79..ce23cac
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluator.java
@@ -16,38 +16,34 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.asterix.external.library;
import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-public class ExternalScalarFunctionEvaluatorFactory implements
IScalarEvaluatorFactory {
+public abstract class ExternalScalarFunctionEvaluator implements
IScalarEvaluator {
- private static final long serialVersionUID = 1L;
- private final IExternalFunctionInfo finfo;
- private final IScalarEvaluatorFactory[] args;
- private final transient IApplicationContext appCtx;
- private final IAType[] argTypes;
+ protected final IExternalFunctionInfo finfo;
+ protected final IScalarEvaluator[] argEvals;
+ protected final IAType[] argTypes;
+ protected final ILibraryManager libraryManager;
- public ExternalScalarFunctionEvaluatorFactory(IExternalFunctionInfo finfo,
IScalarEvaluatorFactory[] args,
- IAType[] argTypes, IApplicationContext appCtx) throws
AlgebricksException {
+ public ExternalScalarFunctionEvaluator(IExternalFunctionInfo finfo,
IScalarEvaluatorFactory[] args,
+ IAType[] argTypes, IEvaluatorContext context) throws
HyracksDataException {
this.finfo = finfo;
- this.args = args;
this.argTypes = argTypes;
- this.appCtx = appCtx;
- }
-
- @Override
- public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx)
throws HyracksDataException {
- return (ExternalScalarFunction)
ExternalFunctionProvider.getExternalFunctionEvaluator(finfo, args, argTypes,
- ctx, appCtx == null ? (IApplicationContext)
ctx.getTaskContext().getJobletContext().getServiceContext()
- .getApplicationContext() : appCtx);
+ argEvals = new IScalarEvaluator[args.length];
+ for (int i = 0; i < args.length; i++) {
+ argEvals[i] = args[i].createScalarEvaluator(context);
+ }
+ libraryManager =
+ ((IApplicationContext)
context.getServiceContext().getApplicationContext()).getLibraryManager();
}
-
-}
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
index 80aee79..ec757a1 100755
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarFunctionEvaluatorFactory.java
@@ -18,10 +18,9 @@
*/
package org.apache.asterix.external.library;
-import org.apache.asterix.common.api.IApplicationContext;
+import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.types.IAType;
-import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
@@ -32,22 +31,25 @@ public class ExternalScalarFunctionEvaluatorFactory
implements IScalarEvaluatorF
private static final long serialVersionUID = 1L;
private final IExternalFunctionInfo finfo;
private final IScalarEvaluatorFactory[] args;
- private final transient IApplicationContext appCtx;
private final IAType[] argTypes;
public ExternalScalarFunctionEvaluatorFactory(IExternalFunctionInfo finfo,
IScalarEvaluatorFactory[] args,
- IAType[] argTypes, IApplicationContext appCtx) throws
AlgebricksException {
+ IAType[] argTypes) {
this.finfo = finfo;
this.args = args;
this.argTypes = argTypes;
- this.appCtx = appCtx;
}
@Override
public IScalarEvaluator createScalarEvaluator(IEvaluatorContext ctx)
throws HyracksDataException {
- return (ExternalScalarFunction)
ExternalFunctionProvider.getExternalFunctionEvaluator(finfo, args, argTypes,
- ctx, appCtx == null ? (IApplicationContext)
ctx.getTaskContext().getJobletContext().getServiceContext()
- .getApplicationContext() : appCtx);
+ switch (finfo.getLanguage()) {
+ case JAVA:
+ return new ExternalScalarJavaFunctionEvaluator(finfo, args,
argTypes, ctx);
+ case PYTHON:
+ return new ExternalScalarPythonFunctionEvaluator(finfo, args,
argTypes, ctx);
+ default:
+ throw new HyracksDataException(ErrorCode.ASTERIX,
ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNSUPPORTED_KIND,
+ finfo.getLanguage());
+ }
}
-
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
similarity index 53%
rename from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
rename to
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
index 55d6f5c..e406f9f 100755
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalFunction.java
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarJavaFunctionEvaluator.java
@@ -16,24 +16,21 @@
* specific language governing permissions and limitations
* under the License.
*/
+
package org.apache.asterix.external.library;
import java.io.IOException;
-import org.apache.asterix.common.api.IApplicationContext;
import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.exceptions.RuntimeDataException;
import org.apache.asterix.common.functions.FunctionSignature;
-import org.apache.asterix.common.library.ILibraryManager;
import org.apache.asterix.common.metadata.DataverseName;
-import org.apache.asterix.external.api.IExternalFunction;
+import org.apache.asterix.external.api.IExternalScalarFunction;
import org.apache.asterix.external.api.IFunctionFactory;
-import org.apache.asterix.external.api.IFunctionHelper;
import org.apache.asterix.om.functions.IExternalFunctionInfo;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
-import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IPointable;
@@ -41,60 +38,58 @@ import org.apache.hyracks.data.std.primitive.VoidPointable;
import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
-public abstract class ExternalFunction implements IExternalFunction {
+class ExternalScalarJavaFunctionEvaluator extends
ExternalScalarFunctionEvaluator {
- protected final IExternalFunctionInfo finfo;
- protected final IFunctionFactory externalFunctionFactory;
- protected final IExternalFunction externalFunctionInstance;
- protected final IScalarEvaluatorFactory[] evaluatorFactories;
- protected final IPointable inputVal = new VoidPointable();
+ protected final IExternalScalarFunction externalFunctionInstance;
+ protected final IPointable inputVal =
VoidPointable.FACTORY.createPointable();
protected final ArrayBackedValueStorage resultBuffer = new
ArrayBackedValueStorage();
- protected final IScalarEvaluator[] argumentEvaluators;
protected final JavaFunctionHelper functionHelper;
- protected final IAType[] argTypes;
- public ExternalFunction(IExternalFunctionInfo finfo,
IScalarEvaluatorFactory args[], IAType[] argTypes,
- IEvaluatorContext context, IApplicationContext appCtx) throws
HyracksDataException {
- this.finfo = finfo;
- this.evaluatorFactories = args;
- this.argTypes = argTypes;
- argumentEvaluators = new IScalarEvaluator[args.length];
- for (int i = 0; i < args.length; i++) {
- argumentEvaluators[i] = args[i].createScalarEvaluator(context);
- }
+ public ExternalScalarJavaFunctionEvaluator(IExternalFunctionInfo finfo,
IScalarEvaluatorFactory[] args,
+ IAType[] argTypes, IEvaluatorContext context) throws
HyracksDataException {
+ super(finfo, args, argTypes, context);
- ILibraryManager libraryManager = appCtx.getLibraryManager();
- String functionLibary = finfo.getLibrary();
- DataverseName dataverse =
FunctionSignature.getDataverseName(finfo.getFunctionIdentifier());
+ DataverseName functionDataverse =
FunctionSignature.getDataverseName(finfo.getFunctionIdentifier());
+ String functionLibrary = finfo.getLibrary();
functionHelper = new JavaFunctionHelper(finfo, argTypes, resultBuffer);
- ClassLoader libraryClassLoader =
libraryManager.getLibraryClassLoader(dataverse, functionLibary);
- String classname = finfo.getFunctionBody().trim();
- Class<?> clazz;
+ ClassLoader libraryClassLoader =
libraryManager.getLibraryClassLoader(functionDataverse, functionLibrary);
+ String classname = finfo.getExternalIdentifier().get(0).trim();
try {
- clazz = Class.forName(classname, true, libraryClassLoader);
- externalFunctionFactory = (IFunctionFactory) clazz.newInstance();
- externalFunctionInstance =
externalFunctionFactory.getExternalFunction();
+ Class<?> clazz = Class.forName(classname, true,
libraryClassLoader);
+ IFunctionFactory externalFunctionFactory = (IFunctionFactory)
clazz.newInstance();
+ externalFunctionInstance = (IExternalScalarFunction)
externalFunctionFactory.getExternalFunction();
} catch (Exception e) {
throw new
RuntimeDataException(ErrorCode.LIBRARY_EXTERNAL_FUNCTION_UNABLE_TO_LOAD_CLASS,
e, classname);
}
- }
- public void setArguments(IFrameTupleReference tuple) throws
AlgebricksException, IOException {
- for (int i = 0; i < evaluatorFactories.length; i++) {
- argumentEvaluators[i].evaluate(tuple, inputVal);
- functionHelper.setArgument(i, inputVal);
+ try {
+ externalFunctionInstance.initialize(functionHelper);
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
}
}
@Override
- public void deinitialize() {
- externalFunctionInstance.deinitialize();
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws
HyracksDataException {
+ try {
+ setArguments(tuple);
+ resultBuffer.reset();
+ externalFunctionInstance.evaluate(functionHelper);
+ if (!functionHelper.isValidResult()) {
+ throw new
RuntimeDataException(ErrorCode.EXTERNAL_UDF_RESULT_TYPE_ERROR);
+ }
+ result.set(resultBuffer.getByteArray(),
resultBuffer.getStartOffset(), resultBuffer.getLength());
+ functionHelper.reset();
+ } catch (Exception e) {
+ throw HyracksDataException.create(e);
+ }
}
- @Override
- public void initialize(IFunctionHelper functionHelper) throws Exception {
- externalFunctionInstance.initialize(functionHelper);
+ public void setArguments(IFrameTupleReference tuple) throws
AlgebricksException, IOException {
+ for (int i = 0; i < argEvals.length; i++) {
+ argEvals[i].evaluate(tuple, inputVal);
+ functionHelper.setArgument(i, inputVal);
+ }
}
-
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
new file mode 100644
index 0000000..dc06a72
--- /dev/null
+++
b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.external.library;
+
+import java.util.Objects;
+
+import org.apache.asterix.common.functions.FunctionSignature;
+import org.apache.asterix.common.metadata.DataverseName;
+import org.apache.asterix.om.functions.IExternalFunctionInfo;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.functions.PointableHelper;
+import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IHyracksTaskContext;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.api.job.JobId;
+import org.apache.hyracks.api.resources.IDeallocatable;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import org.apache.hyracks.dataflow.std.base.AbstractStateObject;
+
+class ExternalScalarPythonFunctionEvaluator extends
ExternalScalarFunctionEvaluator {
+
+ private final PythonLibraryEvaluator libraryEvaluator;
+
+ private final IPointable[] argValues;
+
+ public ExternalScalarPythonFunctionEvaluator(IExternalFunctionInfo finfo,
IScalarEvaluatorFactory[] args,
+ IAType[] argTypes, IEvaluatorContext ctx) throws
HyracksDataException {
+ super(finfo, args, argTypes, ctx);
+ DataverseName dataverseName =
FunctionSignature.getDataverseName(finfo.getFunctionIdentifier());
+ libraryEvaluator = PythonLibraryEvaluator.getInstance(dataverseName,
finfo.getLibrary(), ctx.getTaskContext());
+ argValues = new IPointable[args.length];
+ for (int i = 0; i < argValues.length; i++) {
+ argValues[i] = VoidPointable.FACTORY.createPointable();
+ }
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple, IPointable result) throws
HyracksDataException {
+ for (int i = 0, ln = argEvals.length; i < ln; i++) {
+ argEvals[i].evaluate(tuple, argValues[i]);
+ }
+ PointableHelper.setNull(result);
+ }
+
+ private static class PythonLibraryEvaluator extends AbstractStateObject
implements IDeallocatable {
+
+ private PythonLibraryEvaluator(JobId jobId, PythonLibraryEvaluatorId
evaluatorId) {
+ super(jobId, evaluatorId);
+ }
+
+ @Override
+ public void deallocate() {
+ }
+
+ private static PythonLibraryEvaluator getInstance(DataverseName
dataverseName, String libraryName,
+ IHyracksTaskContext ctx) {
+ PythonLibraryEvaluatorId evaluatorId = new
PythonLibraryEvaluatorId(dataverseName, libraryName);
+ PythonLibraryEvaluator evaluator = (PythonLibraryEvaluator)
ctx.getStateObject(evaluatorId);
+ if (evaluator == null) {
+ evaluator = new
PythonLibraryEvaluator(ctx.getJobletContext().getJobId(), evaluatorId);
+ ctx.registerDeallocatable(evaluator);
+ ctx.setStateObject(evaluator);
+ }
+ return evaluator;
+ }
+ }
+
+ private static final class PythonLibraryEvaluatorId {
+
+ private final DataverseName dataverseName;
+
+ private final String libraryName;
+
+ private PythonLibraryEvaluatorId(DataverseName dataverseName, String
libraryName) {
+ this.dataverseName = Objects.requireNonNull(dataverseName);
+ this.libraryName = Objects.requireNonNull(libraryName);
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o)
+ return true;
+ if (o == null || getClass() != o.getClass())
+ return false;
+ PythonLibraryEvaluatorId that = (PythonLibraryEvaluatorId) o;
+ return dataverseName.equals(that.dataverseName) &&
libraryName.equals(that.libraryName);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(dataverseName, libraryName);
+ }
+ }
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/parser/AQLParserFactory.java
b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/parser/AQLParserFactory.java
index 41ca142..3aa1217 100644
---
a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/parser/AQLParserFactory.java
+++
b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/parser/AQLParserFactory.java
@@ -25,6 +25,9 @@ import org.apache.asterix.lang.common.base.IParserFactory;
public class AQLParserFactory implements IParserFactory {
+ // WARNING: This value is stored in function metadata. Do not modify.
+ public static final String AQL = "AQL";
+
@Override
public IParser createParser(String query) {
return new AQLParser(query);
@@ -35,4 +38,8 @@ public class AQLParserFactory implements IParserFactory {
return new AQLParser(reader);
}
+ @Override
+ public String getLanguage() {
+ return AQL;
+ }
}
diff --git
a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
index d94500d..142a183 100644
---
a/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
+++
b/asterixdb/asterix-lang-aql/src/main/java/org/apache/asterix/lang/aql/rewrites/AqlQueryRewriter.java
@@ -53,7 +53,6 @@ import
org.apache.asterix.lang.common.util.CommonFunctionMapUtil;
import org.apache.asterix.lang.common.util.FunctionUtil;
import org.apache.asterix.lang.common.visitor.GatherFunctionCallsVisitor;
import org.apache.asterix.metadata.declared.MetadataProvider;
-import org.apache.asterix.metadata.entities.Function;
import org.apache.hyracks.algebricks.common.utils.Pair;
class AqlQueryRewriter implements IQueryRewriter {
@@ -67,7 +66,7 @@ class AqlQueryRewriter implements IQueryRewriter {
AqlQueryRewriter(IParserFactory parserFactory) {
this.parserFactory = parserFactory;
- functionParser = new FunctionParser(Function.FunctionLanguage.AQL,
this.parserFactory);
+ functionParser = new FunctionParser(parserFactory);
}
private void setup(List<FunctionDecl> declaredFunctions,
IReturningStatement topStatement,
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
index 3f8fabc..1775fbb 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
@@ -22,8 +22,9 @@ import java.io.Reader;
public interface IParserFactory {
- public IParser createParser(String query);
+ IParser createParser(String query);
- public IParser createParser(Reader reader);
+ IParser createParser(Reader reader);
+ String getLanguage();
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/FunctionParser.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/FunctionParser.java
index 3a5d488..e37868c 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/FunctionParser.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/FunctionParser.java
@@ -30,22 +30,19 @@ import org.apache.asterix.metadata.entities.Function;
public class FunctionParser {
- private final Function.FunctionLanguage language;
-
private final IParserFactory parserFactory;
- public FunctionParser(Function.FunctionLanguage language, IParserFactory
parserFactory) {
- this.language = language;
+ public FunctionParser(IParserFactory parserFactory) {
this.parserFactory = parserFactory;
}
- public Function.FunctionLanguage getFunctionLanguage() {
- return language;
+ public String getLanguage() {
+ return parserFactory.getLanguage();
}
public FunctionDecl getFunctionDecl(Function function) throws
CompilationException {
- if (!function.getLanguage().equals(language)) {
- throw new
CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE,
language,
+ if (!function.getLanguage().equals(getLanguage())) {
+ throw new
CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE,
getLanguage(),
function.getLanguage());
}
IParser parser = parserFactory.createParser(new
StringReader(function.getFunctionBody()));
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java
index 1c21665..72962fa 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/statement/CreateFunctionStatement.java
@@ -47,7 +47,7 @@ public class CreateFunctionStatement extends
AbstractStatement {
private final String lang;
private final String libName;
- private final String externalIdentifier;
+ private final List<String> externalIdentifier;
private final Boolean deterministic;
private final Boolean nullCall;
private final AdmObjectNode resources;
@@ -71,7 +71,7 @@ public class CreateFunctionStatement extends
AbstractStatement {
public CreateFunctionStatement(FunctionSignature signature,
List<Pair<VarIdentifier, IndexedTypeExpression>> parameterList,
IndexedTypeExpression returnType,
- boolean deterministic, boolean nullCall, String lang, String
libName, String externalIdentifier,
+ boolean deterministic, boolean nullCall, String lang, String
libName, List<String> externalIdentifier,
RecordConstructor resources, boolean ifNotExists) throws
CompilationException {
this.signature = signature;
this.ifNotExists = ifNotExists;
@@ -120,7 +120,7 @@ public class CreateFunctionStatement extends
AbstractStatement {
return externalIdentifier != null;
}
- public String getExternalIdentifier() {
+ public List<String> getExternalIdentifier() {
return externalIdentifier;
}
diff --git
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index c9c8abc..0c16f47 100644
---
a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++
b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -200,7 +200,7 @@ public class FunctionUtil {
messageBuilder.toString());
}
- if
(functionParser.getFunctionLanguage().equals(function.getLanguage())) {
+ if (functionParser.getLanguage().equals(function.getLanguage())) {
FunctionDecl functionDecl =
functionParser.getFunctionDecl(function);
if (functionDecl != null) {
if (functionDecls.contains(functionDecl)) {
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
index 1e68a19..5b724f2 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
@@ -25,6 +25,9 @@ import org.apache.asterix.lang.common.base.IParserFactory;
public class SqlppParserFactory implements IParserFactory {
+ // WARNING: This value is stored in function metadata. Do not modify.
+ public static final String SQLPP = "SQLPP";
+
@Override
public IParser createParser(String query) {
return new SQLPPParser(query);
@@ -34,4 +37,9 @@ public class SqlppParserFactory implements IParserFactory {
public IParser createParser(Reader reader) {
return new SQLPPParser(reader);
}
+
+ @Override
+ public String getLanguage() {
+ return SQLPP;
+ }
}
diff --git
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
index dc39425..72de614 100644
---
a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
+++
b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/rewrites/SqlppQueryRewriter.java
@@ -78,7 +78,6 @@ import org.apache.asterix.lang.sqlpp.util.SqlppAstPrintUtil;
import org.apache.asterix.lang.sqlpp.util.SqlppVariableUtil;
import org.apache.asterix.lang.sqlpp.visitor.base.ISqlppVisitor;
import org.apache.asterix.metadata.declared.MetadataProvider;
-import org.apache.asterix.metadata.entities.Function;
import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.logging.log4j.LogManager;
@@ -101,7 +100,7 @@ public class SqlppQueryRewriter implements IQueryRewriter {
public SqlppQueryRewriter(IParserFactory parserFactory) {
this.parserFactory = parserFactory;
- functionParser = new FunctionParser(Function.FunctionLanguage.SQLPP,
parserFactory);
+ functionParser = new FunctionParser(parserFactory);
}
protected void setup(List<FunctionDecl> declaredFunctions,
IReturningStatement topExpr,
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index 3681445..0b9c394 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -1062,6 +1062,7 @@ CreateFunctionStatement FunctionSpecification(Token
startStmtToken) throws Parse
String lang = null;
String libName ="";
String externalIdent = "";
+ List<String> externalIdentList = null;
List<Pair<VarIdentifier,IndexedTypeExpression>> args = null;
RecordConstructor resources = null;
}
@@ -1119,7 +1120,9 @@ CreateFunctionStatement FunctionSpecification(Token
startStmtToken) throws Parse
lang = Identifier()
( (<NOT> <IDENTIFIER> {expectToken(DETERMINISTIC); deterministic
= false;}) | (<IDENTIFIER> {expectToken(DETERMINISTIC); deterministic = true;})
)?
(<NULL> <IDENTIFIER> { expectToken(CALL); nullCall = true;})?
- <AS> libName = ConstantString() <COMMA> externalIdent =
ConstantString()
+ <AS> libName = ConstantString()
+ { externalIdentList = new ArrayList<String>(2); }
+ ( <COMMA> externalIdent = ConstantString() {
externalIdentList.add(externalIdent); } )+
(<WITH> resources = RecordConstructor())?
{
signature = new FunctionSignature(fctName.dataverse,
fctName.function, args.size());
@@ -1128,7 +1131,7 @@ CreateFunctionStatement FunctionSpecification(Token
startStmtToken) throws Parse
try{
stmt =
new CreateFunctionStatement(signature, args, returnType,
deterministic, nullCall,
- lang, libName,
externalIdent, resources, ifNotExists);
+ lang, libName,
externalIdentList, resources, ifNotExists);
} catch (AlgebricksException e) {
throw new
SqlppParseException(getSourceLocation(startStmtToken), e.getMessage());
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
index f9328b8..6884970 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entities/Function.java
@@ -39,7 +39,7 @@ public class Function implements IMetadataEntity<Function> {
private final List<IAType> argTypes;
private final IAType returnType;
private final String body;
- private final FunctionLanguage language;
+ private final String language;
private final String kind;
private final String library;
private final Boolean deterministic; // null for SQL++ and AQL functions
@@ -48,7 +48,7 @@ public class Function implements IMetadataEntity<Function> {
private final List<List<Triple<DataverseName, String, String>>>
dependencies;
public Function(FunctionSignature signature, List<String> argNames,
List<IAType> argTypes, IAType returnType,
- String functionBody, String functionKind, FunctionLanguage
language, String library, Boolean nullCall,
+ String functionBody, String functionKind, String language, String
library, Boolean nullCall,
Boolean deterministic, Map<String, String> params,
List<List<Triple<DataverseName, String, String>>> dependencies) {
this.signature = signature;
@@ -99,7 +99,7 @@ public class Function implements IMetadataEntity<Function> {
return returnType;
}
- public FunctionLanguage getLanguage() {
+ public String getLanguage() {
return language;
}
@@ -107,6 +107,10 @@ public class Function implements IMetadataEntity<Function>
{
return kind;
}
+ public boolean isExternal() {
+ return library != null;
+ }
+
public String getLibrary() {
return library;
}
@@ -136,12 +140,4 @@ public class Function implements IMetadataEntity<Function>
{
public Function dropFromCache(MetadataCache cache) {
return cache.dropFunction(this);
}
-
- // WARNING: These values are stored in function metadata. Do not rename.
- public enum FunctionLanguage {
- AQL,
- SQLPP,
- JAVA,
- PYTHON
- }
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
index 65aa94c..14089c2 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/entitytupletranslators/FunctionTupleTranslator.java
@@ -41,8 +41,6 @@ import java.util.Map;
import org.apache.asterix.builders.IARecordBuilder;
import org.apache.asterix.builders.OrderedListBuilder;
import org.apache.asterix.builders.RecordBuilder;
-import org.apache.asterix.common.exceptions.AsterixException;
-import org.apache.asterix.common.exceptions.ErrorCode;
import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.transactions.TxnId;
@@ -126,15 +124,8 @@ public class FunctionTupleTranslator extends
AbstractDatatypeTupleTranslator<Fun
String definition = ((AString) functionRecord
.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_DEFINITION_FIELD_INDEX)).getStringValue();
- String languageValue = ((AString) functionRecord
+ String language = ((AString) functionRecord
.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX)).getStringValue();
-
- Function.FunctionLanguage language;
- try {
- language = Function.FunctionLanguage.valueOf(languageValue);
- } catch (IllegalArgumentException e) {
- throw new AsterixException(ErrorCode.METADATA_ERROR);
- }
String functionKind =
((AString)
functionRecord.getValueByPos(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_KIND_FIELD_INDEX))
.getStringValue();
@@ -316,7 +307,7 @@ public class FunctionTupleTranslator extends
AbstractDatatypeTupleTranslator<Fun
// write field 6
fieldValue.reset();
- aString.setValue(function.getLanguage().name());
+ aString.setValue(function.getLanguage());
stringSerde.serialize(aString, fieldValue.getDataOutput());
recordBuilder.addField(MetadataRecordTypes.FUNCTION_ARECORD_FUNCTION_LANGUAGE_FIELD_INDEX,
fieldValue);
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
index 12c1c78..537cc24 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtil.java
@@ -18,10 +18,16 @@
*/
package org.apache.asterix.metadata.functions;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+
import org.apache.asterix.common.exceptions.AsterixException;
import org.apache.asterix.common.exceptions.ErrorCode;
+import org.apache.asterix.common.functions.FunctionSignature;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -54,15 +60,23 @@ public class ExternalFunctionCompilerUtil {
private static IFunctionInfo
getScalarFunctionInfo(MetadataTransactionContext txnCtx, Function function)
throws AlgebricksException {
if (function.getDeterministic() == null) {
- throw new AsterixException(ErrorCode.METADATA_ERROR);
+ throw new AsterixException(ErrorCode.METADATA_ERROR, "");
}
IAType returnType = function.getReturnType();
IResultTypeComputer typeComputer = new
ExternalTypeComputer(returnType, function.getArgTypes());
+ ExternalFunctionLanguage lang;
+ try {
+ lang = ExternalFunctionLanguage.valueOf(function.getLanguage());
+ } catch (IllegalArgumentException e) {
+ throw new AsterixException(ErrorCode.METADATA_ERROR,
function.getLanguage());
+ }
+ List<String> externalIdentifier = decodeExternalIdentifier(lang,
function.getFunctionBody());
+
return new
ExternalScalarFunctionInfo(function.getSignature().createFunctionIdentifier(),
returnType,
- function.getFunctionBody(), function.getLanguage().name(),
function.getLibrary(),
- function.getArgTypes(), function.getParams(),
function.getDeterministic(), typeComputer);
+ externalIdentifier, lang, function.getLibrary(),
function.getArgTypes(), function.getParams(),
+ function.getDeterministic(), typeComputer);
}
private static IFunctionInfo
getUnnestFunctionInfo(MetadataTransactionContext txnCtx, Function function) {
@@ -77,4 +91,79 @@ public class ExternalFunctionCompilerUtil {
return null;
}
+ public static String encodeExternalIdentifier(FunctionSignature
functionSignature,
+ ExternalFunctionLanguage language, List<String> identList) throws
AlgebricksException {
+ switch (language) {
+ case JAVA:
+ // input:
+ // [0] = package.class
+ //
+ // output: package.class
+
+ return identList.get(0);
+
+ case PYTHON:
+ // input: either a method or a top-level function
+ // [0] = package.module(:class)?
+ // [1] = (function_or_method)? - if missing then defaults to
declared function name
+ //
+ // output:
+ // case 1 (method): package.module:class.method
+ // case 2 (function): package.module:function
+
+ String ident0 = identList.get(0);
+ String ident1 = identList.size() > 1 ? identList.get(1) :
functionSignature.getName();
+ boolean classExists = ident0.indexOf(':') > 0;
+ return ident0 + (classExists ? '.' : ':') + ident1;
+
+ default:
+ throw new AsterixException(ErrorCode.COMPILATION_ERROR,
language);
+ }
+ }
+
+ public static List<String>
decodeExternalIdentifier(ExternalFunctionLanguage language, String encodedValue)
+ throws AlgebricksException {
+ switch (language) {
+ case JAVA:
+ // input: class
+ //
+ // output:
+ // [0] = class
+ return Collections.singletonList(encodedValue);
+
+ case PYTHON:
+ // input:
+ // case 1 (method): package.module:class.method
+ // case 2 (function): package.module:function
+ //
+ // output:
+ // case 1:
+ // [0] = package.module
+ // [1] = class
+ // [2] = method
+ // case 2:
+ // [0] = package.module
+ // [1] = function
+
+ int d1 = encodedValue.indexOf(':');
+ if (d1 <= 0) {
+ throw new AsterixException(ErrorCode.COMPILATION_ERROR,
encodedValue);
+ }
+ String moduleName = encodedValue.substring(0, d1);
+ int d2 = encodedValue.lastIndexOf('.');
+ if (d2 > d1) {
+ // class.method
+ String className = encodedValue.substring(d1 + 1, d2);
+ String methodName = encodedValue.substring(d2 + 1);
+ return Arrays.asList(moduleName, className, methodName);
+ } else {
+ // function
+ String functionName = encodedValue.substring(d1 + 1);
+ return Arrays.asList(moduleName, functionName);
+ }
+
+ default:
+ throw new AsterixException(ErrorCode.COMPILATION_ERROR,
language);
+ }
+ }
}
diff --git
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
index a81bcf6..c0ef0fb 100644
---
a/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
+++
b/asterixdb/asterix-metadata/src/main/java/org/apache/asterix/metadata/functions/ExternalScalarFunctionInfo.java
@@ -22,6 +22,7 @@ import java.util.List;
import java.util.Map;
import org.apache.asterix.om.functions.ExternalFunctionInfo;
+import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.typecomputer.base.IResultTypeComputer;
import org.apache.asterix.om.types.IAType;
import
org.apache.hyracks.algebricks.core.algebra.expressions.AbstractFunctionCallExpression.FunctionKind;
@@ -29,18 +30,19 @@ import
org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
public class ExternalScalarFunctionInfo extends ExternalFunctionInfo {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
public ExternalScalarFunctionInfo(String namespace, String library, String
name, int arity, IAType returnType,
- String body, String language, List<IAType> argumentTypes,
Map<String, String> params, boolean deterministic,
- IResultTypeComputer rtc) {
- super(namespace, name, arity, FunctionKind.SCALAR, argumentTypes,
returnType, rtc, body, language, library,
- params, deterministic);
+ List<String> externalIdentifier, ExternalFunctionLanguage
language, List<IAType> argumentTypes,
+ Map<String, String> params, boolean deterministic,
IResultTypeComputer rtc) {
+ super(namespace, name, arity, FunctionKind.SCALAR, argumentTypes,
returnType, rtc, language, library,
+ externalIdentifier, params, deterministic);
}
- public ExternalScalarFunctionInfo(FunctionIdentifier fid, IAType
returnType, String body, String language,
- String library, List<IAType> argumentTypes, Map<String, String>
params, boolean deterministic,
- IResultTypeComputer rtc) {
- super(fid, FunctionKind.SCALAR, argumentTypes, returnType, rtc, body,
language, library, params, deterministic);
+ public ExternalScalarFunctionInfo(FunctionIdentifier fid, IAType
returnType, List<String> externalIdentifier,
+ ExternalFunctionLanguage language, String library, List<IAType>
argumentTypes, Map<String, String> params,
+ boolean deterministic, IResultTypeComputer rtc) {
+ super(fid, FunctionKind.SCALAR, argumentTypes, returnType, rtc,
language, library, externalIdentifier, params,
+ deterministic);
}
}
diff --git
a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtilTest.java
b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtilTest.java
index edaabc9..732ae09 100644
---
a/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtilTest.java
+++
b/asterixdb/asterix-metadata/src/test/java/org/apache/asterix/metadata/functions/ExternalFunctionCompilerUtilTest.java
@@ -25,6 +25,7 @@ import org.apache.asterix.common.metadata.DataverseName;
import org.apache.asterix.common.transactions.TxnId;
import org.apache.asterix.metadata.MetadataTransactionContext;
import org.apache.asterix.metadata.entities.Function;
+import org.apache.asterix.om.functions.ExternalFunctionLanguage;
import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.hyracks.algebricks.common.exceptions.AlgebricksException;
@@ -38,7 +39,7 @@ public class ExternalFunctionCompilerUtilTest {
MetadataTransactionContext txnCtx = new MetadataTransactionContext(new
TxnId(1));
FunctionSignature signature = new
FunctionSignature(DataverseName.createSinglePartName("test"), "test", 0);
Function function = new Function(signature, new LinkedList<>(), new
LinkedList<>(), BuiltinType.ASTRING, "",
- "SCALAR", Function.FunctionLanguage.JAVA, "", false, false,
null, null);
+ "SCALAR", ExternalFunctionLanguage.JAVA.name(), "", false,
false, null, null);
// when
ExternalScalarFunctionInfo info =
diff --git
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
index 7220d05..0cf42a6 100644
---
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionInfo.java
@@ -28,33 +28,33 @@ import
org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
public class ExternalFunctionInfo extends FunctionInfo implements
IExternalFunctionInfo {
- private static final long serialVersionUID = 1L;
+ private static final long serialVersionUID = 2L;
private final transient IResultTypeComputer rtc;
private final List<IAType> argumentTypes;
- private final String body;
- private final String language;
+ private final ExternalFunctionLanguage language;
private final FunctionKind kind;
private final IAType returnType;
private final String library;
+ private final List<String> externalIdentifier;
private final Map<String, String> params;
public ExternalFunctionInfo(String namespace, String name, int arity,
FunctionKind kind, List<IAType> argumentTypes,
- IAType returnType, IResultTypeComputer rtc, String body, String
language, String library,
- Map<String, String> params, boolean deterministic) {
- this(new FunctionIdentifier(namespace, name, arity), kind,
argumentTypes, returnType, rtc, body, language,
- library, params, deterministic);
+ IAType returnType, IResultTypeComputer rtc,
ExternalFunctionLanguage language, String library,
+ List<String> externalIdentifier, Map<String, String> params,
boolean deterministic) {
+ this(new FunctionIdentifier(namespace, name, arity), kind,
argumentTypes, returnType, rtc, language, library,
+ externalIdentifier, params, deterministic);
}
public ExternalFunctionInfo(FunctionIdentifier fid, FunctionKind kind,
List<IAType> argumentTypes,
- IAType returnType, IResultTypeComputer rtc, String body, String
language, String library,
- Map<String, String> params, boolean deterministic) {
+ IAType returnType, IResultTypeComputer rtc,
ExternalFunctionLanguage language, String library,
+ List<String> externalIdentifier, Map<String, String> params,
boolean deterministic) {
super(fid, deterministic);
this.rtc = rtc;
this.argumentTypes = argumentTypes;
- this.body = body;
this.library = library;
this.language = language;
+ this.externalIdentifier = externalIdentifier;
this.kind = kind;
this.returnType = returnType;
this.params = params;
@@ -69,17 +69,12 @@ public class ExternalFunctionInfo extends FunctionInfo
implements IExternalFunct
}
@Override
- public String getFunctionBody() {
- return body;
- }
-
- @Override
public List<IAType> getArgumentList() {
return argumentTypes;
}
@Override
- public String getLanguage() {
+ public ExternalFunctionLanguage getLanguage() {
return language;
}
@@ -99,8 +94,12 @@ public class ExternalFunctionInfo extends FunctionInfo
implements IExternalFunct
}
@Override
+ public List<String> getExternalIdentifier() {
+ return externalIdentifier;
+ }
+
+ @Override
public Map<String, String> getParams() {
return params;
}
-
}
diff --git
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ExternalLanguage.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
similarity index 84%
rename from
asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ExternalLanguage.java
rename to
asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
index dfdf2c7..d4788f8 100644
---
a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/api/ExternalLanguage.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/ExternalFunctionLanguage.java
@@ -1,4 +1,3 @@
-package org.apache.asterix.external.api;
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -18,7 +17,10 @@ package org.apache.asterix.external.api;
* under the License.
*/
-public enum ExternalLanguage {
+package org.apache.asterix.om.functions;
+
+// WARNING: These values are stored in function metadata. Do not rename.
+public enum ExternalFunctionLanguage {
JAVA,
PYTHON
-}
+}
\ No newline at end of file
diff --git
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
index 45744e8..8bd00fb 100644
---
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/IExternalFunctionInfo.java
@@ -32,11 +32,11 @@ public interface IExternalFunctionInfo extends
IFunctionInfo {
public IAType getReturnType();
- public String getFunctionBody();
+ public List<String> getExternalIdentifier();
public List<IAType> getArgumentList();
- public String getLanguage();
+ public ExternalFunctionLanguage getLanguage();
public FunctionKind getKind();
diff --git
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
index 1c14c1d..09655da 100644
---
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
+++
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/base/IEvaluatorContext.java
@@ -19,6 +19,7 @@
package org.apache.hyracks.algebricks.runtime.base;
+import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.IWarningCollector;
@@ -27,6 +28,12 @@ import org.apache.hyracks.api.exceptions.IWarningCollector;
*/
public interface IEvaluatorContext {
/**
+ * Returns service context. Available at compile time
+ * (CC context) and at run time (NC context).
+ */
+ IServiceContext getServiceContext();
+
+ /**
* Returns current task's context, or {@code null} if this evaluator
* is being executed by the constant folding rule at compile time.
*/
diff --git
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
index beecee9..8d31f6b 100644
---
a/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
+++
b/hyracks-fullstack/algebricks/algebricks-runtime/src/main/java/org/apache/hyracks/algebricks/runtime/evaluators/EvaluatorContext.java
@@ -22,24 +22,33 @@ package org.apache.hyracks.algebricks.runtime.evaluators;
import java.util.Objects;
import org.apache.hyracks.algebricks.runtime.base.IEvaluatorContext;
+import org.apache.hyracks.api.application.IServiceContext;
import org.apache.hyracks.api.context.IHyracksTaskContext;
import org.apache.hyracks.api.exceptions.IWarningCollector;
public final class EvaluatorContext implements IEvaluatorContext {
+ private final IServiceContext serviceContext;
+
private final IHyracksTaskContext taskContext;
private final IWarningCollector warningCollector;
public EvaluatorContext(IHyracksTaskContext taskContext) {
- this.taskContext = taskContext;
- this.warningCollector = taskContext.getWarningCollector();
+ this.taskContext = Objects.requireNonNull(taskContext);
+ this.serviceContext =
Objects.requireNonNull(taskContext.getJobletContext().getServiceContext());
+ this.warningCollector =
Objects.requireNonNull(taskContext.getWarningCollector());
}
- public EvaluatorContext(IWarningCollector warningCollector) {
- Objects.requireNonNull(warningCollector);
+ public EvaluatorContext(IServiceContext serviceContext, IWarningCollector
warningCollector) {
this.taskContext = null;
- this.warningCollector = warningCollector;
+ this.serviceContext = Objects.requireNonNull(serviceContext);
+ this.warningCollector = Objects.requireNonNull(warningCollector);
+ }
+
+ @Override
+ public IServiceContext getServiceContext() {
+ return serviceContext;
}
@Override
