This is an automated email from the ASF dual-hosted git repository. imaxon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit ec5195a4f58b2ce3971b537b48078f3bc696e0bf Author: Ian Maxon <[email protected]> AuthorDate: Tue May 4 09:35:21 2021 -0700 [ASTERIXDB-2897][FUN] Fix unknown handling in PyUDF - user model changes: no - storage format changes: no - interface changes: no Details: Carry the fix from ASTERIXDB-2883 to Python UDFs, and add a corresponding test for both this new code in the non-batched execution, as well as checking that it works the same in batched mode Change-Id: I64ac16d094caafcc657a21650011ed41d0c3c6d6 Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11324 Reviewed-by: Ian Maxon <[email protected]> Reviewed-by: Dmitry Lychagin <[email protected]> Integration-Tests: Jenkins <[email protected]> Tested-by: Jenkins <[email protected]> --- ...3.query.sqlpp => type_validation.10.query.sqlpp} | 7 +++++-- ...ion.4.ddl.sqlpp => type_validation.11.ddl.sqlpp} | 0 .../type_validation.2.ddl.sqlpp | 4 +++- .../type_validation.3.query.sqlpp | 3 ++- ....3.query.sqlpp => type_validation.4.query.sqlpp} | 5 ++++- ....3.query.sqlpp => type_validation.5.query.sqlpp} | 5 +++-- ....3.query.sqlpp => type_validation.6.query.sqlpp} | 7 +++++-- ....3.query.sqlpp => type_validation.7.query.sqlpp} | 3 ++- ....3.query.sqlpp => type_validation.8.query.sqlpp} | 5 ++++- ....3.query.sqlpp => type_validation.9.query.sqlpp} | 5 +++-- .../type_validation.1.adm | 2 +- .../type_validation.2.adm | 1 + .../type_validation.3.adm | 1 + .../type_validation.4.adm | 1 + .../type_validation.5.adm | 1 + .../type_validation.6.adm | 1 + .../type_validation.7.adm | 1 + .../type_validation.8.adm | 1 + .../ExternalScalarPythonFunctionEvaluator.java | 21 ++++++++++++++++++--- 19 files changed, 57 insertions(+), 17 deletions(-) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp similarity index 83% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp index 0ae7d0c..69a0b25 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.10.query.sqlpp @@ -17,6 +17,9 @@ * under the License. */ use externallibtest; -typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), - unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z"))); + +set `rewrite_attempt_batch_assign` "false"; + +typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), + unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.11.ddl.sqlpp similarity index 100% rename from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.ddl.sqlpp rename to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.11.ddl.sqlpp diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp index a8ba8a1..8041b5e 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.2.ddl.sqlpp @@ -18,6 +18,8 @@ */ use externallibtest; -create function typeValidation(a, b, c, d, e, f, g) +create function typeValidation(a, b, c, d, e, f, g, h) as "roundtrip", "Tests.roundtrip" at testlib; +create function typeValidationNullCall(a, b, c, d, e, f, g, h) + as "roundtrip", "Tests.roundtrip" at testlib with {"null-call": true}; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp index 0ae7d0c..f9aaf57 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp @@ -17,6 +17,7 @@ * under the License. */ use externallibtest; + typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), - unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z"))); + unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp similarity index 92% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp index 0ae7d0c..884f31f 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.4.query.sqlpp @@ -17,6 +17,9 @@ * under the License. */ use externallibtest; + +set `rewrite_attempt_batch_assign` "false"; + typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), - unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z"))); + unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp similarity index 87% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp index 0ae7d0c..fdd71ef 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.5.query.sqlpp @@ -17,6 +17,7 @@ * under the License. */ use externallibtest; -typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), - unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z"))); + +typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), + unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp similarity index 83% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp index 0ae7d0c..b85e0d0 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.6.query.sqlpp @@ -17,6 +17,9 @@ * under the License. */ use externallibtest; -typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), - unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z"))); + +set `rewrite_attempt_batch_assign` "false"; + +typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), + unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), null); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp similarity index 96% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp index 0ae7d0c..fa76f1a 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.7.query.sqlpp @@ -17,6 +17,7 @@ * under the License. */ use externallibtest; + typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), - unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z"))); + unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp similarity index 92% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp index 0ae7d0c..30640fd 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.8.query.sqlpp @@ -17,6 +17,9 @@ * under the License. */ use externallibtest; + +set `rewrite_attempt_batch_assign` "false"; + typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), - unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z"))); + unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp similarity index 86% copy from asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp copy to asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp index 0ae7d0c..810ba3c 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.3.query.sqlpp +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-library/python_open_type_validation/type_validation.9.query.sqlpp @@ -17,6 +17,7 @@ * under the License. */ use externallibtest; -typeValidation(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), - unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z"))); + +typeValidationNullCall(907, 9.07, "907", 9.07, true, unix_time_from_date_in_days(date("2013-01-01")), + unix_time_from_datetime_in_secs(datetime("1989-09-07T12:13:14.039Z")), missing); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm index 93f8aec..19765bd 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.1.adm @@ -1 +1 @@ -[ 907, 9.07, "907", 9.07, true, 15706, 621173594 ] +null diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm new file mode 100644 index 0000000..19765bd --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.2.adm @@ -0,0 +1 @@ +null diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm new file mode 100644 index 0000000..53fd925 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.3.adm @@ -0,0 +1 @@ +[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm new file mode 100644 index 0000000..53fd925 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.4.adm @@ -0,0 +1 @@ +[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm new file mode 100644 index 0000000..ec747fa --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.5.adm @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm new file mode 100644 index 0000000..ec747fa --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.6.adm @@ -0,0 +1 @@ +null \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm new file mode 100644 index 0000000..53fd925 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.7.adm @@ -0,0 +1 @@ +[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ] diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm new file mode 100644 index 0000000..53fd925 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-library/python_open_type_validation/type_validation.8.adm @@ -0,0 +1 @@ +[ 907, 9.07, "907", 9.07, true, 15706, 621173594, null ] diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java index e664f47..eb87399 100644 --- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java +++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/library/ExternalScalarPythonFunctionEvaluator.java @@ -19,6 +19,7 @@ package org.apache.asterix.external.library; +import static org.apache.asterix.om.types.EnumDeserializer.ATYPETAGDESERIALIZER; import static org.msgpack.core.MessagePack.Code.FIXARRAY_PREFIX; import java.io.DataOutput; @@ -88,17 +89,31 @@ class ExternalScalarPythonFunctionEvaluator extends ExternalScalarFunctionEvalua @Override public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException { argHolder.clear(); + boolean nullCall = finfo.getNullCall(); + boolean hasNullArg = false; for (int i = 0, ln = argEvals.length; i < ln; i++) { argEvals[i].evaluate(tuple, argValues[i]); - if (!finfo.getNullCall() && PointableHelper.checkAndSetMissingOrNull(result, argValues[i])) { - return; + if (!nullCall) { + byte[] argBytes = argValues[i].getByteArray(); + int argStart = argValues[i].getStartOffset(); + ATypeTag argType = ATYPETAGDESERIALIZER.deserialize(argBytes[argStart]); + if (argType == ATypeTag.MISSING) { + PointableHelper.setMissing(result); + return; + } else if (argType == ATypeTag.NULL) { + hasNullArg = true; + } } try { - PythonLibraryEvaluator.setArgument(argTypes[i], argValues[i], argHolder, finfo.getNullCall()); + PythonLibraryEvaluator.setArgument(argTypes[i], argValues[i], argHolder, nullCall); } catch (IOException e) { throw new HyracksDataException("Error evaluating Python UDF", e); } } + if (!nullCall && hasNullArg) { + PointableHelper.setNull(result); + return; + } try { ByteBuffer res = libraryEvaluator.callPython(fnId, argHolder, argTypes.length); resultBuffer.reset();
