jorisvandenbossche commented on code in PR #12590:
URL: https://github.com/apache/arrow/pull/12590#discussion_r842484958
##########
python/pyarrow/_compute.pyx:
##########
@@ -2251,3 +2325,144 @@ cdef CExpression _bind(Expression filter, Schema
schema) except *:
return GetResultValue(filter.unwrap().Bind(
deref(pyarrow_unwrap_schema(schema).get())))
+
+
+cdef CFunctionDoc _make_function_doc(dict func_doc):
+ """
+ Helper function to generate the FunctionDoc
+ """
+ cdef:
+ CFunctionDoc f_doc
+ vector[c_string] c_arg_names
+ c_bool c_options_required
+
+ f_doc.summary = tobytes(func_doc["summary"])
+ f_doc.description = tobytes(func_doc["description"])
+ for arg_name in func_doc["arg_names"]:
+ c_arg_names.push_back(tobytes(arg_name))
+ f_doc.arg_names = c_arg_names
+ # UDFOptions integration:
+ # TODO: https://issues.apache.org/jira/browse/ARROW-16041
+ f_doc.options_class = tobytes("None")
+ c_options_required = False
+ f_doc.options_required = c_options_required
+ return f_doc
+
+
+def register_function(func_name, num_args, function_doc, in_types,
+ out_type, callback):
+ """
+ Register a user-defined-function.
+
+ Parameters
+ ----------
+
+ func_name : str
+ function name
+ num_args : int
+ number of arguments in the function
+ function_doc : dict
+ a dictionary object with keys "summary" (str),
+ "description" (str), and "arg_names" (list of str).
+ in_types : List[InputType]
+ list of InputType objects which defines the input
+ types for the function
+ out_type : DataType
+ output type of the function
+ callback : callable
+ user defined function
+ function includes arguments equal to the number
+ of input_types defined. The return type of the
+ function is of the type defined as output_type.
+ The output should be an Array, Scalar, ChunkedArray,
+ Table, or RecordBatch based on the out_type.
+
+ Example
+ -------
+
+ >>> from pyarrow import compute as pc
+ >>> from pyarrow.compute import register_function
+ >>> from pyarrow.compute import InputType
+ >>>
+ >>> func_doc = {}
+ >>> func_doc["summary"] = "simple udf"
+ >>> func_doc["description"] = "add a constant to a scalar"
+ >>> func_doc["arg_names"] = ["x"]
+ >>>
+ >>> def add_constant(array):
+ ... return pc.call_function("add", [array, 1])
+ ...
+ >>>
+ >>> func_name = "py_add_func"
+ >>> arity = 1
+ >>> in_types = [InputType.array(pa.int64())]
+ >>> out_type = pa.int64()
+ >>> register_function(func_name, arity, func_doc,
+ ... in_types, out_type, add_constant)
+ >>>
+ >>> func = pc.get_function(func_name)
+ >>> func.name
+ 'py_add_func'
+ >>> ans = pc.call_function(func_name, [pa.array([20])])
+ >>> ans
+ <pyarrow.lib.Int64Array object at 0x10c22e700>
+ [
+ 21
+ ]
+ """
+ cdef:
+ c_string c_func_name
+ CArity c_arity
+ CFunctionDoc c_func_doc
+ CInputType in_tmp
+ vector[CInputType] c_in_types
+ PyObject* c_callback
+ shared_ptr[CDataType] c_type
+ COutputType* c_out_type
+ CScalarUdfBuilder* c_sc_builder
+ CStatus st
+ CScalarUdfOptions* c_options
+ object obj
+
+ c_func_name = tobytes(func_name)
+
+ if num_args <= 0:
+ raise ValueError("number of arguments must be >= 0")
+ if num_args == 0:
+ c_arity = CArity.Nullary()
+ elif num_args == 1:
+ c_arity = CArity.Unary()
+ elif num_args == 2:
+ c_arity = CArity.Binary()
+ elif num_args == 3:
+ c_arity = CArity.Ternary()
+ elif num_args > 3:
+ c_arity = CArity.VarArgs(num_args)
+
+ c_func_doc = _make_function_doc(function_doc)
Review Comment:
Would it be possible to make the `function_doc` optional?
Or otherwise, we will need to update `_make_function_doc` a bit more robust
to handle user input. Currently it just assumes the required keys are present,
but so you get "ignored exceptions":
```
In [5]: pc.register_function("add_one", 1, {},
[pc.InputType.scalar(pa.int64())], pa.int64(), add_one)
---------------------------------------------------------------------------
KeyError Traceback (most recent call last)
KeyError: 'summary'
Exception ignored in: 'pyarrow._compute._make_function_doc'
Traceback (most recent call last):
File "<ipython-input-5-754cd7fa5caa>", line 1, in <module>
KeyError: 'summary'
```
##########
python/pyarrow/tests/test_udf.py:
##########
@@ -0,0 +1,350 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+from typing import List
+
+import pytest
+
+import pyarrow as pa
+from pyarrow import compute as pc
+from pyarrow.compute import register_function
+from pyarrow.compute import InputType
+
+
+def get_function_doc(summary: str, desc: str, arg_names: List[str]):
+ func_doc = {}
+ func_doc["summary"] = summary
+ func_doc["description"] = desc
+ func_doc["arg_names"] = arg_names
+ return func_doc
+
+# scalar unary function data
+
+
+unary_doc = get_function_doc("add function",
+ "test add function",
+ ["scalar1"])
+
+
+def unary_function(scalar1):
+ return pc.call_function("add", [scalar1, 1])
+
+# scalar binary function data
+
+
+binary_doc = get_function_doc("y=mx",
+ "find y from y = mx",
+ ["m", "x"])
+
+
+def binary_function(m, x):
+ return pc.call_function("multiply", [m, x])
+
+# scalar ternary function data
+
+
+ternary_doc = get_function_doc("y=mx+c",
+ "find y from y = mx + c",
+ ["m", "x", "c"])
+
+
+def ternary_function(m, x, c):
+ mx = pc.call_function("multiply", [m, x])
+ return pc.call_function("add", [mx, c])
+
+# scalar varargs function data
+
+
+varargs_doc = get_function_doc("z=ax+by+c",
+ "find z from z = ax + by + c",
+ ["a", "x", "b", "y", "c"])
+
+
+def varargs_function(a, x, b, y, c):
+ ax = pc.call_function("multiply", [a, x])
+ by = pc.call_function("multiply", [b, y])
+ ax_by = pc.call_function("add", [ax, by])
+ return pc.call_function("add", [ax_by, c])
+
+
[email protected]
+def function_input_types():
+ return [
+ # scalar data input types
+ [
+ InputType.scalar(pa.int64())
+ ],
+ [
+ InputType.scalar(pa.int64()),
+ InputType.scalar(pa.int64())
+ ],
+ [
+ InputType.scalar(pa.int64()),
+ InputType.scalar(pa.int64()),
+ InputType.scalar(pa.int64())
+ ],
+ [
+ InputType.scalar(pa.int64()),
+ InputType.scalar(pa.int64()),
+ InputType.scalar(pa.int64()),
+ InputType.scalar(pa.int64()),
+ InputType.scalar(pa.int64())
+ ],
+ # array data input types
+ [
+ InputType.array(pa.int64())
+ ],
+ [
+ InputType.array(pa.int64()),
+ InputType.array(pa.int64())
+ ],
+ [
+ InputType.array(pa.int64()),
+ InputType.array(pa.int64()),
+ InputType.array(pa.int64())
+ ],
+ [
+ InputType.array(pa.int64()),
+ InputType.array(pa.int64()),
+ InputType.array(pa.int64()),
+ InputType.array(pa.int64()),
+ InputType.array(pa.int64())
+ ]
+ ]
+
+
[email protected]
+def function_output_types():
+ return [
+ pa.int64(),
+ pa.int64(),
+ pa.int64(),
+ pa.int64()
+ ]
+
+
[email protected]
+def function_names():
+ return [
+ # scalar data function names
+ "scalar_y=x+k",
+ "scalar_y=mx",
+ "scalar_y=mx+c",
+ "scalar_z=ax+by+c",
+ # array data function names
+ "array_y=x+k",
+ "array_y=mx",
+ "array_y=mx+c",
+ "array_z=ax+by+c"
+ ]
+
+
[email protected]
+def function_arities():
+ return [
+ 1,
+ 2,
+ 3,
+ 5,
+ ]
+
+
[email protected]
+def function_docs():
+ return [
+ unary_doc,
+ binary_doc,
+ ternary_doc,
+ varargs_doc
+ ]
+
+
[email protected]
+def functions():
+ return [
+ unary_function,
+ binary_function,
+ ternary_function,
+ varargs_function
+ ]
+
+
[email protected]
+def function_inputs():
+ return [
+ # scalar input data
+ [
+ pa.scalar(10, pa.int64())
+ ],
+ [
+ pa.scalar(10, pa.int64()),
+ pa.scalar(2, pa.int64())
+ ],
+ [
+ pa.scalar(10, pa.int64()),
+ pa.scalar(2, pa.int64()),
+ pa.scalar(5, pa.int64())
+ ],
+ [
+ pa.scalar(2, pa.int64()),
+ pa.scalar(10, pa.int64()),
+ pa.scalar(3, pa.int64()),
+ pa.scalar(20, pa.int64()),
+ pa.scalar(5, pa.int64())
+ ],
+ # array input data
+ [
+ pa.array([10, 20], pa.int64())
+ ],
+ [
+ pa.array([10, 20], pa.int64()),
+ pa.array([2, 4], pa.int64())
+ ],
+ [
+ pa.array([10, 20], pa.int64()),
+ pa.array([2, 4], pa.int64()),
+ pa.array([5, 10], pa.int64())
+ ],
+ [
+ pa.array([2, 3], pa.int64()),
+ pa.array([10, 20], pa.int64()),
+ pa.array([3, 7], pa.int64()),
+ pa.array([20, 30], pa.int64()),
+ pa.array([5, 10], pa.int64())
+ ]
+ ]
+
+
[email protected]
+def expected_outputs():
+ return [
+ # scalar output data
+ pa.scalar(11, pa.int64()), # 10 + 1
+ pa.scalar(20, pa.int64()), # 10 * 2
+ pa.scalar(25, pa.int64()), # 10 * 2 + 5
+ pa.scalar(85, pa.int64()), # (2 * 10) + (3 * 20) + 5
+ # array output data
+ pa.array([11, 21], pa.int64()), # [10 + 1, 20 + 1]
+ pa.array([20, 80], pa.int64()), # [10 * 2, 20 * 4]
+ pa.array([25, 90], pa.int64()), # [(10 * 2) + 5, (20 * 4) + 10]
+ # [(2 * 10) + (3 * 20) + 5, (3 * 20) + (7 * 30) + 10]
+ pa.array([85, 280], pa.int64())
+ ]
+
+
+def test_scalar_udf_function_with_scalar_data(function_names,
+ function_arities,
+ function_input_types,
+ function_output_types,
+ function_docs,
+ functions,
+ function_inputs,
+ expected_outputs):
Review Comment:
Indeed, I wanted to comment something along the same lines. I would maybe
also suggest to keep the the arguments that will be passed together to
`register_function` also close together in the code (instead of first defining
all input types, all output types, all names, etc, which makes it hard to see
what is exactly being passed to `register_function`).
Even writing separate test functions for the different cases might be
simpler to follow.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
