This is an automated email from the ASF dual-hosted git repository.
michaelsmith pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/impala.git
The following commit(s) were added to refs/heads/master by this push:
new 7d71ec141 IMPALA-14575: Add constant handling for Hive GenericUDFs
7d71ec141 is described below
commit 7d71ec141f85746de50685d77e05314553b235a2
Author: Balazs Hevele <[email protected]>
AuthorDate: Wed Feb 11 14:41:41 2026 +0100
IMPALA-14575: Add constant handling for Hive GenericUDFs
Based on PoC by Csaba Ringhofer.
If an argument is constant, it is now only evaluated and copied
to the input buffer once, in HiveUdfCall::OpenEvaluator.
This means, we save the re-evaluation and re-copying of the value
for each evaluation.
The "isConstant" flags for arguments are also sent to the frontend,
where ConstantObjectInspectors will be created for them, so any
constant optimization in the UDF on Hive's side will be enabled.
Moved input handling for Hive UDF calls to a separate class
HiveUdfInputHandler.
Benchmark:
Checked with the following:
set num_nodes=1; set mt_dop=1;
select count(*) from tpch.lineitem where st_intersects(
st_point(l_partkey, l_suppkey),
st_geomfromtext(
"polygon ((0 0, 0 500000, 500000 500000, 500000 0, 0 0))"
)
);
Before change: 3.15s (MaterializeTupleTime: 2s601ms)
After change: 1.54s (MaterializeTupleTime: 879.397ms)
In some cases with geospatial, even bigger performance gain:
select count(*) from tpch.lineitem where st_intersects(
st_point(l_partkey, l_suppkey),
st_buffer(st_geomfromtext("point (250000 250000)"), 250000)
);
Before change: MaterializeTupleTime: 19s266ms
After change: MaterializeTupleTime: 1s587ms
Note that st_intersects is optimized on the Hive side when one argument
is a constant, contributing to most of the gain in performance.
The skipping of re-evaluation and copying is relatively insignificant,
comparing only that doesn't yield any measurable difference.
Testing:
-added a test UDF GenericAlltypeArgConstCheckUdf that prints
const information about arguments
-added const arg check cases to generic-java-udf.test
-added legacy udf call tests for several types to java-udf.test
to make sure it still works for all types
Change-Id: I4a6ca8c0bab499dffed88bb9786753da559af4c5
Reviewed-on: http://gerrit.cloudera.org:8080/23963
Reviewed-by: Csaba Ringhofer <[email protected]>
Tested-by: Michael Smith <[email protected]>
---
be/src/exprs/expr-value.h | 40 ++++
be/src/exprs/hive-udf-call.cc | 178 ++++++++++-------
be/src/exprs/hive-udf-call.h | 7 +
common/thrift/Frontend.thrift | 7 +
.../hive/executor/HiveGenericJavaFunction.java | 44 +++--
.../hive/executor/HiveJavaFunctionFactoryImpl.java | 2 +-
.../impala/hive/executor/HiveUdfExecutor.java | 94 ++-------
.../hive/executor/HiveUdfExecutorGeneric.java | 10 +-
.../hive/executor/HiveUdfExecutorLegacy.java | 61 +-----
.../impala/hive/executor/HiveUdfInputHandler.java | 220 +++++++++++++++++++++
.../apache/impala/hive/executor/UdfExecutor.java | 14 +-
.../impala/hive/executor/UdfExecutorTest.java | 3 +
.../impala/GenericAlltypeArgConstCheckUdf.java | 116 +++++++++++
.../queries/QueryTest/generic-java-udf.test | 72 +++++++
.../queries/QueryTest/java-udf.test | 19 ++
.../queries/QueryTest/load-generic-java-udfs.test | 4 +
16 files changed, 662 insertions(+), 229 deletions(-)
diff --git a/be/src/exprs/expr-value.h b/be/src/exprs/expr-value.h
index b0cc048a7..7d3927292 100644
--- a/be/src/exprs/expr-value.h
+++ b/be/src/exprs/expr-value.h
@@ -208,6 +208,46 @@ struct ExprValue {
}
}
+ void* SetToAnyVal(impala_udf::AnyVal* v, const ColumnType& type) {
+ if (v->is_null) return nullptr;
+ switch (type.type) {
+ case TYPE_BOOLEAN:
+ bool_val = ((impala_udf::BooleanVal*)v)->val;
+ return &bool_val;
+ case TYPE_TINYINT:
+ tinyint_val = ((impala_udf::TinyIntVal*)v)->val;
+ return &tinyint_val;
+ case TYPE_SMALLINT:
+ smallint_val = ((impala_udf::SmallIntVal*)v)->val;
+ return &smallint_val;
+ case TYPE_INT:
+ int_val = ((impala_udf::IntVal*)v)->val;
+ return &int_val;
+ case TYPE_BIGINT:
+ bigint_val = ((impala_udf::BigIntVal*)v)->val;
+ return &bigint_val;
+ case TYPE_FLOAT:
+ float_val = ((impala_udf::FloatVal*)v)->val;
+ return &float_val;
+ case TYPE_DOUBLE:
+ double_val = ((impala_udf::DoubleVal*)v)->val;
+ return &double_val;
+ case TYPE_TIMESTAMP:
+ timestamp_val =
TimestampValue::FromTimestampVal(*((impala_udf::TimestampVal*)v));
+ return ×tamp_val;
+ case TYPE_DATE:
+ date_val = DateValue(((impala_udf::DateVal*)v)->val);
+ return &date_val;
+ case TYPE_STRING:
+ case TYPE_VARCHAR:
+ string_val = StringValue::FromStringVal(*((impala_udf::StringVal*)v));
+ return &string_val;
+ default:
+ DCHECK(false) << "NYI";
+ return nullptr;
+ }
+ }
+
/// Returns whether the two ExprValue's are equal if they both have the
given type.
bool EqualsWithType(const ExprValue& other, const ColumnType& type) const {
switch (type.type) {
diff --git a/be/src/exprs/hive-udf-call.cc b/be/src/exprs/hive-udf-call.cc
index 37bb3fb9b..fcbf1bcec 100644
--- a/be/src/exprs/hive-udf-call.cc
+++ b/be/src/exprs/hive-udf-call.cc
@@ -56,53 +56,60 @@ HiveUdfCall::HiveUdfCall(const TExprNode& node)
DCHECK(executor_cl_ != NULL) << "Init() was not called!";
}
-AnyVal* HiveUdfCall::Evaluate(ScalarExprEvaluator* eval, const TupleRow* row)
const {
- FunctionContext* fn_ctx = eval->fn_context(fn_ctx_idx_);
- JniContext* jni_ctx = reinterpret_cast<JniContext*>(
- fn_ctx->GetFunctionState(FunctionContext::THREAD_LOCAL));
- DCHECK(jni_ctx != nullptr);
-
- JNIEnv* env = JniUtil::GetJNIEnv();
- DCHECK(env != nullptr);
-
- // Evaluate all the children values and put the results in
input_values_buffer
- for (int i = 0; i < GetNumChildren(); ++i) {
- void* v = eval->GetValue(*GetChild(i), row);
-
- if (v == nullptr) {
- jni_ctx->input_nulls_buffer[i] = 1;
+void HiveUdfCall::CopyToInputBuffer(JniContext* jni_ctx, int idx, void* val)
const {
+ if (val == nullptr) {
+ jni_ctx->input_nulls_buffer[idx] = 1;
} else {
- uint8_t* input_ptr = jni_ctx->input_values_buffer +
input_byte_offsets_[i];
- jni_ctx->input_nulls_buffer[i] = 0;
- switch (GetChild(i)->type().type) {
+ uint8_t* input_ptr = jni_ctx->input_values_buffer +
input_byte_offsets_[idx];
+ jni_ctx->input_nulls_buffer[idx] = 0;
+ switch (GetChild(idx)->type().type) {
case TYPE_BOOLEAN:
case TYPE_TINYINT:
// Using explicit sizes helps the compiler unroll memcpy
- memcpy(input_ptr, v, 1);
+ memcpy(input_ptr, val, 1);
break;
case TYPE_SMALLINT:
- memcpy(input_ptr, v, 2);
+ memcpy(input_ptr, val, 2);
break;
case TYPE_INT:
case TYPE_FLOAT:
case TYPE_DATE:
- memcpy(input_ptr, v, 4);
+ memcpy(input_ptr, val, 4);
break;
case TYPE_BIGINT:
case TYPE_DOUBLE:
- memcpy(input_ptr, v, 8);
+ memcpy(input_ptr, val, 8);
break;
case TYPE_TIMESTAMP:
- memcpy(input_ptr, v, sizeof(TimestampValue));
+ memcpy(input_ptr, val, sizeof(TimestampValue));
break;
case TYPE_STRING:
case TYPE_VARCHAR:
- memcpy(input_ptr, v, sizeof(StringValue));
+ memcpy(input_ptr, val, sizeof(StringValue));
break;
default:
DCHECK(false) << "NYI";
}
}
+}
+
+AnyVal* HiveUdfCall::Evaluate(ScalarExprEvaluator* eval, const TupleRow* row)
const {
+ FunctionContext* fn_ctx = eval->fn_context(fn_ctx_idx_);
+ JniContext* jni_ctx = reinterpret_cast<JniContext*>(
+ fn_ctx->GetFunctionState(FunctionContext::THREAD_LOCAL));
+ DCHECK(jni_ctx != nullptr);
+
+ JNIEnv* env = JniUtil::GetJNIEnv();
+ DCHECK(env != nullptr);
+
+ // Evaluate all the children values and put the results in
input_values_buffer
+ for (int i = 0; i < GetNumChildren(); ++i) {
+ // Skip re-evaluation for constant arguments, they are evaluated once and
copied
+ // to the input buffer in OpenEvaluator
+ if (is_constant_arg_[i]) continue;
+
+ void* v = eval->GetValue(*GetChild(i), row);
+ CopyToInputBuffer(jni_ctx, i, v);
}
// Using this version of Call has the lowest overhead. This eliminates the
@@ -164,6 +171,7 @@ Status HiveUdfCall::Init(
// Align all values up to 8 bytes. We don't care about footprint since we
allocate
// one buffer for all rows and we never copy the entire buffer.
input_buffer_size_ = BitUtil::RoundUpNumBytes(input_buffer_size_) * 8;
+ is_constant_arg_.push_back(GetChild(i)->is_constant());
}
return Status::OK();
}
@@ -186,6 +194,18 @@ Status
HiveUdfCall::OpenEvaluator(FunctionContext::FunctionStateScope scope,
jni_ctx->hdfs_location = fn_.hdfs_location.c_str();
jni_ctx->scalar_fn_symbol = fn_.scalar_fn.symbol.c_str();
+ // Only evaluate constant arguments at the top level of function contexts.
+ // If 'eval' was cloned, the constant values were copied from the parent.
+ if (scope == FunctionContext::FRAGMENT_LOCAL) {
+ vector<AnyVal*> constant_args;
+ for (const ScalarExpr* child : children()) {
+ AnyVal* const_val;
+ RETURN_IF_ERROR(eval->GetConstValue(state, *child, &const_val));
+ constant_args.push_back(const_val);
+ }
+ fn_ctx->impl()->SetConstantArgs(move(constant_args));
+ }
+
// Add a scoped cleanup jni reference object. This cleans up local refs made
below.
JniLocalFrame jni_frame;
{
@@ -213,6 +233,18 @@ Status
HiveUdfCall::OpenEvaluator(FunctionContext::FunctionStateScope scope,
ctor_params.output_buffer_ptr = (int64_t)jni_ctx->output_value_buffer;
ctor_params.output_null_ptr = (int64_t)&jni_ctx->output_null_value;
+ // Copy constant arguments to input buffer here once
+ // They will no longer be evaluted/copied during evaluation
+ for (int i = 0; i < is_constant_arg_.size(); i++) {
+ if (!is_constant_arg_[i]) continue;
+ AnyVal* val = fn_ctx->GetConstantArg(i);
+ DCHECK(val != nullptr);
+ ExprValue exprVal;
+ void* valptr = exprVal.SetToAnyVal(val, children_[i]->type());
+ CopyToInputBuffer(jni_ctx, i, valptr);
+ }
+ ctor_params.__set_is_constant_arg(is_constant_arg_);
+
jbyteArray ctor_params_bytes;
// Pushed frame will be popped when jni_frame goes out-of-scope.
@@ -272,54 +304,58 @@ Status HiveUdfCall::CodegenEvalChildren(LlvmCodeGen*
codegen, LlvmBuilder* build
llvm::Function* function, llvm::Value* (*args)[2], llvm::Value* jni_ctx,
llvm::BasicBlock* const first_block, llvm::BasicBlock** next_block) {
- llvm::Function* const set_input_null_buff_elem_fn =
- codegen->GetFunction(IRFunction::JNI_CTX_SET_INPUT_NULL_BUFF_ELEM,
false);
- llvm::Function* const get_input_val_buff_at_offset_fn =
- codegen->GetFunction(IRFunction::JNI_CTX_INPUT_VAL_BUFF_AT_OFFSET,
false);
-
- llvm::LLVMContext& context = codegen->context();
- llvm::BasicBlock* current_eval_child_block = first_block;
- const int num_children = GetNumChildren();
- for (int i = 0; i < num_children; ++i) {
- ScalarExpr* const child_expr = GetChild(i);
- llvm::Function* child_fn = nullptr;
- RETURN_IF_ERROR(child_expr->GetCodegendComputeFn(codegen, false,
&child_fn));
-
- builder->SetInsertPoint(current_eval_child_block);
-
- const ColumnType& child_type = child_expr->type();
- CodegenAnyVal child_wrapped = CodegenAnyVal::CreateCallWrapped(
- codegen, builder, child_type, child_fn, *args, "child");
-
- CodegenAnyValReadWriteInfo rwi = child_wrapped.ToReadWriteInfo();
- rwi.entry_block().BranchTo(builder);
-
- llvm::BasicBlock* next_eval_child_block = llvm::BasicBlock::Create(
- context, "eval_child", function);
-
- // Child is null
- builder->SetInsertPoint(rwi.null_block());
- builder->CreateCall(set_input_null_buff_elem_fn,
- {jni_ctx, codegen->GetI32Constant(i), codegen->GetI8Constant(1)});
- builder->CreateBr(next_eval_child_block);
-
- // Child is not null.
- builder->SetInsertPoint(rwi.non_null_block());
- builder->CreateCall(set_input_null_buff_elem_fn,
- {jni_ctx, codegen->GetI32Constant(i), codegen->GetI8Constant(0)});
- llvm::Value* const input_ptr =
builder->CreateCall(get_input_val_buff_at_offset_fn,
- {jni_ctx, codegen->GetI32Constant(input_byte_offsets_[i])},
"input_ptr");
-
- llvm::Value* const child_val_ptr =
- SlotDescriptor::CodegenStoreNonNullAnyValToNewAlloca(rwi);
- const std::size_t size = CodeGenUtil::GetTypeSize(child_type.type);
- codegen->CodegenMemcpy(builder, input_ptr, child_val_ptr, size);
- builder->CreateBr(next_eval_child_block);
- current_eval_child_block = next_eval_child_block;
- }
+ llvm::Function* const set_input_null_buff_elem_fn =
+ codegen->GetFunction(IRFunction::JNI_CTX_SET_INPUT_NULL_BUFF_ELEM,
false);
+ llvm::Function* const get_input_val_buff_at_offset_fn =
+ codegen->GetFunction(IRFunction::JNI_CTX_INPUT_VAL_BUFF_AT_OFFSET,
false);
+
+ llvm::LLVMContext& context = codegen->context();
+ llvm::BasicBlock* current_eval_child_block = first_block;
+ const int num_children = GetNumChildren();
+ for (int i = 0; i < num_children; ++i) {
+ // Skip re-evaluation for constant arguments, they are evaluated once and
copied
+ // to the input buffer in OpenEvaluator
+ if (is_constant_arg_[i]) continue;
+
+ ScalarExpr* const child_expr = GetChild(i);
+ llvm::Function* child_fn = nullptr;
+ RETURN_IF_ERROR(child_expr->GetCodegendComputeFn(codegen, false,
&child_fn));
+
+ builder->SetInsertPoint(current_eval_child_block);
+
+ const ColumnType& child_type = child_expr->type();
+ CodegenAnyVal child_wrapped = CodegenAnyVal::CreateCallWrapped(
+ codegen, builder, child_type, child_fn, *args, "child");
+
+ CodegenAnyValReadWriteInfo rwi = child_wrapped.ToReadWriteInfo();
+ rwi.entry_block().BranchTo(builder);
+
+ llvm::BasicBlock* next_eval_child_block = llvm::BasicBlock::Create(
+ context, "eval_child", function);
+
+ // Child is null
+ builder->SetInsertPoint(rwi.null_block());
+ builder->CreateCall(set_input_null_buff_elem_fn,
+ {jni_ctx, codegen->GetI32Constant(i), codegen->GetI8Constant(1)});
+ builder->CreateBr(next_eval_child_block);
+
+ // Child is not null.
+ builder->SetInsertPoint(rwi.non_null_block());
+ builder->CreateCall(set_input_null_buff_elem_fn,
+ {jni_ctx, codegen->GetI32Constant(i), codegen->GetI8Constant(0)});
+ llvm::Value* const input_ptr =
builder->CreateCall(get_input_val_buff_at_offset_fn,
+ {jni_ctx, codegen->GetI32Constant(input_byte_offsets_[i])},
"input_ptr");
+
+ llvm::Value* const child_val_ptr =
+ SlotDescriptor::CodegenStoreNonNullAnyValToNewAlloca(rwi);
+ const std::size_t size = CodeGenUtil::GetTypeSize(child_type.type);
+ codegen->CodegenMemcpy(builder, input_ptr, child_val_ptr, size);
+ builder->CreateBr(next_eval_child_block);
+ current_eval_child_block = next_eval_child_block;
+ }
- *next_block = current_eval_child_block;
- return Status::OK();
+ *next_block = current_eval_child_block;
+ return Status::OK();
}
llvm::Value* CastPtrAndLoad(LlvmCodeGen* codegen, LlvmBuilder* builder,
diff --git a/be/src/exprs/hive-udf-call.h b/be/src/exprs/hive-udf-call.h
index d3a151c7f..3467c4fbf 100644
--- a/be/src/exprs/hive-udf-call.h
+++ b/be/src/exprs/hive-udf-call.h
@@ -118,6 +118,11 @@ class HiveUdfCall : public ScalarExpr {
/// be written to.
std::vector<int> input_byte_offsets_;
+ /// is_constant_arg_[i] is true iff the ith argument is a constant.
+ /// Constant arguments are evaluated and copied into the input buffer once in
+ /// OpenEvaluation, so they are not re-evaluated and copied for each
evaluation.
+ std::vector<bool> is_constant_arg_;
+
/// The size of the buffer for passing in input arguments.
int input_buffer_size_;
@@ -151,6 +156,8 @@ class HiveUdfCall : public ScalarExpr {
static uint8_t* GetInputValuesBufferAtOffset(JniContext* jni_ctx, int
offset);
};
+ void CopyToInputBuffer(JniContext* jni_ctx, int idx, void* val) const;
+
/// Static helper functions for codegen.
static jclass* GetExecutorClass();
static jmethodID* GetExecutorEvaluateId();
diff --git a/common/thrift/Frontend.thrift b/common/thrift/Frontend.thrift
index 8de375f45..74cfb6363 100644
--- a/common/thrift/Frontend.thrift
+++ b/common/thrift/Frontend.thrift
@@ -63,6 +63,13 @@ struct THiveUdfExecutorCtorParams {
// NULL.
6: required i64 output_null_ptr
7: required i64 output_buffer_ptr
+
+ // is_constant_arg[i] is true iff the i-th argument is a constant.
+ // Constant arguments are only copied to the input buffer once, to save
+ // re-evaluation and extra copying for every evaluation.
+ // The information about an argument being a constant is also forwarded
+ // to Hive UDF executor, for further optimization when possible.
+ 8: optional list<bool> is_constant_arg
}
// Arguments to getTableNames, which returns a list of tables that are of
specified table
diff --git
a/fe/src/main/java/org/apache/impala/hive/executor/HiveGenericJavaFunction.java
b/fe/src/main/java/org/apache/impala/hive/executor/HiveGenericJavaFunction.java
index d067790cb..c4d7b1144 100644
---
a/fe/src/main/java/org/apache/impala/hive/executor/HiveGenericJavaFunction.java
+++
b/fe/src/main/java/org/apache/impala/hive/executor/HiveGenericJavaFunction.java
@@ -20,9 +20,12 @@ package org.apache.impala.hive.executor;
import org.apache.hadoop.hive.metastore.api.Function;
import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.PrimitiveTypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
import org.apache.impala.catalog.CatalogException;
import org.apache.impala.catalog.ScalarFunction;
import org.apache.impala.catalog.Type;
@@ -53,14 +56,15 @@ public class HiveGenericJavaFunction implements
HiveJavaFunction {
private final GenericUDF genericUDF_;
public HiveGenericJavaFunction(Class<?> udfClass,
- Function hiveFn, Type retType, Type[] parameterTypes)
+ Function hiveFn, Type retType, Type[] parameterTypes,
+ HiveUdfInputHandler inputHandler)
throws CatalogException {
try {
hiveFn_ = hiveFn;
retType_ = retType;
parameterTypes_ = parameterTypes;
genericUDF_ = createGenericUDFInstance(udfClass);
- returnOi_ = initializeWrapper();
+ returnOi_ = initializeWrapper(inputHandler);
checkValidFunction();
} catch (CatalogException e) {
String errorMsg = "Error retrieving class " + udfClass + ": " +
e.getMessage();
@@ -69,8 +73,9 @@ public class HiveGenericJavaFunction implements
HiveJavaFunction {
}
public HiveGenericJavaFunction(Class<?> udfClass,
- Type retType, Type[] parameterTypes) throws CatalogException {
- this(udfClass, null, retType, parameterTypes);
+ Type retType, Type[] parameterTypes, HiveUdfInputHandler inputHandler)
+ throws CatalogException {
+ this(udfClass, null, retType, parameterTypes, inputHandler);
}
@Override
@@ -124,16 +129,20 @@ public class HiveGenericJavaFunction implements
HiveJavaFunction {
}
private void checkValidFunction() throws CatalogException {
- if (returnOi_ != getInspector(retType_, true)
- && returnOi_ != getInspector(retType_, false)
+ boolean isRetTypeConst = returnOi_ instanceof ConstantObjectInspector;
+ if (returnOi_.getClass()
+ != getInspector(retType_, true, isRetTypeConst, null).getClass()
+ && returnOi_.getClass()
+ != getInspector(retType_, false, isRetTypeConst, null).getClass()
&& !returnOi_.getTypeName().equals("void")) {
throw new CatalogException("Function expected return type " +
returnOi_.getTypeName() + " but was created with " + retType_);
}
}
- private ObjectInspector initializeWrapper() throws CatalogException {
- ObjectInspector[] parameterOIs = getInspectors(parameterTypes_, true);
+ private ObjectInspector initializeWrapper(HiveUdfInputHandler inputHandler)
+ throws CatalogException {
+ ObjectInspector[] parameterOIs = getInspectors(parameterTypes_, true,
inputHandler);
try {
return genericUDF_.initialize(parameterOIs);
} catch (UDFArgumentException e) {
@@ -143,23 +152,36 @@ public class HiveGenericJavaFunction implements
HiveJavaFunction {
}
}
- private ObjectInspector[] getInspectors(Type[] typeArray, boolean
useWritable)
+ private ObjectInspector[] getInspectors(Type[] typeArray, boolean
useWritable,
+ HiveUdfInputHandler inputHandler)
throws CatalogException {
ObjectInspector[] OIArray = new ObjectInspector[typeArray.length];
for (int i = 0; i < typeArray.length; ++i) {
- OIArray[i] = getInspector(typeArray[i], useWritable);
+ boolean isConst = inputHandler == null ? false :
inputHandler.isArgConst(i);
+ Object constObj = isConst ? inputHandler.getConstObj(i) : null;
+ OIArray[i] = getInspector(typeArray[i], useWritable, isConst, constObj);
}
return OIArray;
}
- private ObjectInspector getInspector(Type t, boolean useWritable)
+ private ObjectInspector getInspector(Type t, boolean useWritable, boolean
isConst,
+ Object constObj)
throws CatalogException {
+ if (isConst) {
+ return getConstInspector(t, constObj);
+ }
PrimitiveCategory cat = getPrimitiveCategory(t);
return useWritable
?
PrimitiveObjectInspectorFactory.getPrimitiveWritableObjectInspector(cat)
: PrimitiveObjectInspectorFactory.getPrimitiveJavaObjectInspector(cat);
}
+ private ConstantObjectInspector getConstInspector(Type t, Object constObj) {
+ PrimitiveTypeInfo pt =
TypeInfoFactory.getPrimitiveTypeInfo(t.toSql().toLowerCase());
+ return
PrimitiveObjectInspectorFactory.getPrimitiveWritableConstantObjectInspector(
+ pt, constObj);
+ }
+
private PrimitiveCategory getPrimitiveCategory(Type t) throws
CatalogException {
switch (t.getPrimitiveType().toThrift()) {
case BOOLEAN:
diff --git
a/fe/src/main/java/org/apache/impala/hive/executor/HiveJavaFunctionFactoryImpl.java
b/fe/src/main/java/org/apache/impala/hive/executor/HiveJavaFunctionFactoryImpl.java
index ba04f002c..789fbd401 100644
---
a/fe/src/main/java/org/apache/impala/hive/executor/HiveJavaFunctionFactoryImpl.java
+++
b/fe/src/main/java/org/apache/impala/hive/executor/HiveJavaFunctionFactoryImpl.java
@@ -52,7 +52,7 @@ public class HiveJavaFunctionFactoryImpl implements
HiveJavaFunctionFactory {
paramTypes);
case GENERIC_UDF:
return new HiveGenericJavaFunction(javaClass.getUDFClass(), hiveFn,
retType,
- paramTypes);
+ paramTypes, null);
default:
throw new CatalogException("Function " + fnName + ": The class "
+ jarUri + " does not derive "
diff --git
a/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutor.java
b/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutor.java
index ee54066e9..596e87db2 100644
--- a/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutor.java
+++ b/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutor.java
@@ -45,20 +45,10 @@ public abstract class HiveUdfExecutor {
private static final Logger LOG = Logger.getLogger(HiveUdfExecutor.class);
- // Return and argument types of the function inferred from the udf method
signature.
+ // Return type of the function inferred from the udf method signature.
// The JavaUdfDataType enum maps it to corresponding primitive type.
- private final JavaUdfDataType[] argTypes_;
private final JavaUdfDataType retType_;
- // Input buffer from the backend. This is valid for the duration of an
evaluate() call.
- // These buffers are allocated in the BE.
- private final long inputBufferPtr_;
- private final long inputNullsPtr_;
-
- // This is the byte offset in inputBufferPtr to the start of the input
argument.
- // e.g. *inputBufferPtr_[inputBufferOffsets[i]] is the ith input argument.
- private final int[] inputBufferOffsets_;
-
// Output buffer to return non-string values. These buffers are allocated in
the BE.
private final long outputBufferPtr_;
private final long outputNullPtr_;
@@ -72,27 +62,17 @@ public abstract class HiveUdfExecutor {
// Size of outBufferStringPtr_.
private int outBufferStringCapacity_;
- // Preconstructed input objects for the UDF. This minimizes object creation
overhead
- // as these objects are reused across calls to evaluate().
- private final Writable[] inputObjects_;
+ protected final HiveUdfInputHandler inputHandler_;
protected HiveUdfExecutor(
- THiveUdfExecutorCtorParams request,
- JavaUdfDataType retType, JavaUdfDataType[] argTypes) throws
ImpalaRuntimeException {
+ THiveUdfExecutorCtorParams request, JavaUdfDataType retType,
+ HiveUdfInputHandler inputHandler) throws ImpalaRuntimeException {
retType_ = retType;
- argTypes_ = argTypes;
- inputBufferPtr_ = request.input_buffer_ptr;
- inputNullsPtr_ = request.input_nulls_ptr;
outputBufferPtr_ = request.output_buffer_ptr;
outputNullPtr_ = request.output_null_ptr;
outBufferStringPtr_ = 0;
outBufferStringCapacity_ = 0;
- inputBufferOffsets_ = new int[request.input_byte_offsets.size()];
- for (int i = 0; i < request.input_byte_offsets.size(); ++i) {
- inputBufferOffsets_[i] = request.input_byte_offsets.get(i).intValue();
- }
- inputObjects_ = new Writable[argTypes_.length];
- allocateInputObjects();
+ inputHandler_ = inputHandler;
}
/**
@@ -108,10 +88,10 @@ public abstract class HiveUdfExecutor {
/**
* Evaluate function called by the backend. The inputs to the UDF have
- * been serialized to 'inputObjects_'
+ * been serialized to 'inputHandler_.getInputObjects()'
*/
public long evaluate() throws ImpalaRuntimeException {
- return storeUdfResult(evaluateDerived(argTypes_, inputNullsPtr_,
inputObjects_));
+ return storeUdfResult(evaluateDerived(inputHandler_.getInputObjects()));
}
/**
@@ -119,7 +99,7 @@ public abstract class HiveUdfExecutor {
* for testing and not the version of evaluate() the backend uses.
*/
public long evaluateForTesting(Object... args) throws ImpalaRuntimeException
{
- return storeUdfResult(evaluateDerived(argTypes_, inputNullsPtr_, args));
+ return storeUdfResult(evaluateDerived(args));
}
// Sets the result object 'obj' into the outputBufferPtr_
@@ -218,11 +198,11 @@ public abstract class HiveUdfExecutor {
}
protected int getNumParams() {
- return inputObjects_.length;
+ return inputHandler_.getNumParams();
}
protected Object getInputObject(int i) {
- return inputObjects_[i];
+ return inputHandler_.getInputObject(i);
}
private void copyBytesToOutputBuffer(byte[] bytes) {
@@ -237,56 +217,6 @@ public abstract class HiveUdfExecutor {
outputBufferPtr_ + JavaUdfDataType.STRING_VALUE_LEN_OFFSET,
bytes.length);
}
- // Preallocate the input objects that will be passed to the underlying UDF.
- // These objects are allocated once and reused across calls to evaluate()
- private void allocateInputObjects() throws ImpalaRuntimeException {
- for (int i = 0; i < argTypes_.length; ++i) {
- int offset = inputBufferOffsets_[i];
- switch (argTypes_[i]) {
- case BOOLEAN:
- case BOOLEAN_WRITABLE:
- inputObjects_[i] = new ImpalaBooleanWritable(inputBufferPtr_ +
offset);
- break;
- case TINYINT:
- case BYTE_WRITABLE:
- inputObjects_[i] = new ImpalaTinyIntWritable(inputBufferPtr_ +
offset);
- break;
- case SMALLINT:
- case SHORT_WRITABLE:
- inputObjects_[i] = new ImpalaSmallIntWritable(inputBufferPtr_ +
offset);
- break;
- case INT:
- case INT_WRITABLE:
- inputObjects_[i] = new ImpalaIntWritable(inputBufferPtr_ + offset);
- break;
- case BIGINT:
- case LONG_WRITABLE:
- inputObjects_[i] = new ImpalaBigIntWritable(inputBufferPtr_ +
offset);
- break;
- case FLOAT:
- case FLOAT_WRITABLE:
- inputObjects_[i] = new ImpalaFloatWritable(inputBufferPtr_ + offset);
- break;
- case DOUBLE:
- case DOUBLE_WRITABLE:
- inputObjects_[i] = new ImpalaDoubleWritable(inputBufferPtr_ +
offset);
- break;
- case TEXT:
- inputObjects_[i] = new ImpalaTextWritable(inputBufferPtr_ + offset);
- break;
- case BYTES_WRITABLE:
- inputObjects_[i] = new ImpalaBytesWritable(inputBufferPtr_ + offset);
- break;
- case STRING:
- // String can be mapped to any String-like Writable class.
- inputObjects_[i] = new ImpalaBytesWritable(inputBufferPtr_ + offset);
- break;
- default:
- throw new ImpalaRuntimeException("Unsupported argument type: " +
argTypes_[i]);
- }
- }
- }
-
public static Type getRetType(THiveUdfExecutorCtorParams request) {
return Type.fromThrift(request.fn.ret_type);
}
@@ -307,8 +237,8 @@ public abstract class HiveUdfExecutor {
/**
* Abstract method allowing derived class to evaluate the function.
*/
- abstract protected Object evaluateDerived(JavaUdfDataType[] argTypes,
- long inputNullsPtr, Object[] inputObjectArgs) throws
ImpalaRuntimeException;
+ abstract protected Object evaluateDerived(Object[] inputObjectArgs)
+ throws ImpalaRuntimeException;
/**
* Abstract method returning the Java reflection Method type of the
'evaluate' method.
diff --git
a/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutorGeneric.java
b/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutorGeneric.java
index 9024ab536..fdcc465aa 100644
---
a/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutorGeneric.java
+++
b/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutorGeneric.java
@@ -56,9 +56,10 @@ public class HiveUdfExecutorGeneric extends HiveUdfExecutor {
* the backend.
*/
public HiveUdfExecutorGeneric(THiveUdfExecutorCtorParams request,
- HiveGenericJavaFunction hiveJavaFn) throws ImpalaRuntimeException {
+ HiveGenericJavaFunction hiveJavaFn, HiveUdfInputHandler inputHandler)
+ throws ImpalaRuntimeException {
super(request,
JavaUdfDataType.getType(hiveJavaFn.getReturnObjectInspector()),
- JavaUdfDataType.getTypes(hiveJavaFn.getParameterTypes()));
+ inputHandler);
genericUDF_ = hiveJavaFn.getGenericUDFInstance();
deferredParameters_ = createDeferredObjects();
deferredNullParameter_ = new DeferredJavaObject(null);
@@ -73,9 +74,10 @@ public class HiveUdfExecutorGeneric extends HiveUdfExecutor {
* Evaluates the UDF with 'args' as the input to the UDF.
*/
@Override
- protected Object evaluateDerived(JavaUdfDataType[] argTypes,
- long inputNullsPtr, Object[] inputObjectArgs) throws
ImpalaRuntimeException {
+ protected Object evaluateDerived(Object[] inputObjectArgs)
+ throws ImpalaRuntimeException {
try {
+ long inputNullsPtr = inputHandler_.getInputNullsPtr();
for (int i = 0; i < runtimeDeferredParameters_.length; ++i) {
if (UnsafeUtil.UNSAFE.getByte(inputNullsPtr + i) == 0) {
runtimeDeferredParameters_[i] = deferredParameters_[i];
diff --git
a/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutorLegacy.java
b/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutorLegacy.java
index c665997d9..16079d41c 100644
---
a/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutorLegacy.java
+++
b/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfExecutorLegacy.java
@@ -47,9 +47,9 @@ public class HiveUdfExecutorLegacy extends HiveUdfExecutor {
* Used by the backend.
*/
public HiveUdfExecutorLegacy(THiveUdfExecutorCtorParams request,
- HiveLegacyJavaFunction hiveJavaFn) throws ImpalaRuntimeException {
- super(request, JavaUdfDataType.getType(hiveJavaFn.getRetType()),
- JavaUdfDataType.getTypes(hiveJavaFn.getParameterTypes()));
+ HiveLegacyJavaFunction hiveJavaFn, HiveUdfInputHandler inputHandler)
+ throws ImpalaRuntimeException {
+ super(request, JavaUdfDataType.getType(hiveJavaFn.getRetType()),
inputHandler);
udf_ = hiveJavaFn.getUDFInstance();
method_ = hiveJavaFn.getMethod();
inputArgs_ = new Object[getNumParams()];
@@ -69,61 +69,10 @@ public class HiveUdfExecutorLegacy extends HiveUdfExecutor {
* Returns Object returned by UDF.
*/
@Override
- protected Object evaluateDerived(JavaUdfDataType[] argTypes,
- long inputNullsPtr, Object... inputObjects)
+ protected Object evaluateDerived(Object[] inputObjects)
throws ImpalaRuntimeException {
try {
- for (int i = 0; i < argTypes.length; ++i) {
- if (UnsafeUtil.UNSAFE.getByte(inputNullsPtr + i) == 0) {
- switch (argTypes[i]) {
- case BOOLEAN_WRITABLE:
- case BYTE_WRITABLE:
- case SHORT_WRITABLE:
- case INT_WRITABLE:
- case LONG_WRITABLE:
- case FLOAT_WRITABLE:
- case DOUBLE_WRITABLE: inputArgs_[i] = inputObjects[i]; break;
- case BYTE_ARRAY:
- case BYTES_WRITABLE:
- ((ImpalaBytesWritable) inputObjects[i]).reload();
- inputArgs_[i] = inputObjects[i];
- break;
- case TEXT:
- ((ImpalaTextWritable) inputObjects[i]).reload();
- inputArgs_[i] = inputObjects[i];
- break;
- case BOOLEAN:
- inputArgs_[i] = ((ImpalaBooleanWritable)inputObjects[i]).get();
- break;
- case TINYINT:
- inputArgs_[i] = ((ImpalaTinyIntWritable)inputObjects[i]).get();
- break;
- case SMALLINT:
- inputArgs_[i] = ((ImpalaSmallIntWritable)inputObjects[i]).get();
- break;
- case INT:
- inputArgs_[i] = ((ImpalaIntWritable)inputObjects[i]).get();
- break;
- case BIGINT:
- inputArgs_[i] = ((ImpalaBigIntWritable)inputObjects[i]).get();
- break;
- case FLOAT:
- inputArgs_[i] = ((ImpalaFloatWritable)inputObjects[i]).get();
- break;
- case DOUBLE:
- inputArgs_[i] = ((ImpalaDoubleWritable)inputObjects[i]).get();
- break;
- case STRING:
- Preconditions.checkState(inputObjects[i] instanceof
ImpalaBytesWritable);
- ImpalaBytesWritable inputObject = (ImpalaBytesWritable)
inputObjects[i];
- inputObject.reload();
- inputArgs_[i] = new String(inputObject.getBytes());
- break;
- }
- } else {
- inputArgs_[i] = null;
- }
- }
+ inputHandler_.fillArgArray(inputArgs_, inputObjects, false);
return method_.invoke(udf_, inputArgs_);
} catch (Exception e) {
e.printStackTrace(System.err);
diff --git
a/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfInputHandler.java
b/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfInputHandler.java
new file mode 100644
index 000000000..ef98bb098
--- /dev/null
+++ b/fe/src/main/java/org/apache/impala/hive/executor/HiveUdfInputHandler.java
@@ -0,0 +1,220 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala.hive.executor;
+
+import sun.misc.Unsafe;
+
+import java.lang.reflect.Method;
+
+import com.google.common.base.Preconditions;
+
+import org.apache.hadoop.io.Writable;
+import org.apache.impala.common.ImpalaRuntimeException;
+import org.apache.impala.thrift.THiveUdfExecutorCtorParams;
+import org.apache.impala.util.UnsafeUtil;
+import org.apache.log4j.Logger;
+
+public class HiveUdfInputHandler {
+
+ private static final Logger LOG =
Logger.getLogger(HiveUdfInputHandler.class);
+
+ // Argument types of the function inferred from the udf method signature.
+ // The JavaUdfDataType enum maps it to corresponding primitive type.
+ private final JavaUdfDataType[] argTypes_;
+
+ // Input buffer from the backend. This is valid for the duration of an
evaluate() call.
+ // These buffers are allocated in the BE.
+ private final long inputBufferPtr_;
+ private final long inputNullsPtr_;
+
+ // This is the byte offset in inputBufferPtr to the start of the input
argument.
+ // e.g. *inputBufferPtr_[inputBufferOffsets[i]] is the ith input argument.
+ private final int[] inputBufferOffsets_;
+
+ private final boolean[] isArgConst_;
+
+ // Preconstructed input objects for the UDF. This minimizes object creation
overhead
+ // as these objects are reused across calls to evaluate().
+ private final Writable[] inputObjects_;
+
+ private final Object[] constObjects_;
+
+ public HiveUdfInputHandler(
+ THiveUdfExecutorCtorParams request,
+ JavaUdfDataType[] argTypes) throws ImpalaRuntimeException {
+ argTypes_ = argTypes;
+ inputBufferPtr_ = request.input_buffer_ptr;
+ inputNullsPtr_ = request.input_nulls_ptr;
+ inputBufferOffsets_ = new int[request.input_byte_offsets.size()];
+ for (int i = 0; i < request.input_byte_offsets.size(); ++i) {
+ inputBufferOffsets_[i] = request.input_byte_offsets.get(i).intValue();
+ }
+ isArgConst_ = new boolean[request.fn.arg_types.size()];
+ if (request.is_constant_arg != null) {
+ for (int i = 0; i < request.is_constant_arg.size(); i++) {
+ isArgConst_[i] = request.is_constant_arg.get(i);
+ }
+ }
+ inputObjects_ = allocateInputObjects(argTypes_, inputBufferOffsets_,
inputBufferPtr_);
+ constObjects_ = new Object[isArgConst_.length];
+ fillArgArray(constObjects_, inputObjects_, true);
+ }
+
+ public int getNumParams() {
+ return inputObjects_.length;
+ }
+
+ public Object getInputObject(int i) {
+ return inputObjects_[i];
+ }
+
+ public Writable[] getInputObjects() {
+ return inputObjects_;
+ }
+
+ public boolean isArgConst(int i) {
+ return isArgConst_[i];
+ }
+
+ public Object getConstObj(int i) {
+ return constObjects_[i];
+ }
+
+ public long getInputNullsPtr() {
+ return inputNullsPtr_;
+ }
+
+ public JavaUdfDataType[] getArgTypes() {
+ return argTypes_;
+ }
+
+ // Preallocate the input objects that will be passed to the underlying UDF.
+ // These objects are allocated once and reused across calls to evaluate()
+ public Writable[] allocateInputObjects(JavaUdfDataType[] argTypes,
+ int[] inputBufferOffsets, long inputBufferPtr) throws
ImpalaRuntimeException {
+ Writable[] inputObjects = new Writable[argTypes.length];
+ for (int i = 0; i < argTypes.length; ++i) {
+ int offset = inputBufferOffsets[i];
+ switch (argTypes[i]) {
+ case BOOLEAN:
+ case BOOLEAN_WRITABLE:
+ inputObjects[i] = new ImpalaBooleanWritable(inputBufferPtr + offset);
+ break;
+ case TINYINT:
+ case BYTE_WRITABLE:
+ inputObjects[i] = new ImpalaTinyIntWritable(inputBufferPtr + offset);
+ break;
+ case SMALLINT:
+ case SHORT_WRITABLE:
+ inputObjects[i] = new ImpalaSmallIntWritable(inputBufferPtr +
offset);
+ break;
+ case INT:
+ case INT_WRITABLE:
+ inputObjects[i] = new ImpalaIntWritable(inputBufferPtr+ offset);
+ break;
+ case BIGINT:
+ case LONG_WRITABLE:
+ inputObjects[i] = new ImpalaBigIntWritable(inputBufferPtr + offset);
+ break;
+ case FLOAT:
+ case FLOAT_WRITABLE:
+ inputObjects[i] = new ImpalaFloatWritable(inputBufferPtr + offset);
+ break;
+ case DOUBLE:
+ case DOUBLE_WRITABLE:
+ inputObjects[i] = new ImpalaDoubleWritable(inputBufferPtr + offset);
+ break;
+ case TEXT:
+ inputObjects[i] = new ImpalaTextWritable(inputBufferPtr + offset);
+ break;
+ case BYTES_WRITABLE:
+ inputObjects[i] = new ImpalaBytesWritable(inputBufferPtr + offset);
+ break;
+ case STRING:
+ // String can be mapped to any String-like Writable class.
+ inputObjects[i] = new ImpalaBytesWritable(inputBufferPtr + offset);
+ break;
+ default:
+ throw new ImpalaRuntimeException("Unsupported argument type: " +
argTypes[i]);
+ }
+ }
+ return inputObjects;
+ }
+
+ public void fillArgArray(Object[] inputArgs, Object[] inputObjects, boolean
onlyConst)
+ throws ImpalaRuntimeException {
+ try {
+ for (int i = 0; i < argTypes_.length; ++i) {
+ if (onlyConst && !isArgConst_[i]) continue;
+ if (UnsafeUtil.UNSAFE.getByte(inputNullsPtr_ + i) == 0) {
+ switch (argTypes_[i]) {
+ case BOOLEAN_WRITABLE:
+ case BYTE_WRITABLE:
+ case SHORT_WRITABLE:
+ case INT_WRITABLE:
+ case LONG_WRITABLE:
+ case FLOAT_WRITABLE:
+ case DOUBLE_WRITABLE: inputArgs[i] = inputObjects[i]; break;
+ case BYTE_ARRAY:
+ case BYTES_WRITABLE:
+ ((ImpalaBytesWritable) inputObjects[i]).reload();
+ inputArgs[i] = inputObjects[i];
+ break;
+ case TEXT:
+ ((ImpalaTextWritable) inputObjects[i]).reload();
+ inputArgs[i] = inputObjects[i];
+ break;
+ case BOOLEAN:
+ inputArgs[i] = ((ImpalaBooleanWritable)inputObjects[i]).get();
+ break;
+ case TINYINT:
+ inputArgs[i] = ((ImpalaTinyIntWritable)inputObjects[i]).get();
+ break;
+ case SMALLINT:
+ inputArgs[i] = ((ImpalaSmallIntWritable)inputObjects[i]).get();
+ break;
+ case INT:
+ inputArgs[i] = ((ImpalaIntWritable)inputObjects[i]).get();
+ break;
+ case BIGINT:
+ inputArgs[i] = ((ImpalaBigIntWritable)inputObjects[i]).get();
+ break;
+ case FLOAT:
+ inputArgs[i] = ((ImpalaFloatWritable)inputObjects[i]).get();
+ break;
+ case DOUBLE:
+ inputArgs[i] = ((ImpalaDoubleWritable)inputObjects[i]).get();
+ break;
+ case STRING:
+ Preconditions.checkState(inputObjects[i] instanceof
ImpalaBytesWritable);
+ ImpalaBytesWritable inputObject = (ImpalaBytesWritable)
inputObjects[i];
+ inputObject.reload();
+ inputArgs[i] = new String(inputObject.getBytes());
+ break;
+ }
+ } else {
+ inputArgs[i] = null;
+ }
+ }
+ } catch (Exception e) {
+ throw new ImpalaRuntimeException(
+ "HiveUdfInputHandler::fillArgArray() ran into a problem.", e);
+ }
+ }
+
+}
diff --git a/fe/src/main/java/org/apache/impala/hive/executor/UdfExecutor.java
b/fe/src/main/java/org/apache/impala/hive/executor/UdfExecutor.java
index 21c75078e..4866ebeaf 100644
--- a/fe/src/main/java/org/apache/impala/hive/executor/UdfExecutor.java
+++ b/fe/src/main/java/org/apache/impala/hive/executor/UdfExecutor.java
@@ -106,21 +106,27 @@ public class UdfExecutor implements AutoCloseable {
*/
private HiveUdfExecutor createHiveUdfExecutor(THiveUdfExecutorCtorParams
request,
HiveUdfLoader udfLoader) throws ImpalaRuntimeException {
+ Type[] parameterTypes = HiveUdfExecutor.getParameterTypes(request);
try {
switch (udfLoader.getUDFClassType()) {
case UDF: {
HiveLegacyJavaFunction function =
new HiveLegacyJavaFunction(udfLoader.getUDFClass(),
HiveUdfExecutor.getRetType(request),
- HiveUdfExecutor.getParameterTypes(request));
- return new HiveUdfExecutorLegacy(request, function);
+ parameterTypes);
+ HiveUdfInputHandler inputHandler = new HiveUdfInputHandler(
+ request,
JavaUdfDataType.getTypes(function.getParameterTypes()));
+ return new HiveUdfExecutorLegacy(request, function, inputHandler);
}
case GENERIC_UDF: {
+ HiveUdfInputHandler inputHandler = new HiveUdfInputHandler(
+ request, JavaUdfDataType.getTypes(parameterTypes));
HiveGenericJavaFunction function =
new HiveGenericJavaFunction(udfLoader.getUDFClass(),
HiveUdfExecutor.getRetType(request),
- HiveUdfExecutor.getParameterTypes(request));
- return new HiveUdfExecutorGeneric(request, function);
+ parameterTypes,
+ inputHandler);
+ return new HiveUdfExecutorGeneric(request, function, inputHandler);
}
default:
throw new ImpalaRuntimeException("The class " +
request.fn.scalar_fn.symbol +
diff --git
a/fe/src/test/java/org/apache/impala/hive/executor/UdfExecutorTest.java
b/fe/src/test/java/org/apache/impala/hive/executor/UdfExecutorTest.java
index 7756293c1..5ebac912d 100644
--- a/fe/src/test/java/org/apache/impala/hive/executor/UdfExecutorTest.java
+++ b/fe/src/test/java/org/apache/impala/hive/executor/UdfExecutorTest.java
@@ -241,12 +241,14 @@ public class UdfExecutorTest {
Object[] originalArgs, Object[] args) throws ImpalaException, TException
{
int inputBufferSize = 0;
ArrayList<Integer> inputByteOffsets = Lists.newArrayList();
+ ArrayList<Boolean> isConstantArg = Lists.newArrayList();
ArrayList<Type> argTypes = Lists.newArrayList();
for (Object originalArg: originalArgs) {
Preconditions.checkNotNull(originalArg);
Type argType = getType(originalArg);
inputByteOffsets.add(Integer.valueOf(inputBufferSize));
inputBufferSize += argType.getSlotSize();
+ isConstantArg.add(false);
argTypes.add(argType);
}
@@ -266,6 +268,7 @@ public class UdfExecutorTest {
THiveUdfExecutorCtorParams params = new THiveUdfExecutorCtorParams(fn,
jarFile,
inputByteOffsets, inputNullsPtr, inputBufferPtr, outputNullPtr,
outputBufferPtr);
+ params.setIs_constant_arg(isConstantArg);
TSerializer serializer = new TSerializer(PROTOCOL_FACTORY);
return new UdfExecutor(serializer.serialize(params));
}
diff --git
a/java/test-hive-udfs/src/main/java/org/apache/impala/GenericAlltypeArgConstCheckUdf.java
b/java/test-hive-udfs/src/main/java/org/apache/impala/GenericAlltypeArgConstCheckUdf.java
new file mode 100644
index 000000000..6e6098dbe
--- /dev/null
+++
b/java/test-hive-udfs/src/main/java/org/apache/impala/GenericAlltypeArgConstCheckUdf.java
@@ -0,0 +1,116 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.impala;
+
+import java.nio.charset.StandardCharsets;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.hive.ql.exec.UDFArgumentException;
+import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredJavaObject;
+import org.apache.hadoop.hive.ql.udf.generic.GenericUDF.DeferredObject;
+import org.apache.hadoop.hive.serde2.objectinspector.ConstantObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.
+ PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.objectinspector.primitive.
+ PrimitiveObjectInspectorFactory;
+import org.apache.hadoop.io.Text;
+
+// string GenericAlltypeArgConstCheckUdf(bool, tinyint, smallint, int, bigint,
float,
+// double, string, binary)
+// The returned string prints arguments and contains info on whether they are
const.
+public class GenericAlltypeArgConstCheckUdf extends GenericUDF {
+ public GenericAlltypeArgConstCheckUdf() {
+ }
+
+ static final List<String> ARGS = Arrays.asList(
+ "BOOLEAN", "BYTE", "SHORT", "INT", "LONG",
+ "FLOAT", "DOUBLE", "STRING", "BINARY");
+
+ boolean[] isConstArg_;
+ Object[] constArgs_;
+ PrimitiveObjectInspector[] inspectors_;
+
+ @Override
+ public ObjectInspector initialize(ObjectInspector[] arguments)
+ throws UDFArgumentException {
+ if (arguments.length != ARGS.size()) {
+ throw new UDFArgumentException(
+ "GenericAlltypeArgConstCheckUdf takes 9 arguments.");
+ }
+ isConstArg_ = new boolean[arguments.length];
+ constArgs_ = new Object[arguments.length];
+ inspectors_ = new PrimitiveObjectInspector[arguments.length];
+ for (int i = 0; i < arguments.length; i++) {
+ ObjectInspector oi = arguments[i];
+ if (!(oi instanceof PrimitiveObjectInspector)) {
+ throw new UDFArgumentException("Found an input that is not a
primitive.");
+ }
+ PrimitiveObjectInspector poi = (PrimitiveObjectInspector) oi;
+ if (poi.getPrimitiveCategory() !=
PrimitiveCategory.valueOf(ARGS.get(i))) {
+ throw new UDFArgumentException("Incorrect input type");
+ }
+ inspectors_[i] = poi;
+ isConstArg_[i] = oi instanceof ConstantObjectInspector;
+ if (isConstArg_[i]) {
+ ConstantObjectInspector coi = (ConstantObjectInspector) oi;
+ constArgs_[i] = coi.getWritableConstantValue();
+ }
+ }
+ return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
+ }
+
+ @Override
+ public Object evaluate(DeferredObject[] arguments)
+ throws HiveException {
+ if (arguments.length != ARGS.size()) {
+ throw new RuntimeException("Number of expected args did not match.");
+ }
+ StringBuilder sb = new StringBuilder();
+ sb.append("Args: ");
+ for (int i = 0; i < arguments.length; i++) {
+ Object o = arguments[i].get();
+ if (isConstArg_[i] && o != constArgs_[i]) {
+ throw new RuntimeException("Got different object for const argument.");
+ }
+ sb.append(isConstArg_[i] ? "const " : "non-const ");
+ if (o == null) {
+ sb.append("null");
+ } else {
+ Object jo = inspectors_[i].getPrimitiveJavaObject(o);
+ if (jo instanceof byte[]) {
+ jo = new String((byte[])jo, StandardCharsets.UTF_8);
+ }
+ sb.append(jo.toString());
+ }
+ sb.append("; ");
+ }
+ Text resultString = new Text();
+ resultString.set(sb.toString());
+ return resultString;
+ }
+
+ @Override
+ public String getDisplayString(String[] children) {
+ return "GenericAlltypeArgConstCheckUdf";
+ }
+}
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/generic-java-udf.test
b/testdata/workloads/functional-query/queries/QueryTest/generic-java-udf.test
index 3bd35e8ed..d5752ccf8 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/generic-java-udf.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/generic-java-udf.test
@@ -363,3 +363,75 @@ BINARY
---- RESULTS
'b'
====
+---- QUERY
+select alltypeargs_constcheck(bool_col, tinyint_col, smallint_col, int_col,
bigint_col,
+ float_col, double_col, string_col, cast(string_col as
binary))
+from functional.alltypestiny;
+---- TYPES
+STRING
+---- RESULTS
+'Args: non-const true; non-const 0; non-const 0; non-const 0; non-const 0;
non-const 0.0; non-const 0.0; non-const 0; non-const 0; '
+'Args: non-const false; non-const 1; non-const 1; non-const 1; non-const 10;
non-const 1.1; non-const 10.1; non-const 1; non-const 1; '
+'Args: non-const true; non-const 0; non-const 0; non-const 0; non-const 0;
non-const 0.0; non-const 0.0; non-const 0; non-const 0; '
+'Args: non-const false; non-const 1; non-const 1; non-const 1; non-const 10;
non-const 1.1; non-const 10.1; non-const 1; non-const 1; '
+'Args: non-const true; non-const 0; non-const 0; non-const 0; non-const 0;
non-const 0.0; non-const 0.0; non-const 0; non-const 0; '
+'Args: non-const false; non-const 1; non-const 1; non-const 1; non-const 10;
non-const 1.1; non-const 10.1; non-const 1; non-const 1; '
+'Args: non-const true; non-const 0; non-const 0; non-const 0; non-const 0;
non-const 0.0; non-const 0.0; non-const 0; non-const 0; '
+'Args: non-const false; non-const 1; non-const 1; non-const 1; non-const 10;
non-const 1.1; non-const 10.1; non-const 1; non-const 1; '
+====
+---- QUERY
+select alltypeargs_constcheck(true, 1, 2, 3, 4,
+ float_col, double_col, string_col, cast(string_col as
binary))
+from functional.alltypestiny;
+---- TYPES
+STRING
+---- RESULTS
+'Args: const true; const 1; const 2; const 3; const 4; non-const 0.0;
non-const 0.0; non-const 0; non-const 0; '
+'Args: const true; const 1; const 2; const 3; const 4; non-const 1.1;
non-const 10.1; non-const 1; non-const 1; '
+'Args: const true; const 1; const 2; const 3; const 4; non-const 0.0;
non-const 0.0; non-const 0; non-const 0; '
+'Args: const true; const 1; const 2; const 3; const 4; non-const 1.1;
non-const 10.1; non-const 1; non-const 1; '
+'Args: const true; const 1; const 2; const 3; const 4; non-const 0.0;
non-const 0.0; non-const 0; non-const 0; '
+'Args: const true; const 1; const 2; const 3; const 4; non-const 1.1;
non-const 10.1; non-const 1; non-const 1; '
+'Args: const true; const 1; const 2; const 3; const 4; non-const 0.0;
non-const 0.0; non-const 0; non-const 0; '
+'Args: const true; const 1; const 2; const 3; const 4; non-const 1.1;
non-const 10.1; non-const 1; non-const 1; '
+====
+---- QUERY
+select alltypeargs_constcheck(bool_col, tinyint_col, smallint_col, int_col,
bigint_col,
+ 5.0, 6.0, "const string", cast("const string 2" as binary))
+from functional.alltypestiny;
+---- TYPES
+STRING
+---- RESULTS
+'Args: non-const true; non-const 0; non-const 0; non-const 0; non-const 0;
const 5.0; const 6.0; const const string; const const string 2; '
+'Args: non-const false; non-const 1; non-const 1; non-const 1; non-const 10;
const 5.0; const 6.0; const const string; const const string 2; '
+'Args: non-const true; non-const 0; non-const 0; non-const 0; non-const 0;
const 5.0; const 6.0; const const string; const const string 2; '
+'Args: non-const false; non-const 1; non-const 1; non-const 1; non-const 10;
const 5.0; const 6.0; const const string; const const string 2; '
+'Args: non-const true; non-const 0; non-const 0; non-const 0; non-const 0;
const 5.0; const 6.0; const const string; const const string 2; '
+'Args: non-const false; non-const 1; non-const 1; non-const 1; non-const 10;
const 5.0; const 6.0; const const string; const const string 2; '
+'Args: non-const true; non-const 0; non-const 0; non-const 0; non-const 0;
const 5.0; const 6.0; const const string; const const string 2; '
+'Args: non-const false; non-const 1; non-const 1; non-const 1; non-const 10;
const 5.0; const 6.0; const const string; const const string 2; '
+====
+---- QUERY
+select alltypeargs_constcheck(true, tinyint_col, 2, int_col, 3,
+ float_col, 4.0, string_col, cast("const string" as binary))
+from functional.alltypestiny;
+---- TYPES
+STRING
+---- RESULTS
+'Args: const true; non-const 0; const 2; non-const 0; const 3; non-const 0.0;
const 4.0; non-const 0; const const string; '
+'Args: const true; non-const 1; const 2; non-const 1; const 3; non-const 1.1;
const 4.0; non-const 1; const const string; '
+'Args: const true; non-const 0; const 2; non-const 0; const 3; non-const 0.0;
const 4.0; non-const 0; const const string; '
+'Args: const true; non-const 1; const 2; non-const 1; const 3; non-const 1.1;
const 4.0; non-const 1; const const string; '
+'Args: const true; non-const 0; const 2; non-const 0; const 3; non-const 0.0;
const 4.0; non-const 0; const const string; '
+'Args: const true; non-const 1; const 2; non-const 1; const 3; non-const 1.1;
const 4.0; non-const 1; const const string; '
+'Args: const true; non-const 0; const 2; non-const 0; const 3; non-const 0.0;
const 4.0; non-const 0; const const string; '
+'Args: const true; non-const 1; const 2; non-const 1; const 3; non-const 1.1;
const 4.0; non-const 1; const const string; '
+====
+---- QUERY
+select alltypeargs_constcheck(true, 1, 2, 3, 4,
+ 5.0, 6.0, "const string", cast("const string 2" as binary));
+---- TYPES
+STRING
+---- RESULTS
+'Args: const true; const 1; const 2; const 3; const 4; const 5.0; const 6.0;
const const string; const const string 2; '
+====
\ No newline at end of file
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/java-udf.test
b/testdata/workloads/functional-query/queries/QueryTest/java-udf.test
index 4b16405c3..856e8ebfc 100644
--- a/testdata/workloads/functional-query/queries/QueryTest/java-udf.test
+++ b/testdata/workloads/functional-query/queries/QueryTest/java-udf.test
@@ -377,3 +377,22 @@ STRING
---- RESULTS
'baaa'
====
+---- QUERY
+select identity(bool_col), identity(tinyint_col),
+ identity(smallint_col), identity(int_col),
+ identity(bigint_col), identity(float_col),
+ identity(double_col), identity(string_col),
+ identity(cast(string_col as binary))
+from functional.alltypestiny;
+---- TYPES
+boolean, tinyint, smallint, int, bigint, float, double, string, binary
+---- RESULTS
+true,0,0,0,0,0.0,0.0,'0','0'
+false,1,1,1,10,1.100000023841858,10.1,'1','1'
+true,0,0,0,0,0.0,0.0,'0','0'
+false,1,1,1,10,1.100000023841858,10.1,'1','1'
+true,0,0,0,0,0.0,0.0,'0','0'
+false,1,1,1,10,1.100000023841858,10.1,'1','1'
+true,0,0,0,0,0.0,0.0,'0','0'
+false,1,1,1,10,1.100000023841858,10.1,'1','1'
+====
\ No newline at end of file
diff --git
a/testdata/workloads/functional-query/queries/QueryTest/load-generic-java-udfs.test
b/testdata/workloads/functional-query/queries/QueryTest/load-generic-java-udfs.test
index e029a1a59..ae70693ef 100644
---
a/testdata/workloads/functional-query/queries/QueryTest/load-generic-java-udfs.test
+++
b/testdata/workloads/functional-query/queries/QueryTest/load-generic-java-udfs.test
@@ -147,4 +147,8 @@ symbol='org.apache.impala.GenericBufferAlteringUdf';
create function increment(string) returns string
location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
symbol='org.apache.impala.GenericBufferAlteringUdf';
+
+create function alltypeargs_constcheck(boolean, tinyint, smallint, int,
bigint, float, double, string, binary) returns string
+location '$FILESYSTEM_PREFIX/test-warehouse/impala-hive-udfs.jar'
+symbol='org.apache.impala.GenericAlltypeArgConstCheckUdf';
====
