Github user gatorsmile commented on a diff in the pull request:
https://github.com/apache/spark/pull/17848#discussion_r128626143
--- Diff:
sql/core/src/main/scala/org/apache/spark/sql/UDFRegistration.scala ---
@@ -130,460 +138,507 @@ class UDFRegistration private[sql]
(functionRegistry: FunctionRegistry) extends
val version = if (i == 0) "2.3.0" else "1.3.0"
val funcCall = if (i == 0) "() => func" else "func"
println(s"""
- |/**
- | * Register a user-defined function with ${i} arguments.
- | * @since $version
- | */
- |def register(name: String, f: UDF$i[$extTypeArgs], returnType:
DataType): Unit = {
- | val func = f$anyCast.call($anyParams)
- | def builder(e: Seq[Expression]) = if (e.length == $i) {
- | ScalaUDF($funcCall, returnType, e)
- | } else {
- | throw new AnalysisException("Invalid number of arguments for
function " + name +
- | ". Expected: $i; Found: " + e.length)
- | }
- | functionRegistry.createOrReplaceTempFunction(name, builder)
- |}""".stripMargin)
+ |/**
+ | * Registers a deterministic user-defined function with ${i}
arguments.
+ | * @since $version
+ | */
+ |def register(name: String, f: UDF$i[$extTypeArgs], returnType:
DataType): Unit = {
+ | register(name, f, returnType, deterministic = true)
+ |}
+ |
+ |/**
+ | * Registers a user-defined function with ${i} arguments.
+ | * @since 2.3.0
+ | */
+ |def register(name: String, f: UDF$i[$extTypeArgs], returnType:
DataType, deterministic: Boolean): Unit = {
+ | val func = f$anyCast.call($anyParams)
+ | def builder(e: Seq[Expression]) = if (e.length == $i) {
+ | ScalaUDF($funcCall, returnType, e, inputTypes = Nil, udfName
= Some(name), nullable = true, deterministic)
+ | } else {
+ | throw new AnalysisException("Invalid number of arguments for
function " + name +
+ | ". Expected: $i; Found: " + e.length)
+ | }
+ | functionRegistry.createOrReplaceTempFunction(name, builder)
+ |}""".stripMargin)
}
*/
/**
- * Register a Scala closure of 0 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 0 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag](name: String, func: Function0[RT]):
UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 0) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 0; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 1 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 1 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag](name: String, func: Function1[A1,
RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 1) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 1; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 2 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 2 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag](name: String, func:
Function2[A1, A2, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 2) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 2; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 3 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 3 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag](name:
String, func: Function3[A1, A2, A3, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 3) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 3; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 4 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 4 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag](name: String, func: Function4[A1, A2, A3, A4, RT]):
UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 4) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 4; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 5 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 5 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag](name: String, func: Function5[A1, A2, A3, A4, A5, RT]):
UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 5) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 5; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 6 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 6 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag](name: String, func: Function6[A1, A2, A3,
A4, A5, A6, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 6) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 6; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 7 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 7 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag](name: String, func:
Function7[A1, A2, A3, A4, A5, A6, A7, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 7) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 7; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 8 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 8 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag](name: String,
func: Function8[A1, A2, A3, A4, A5, A6, A7, A8, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 8) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 8; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 9 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 9 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag](name:
String, func: Function9[A1, A2, A3, A4, A5, A6, A7, A8, A9, RT]):
UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 9) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 9; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 10 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 10 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag](name: String, func: Function10[A1, A2, A3, A4, A5, A6, A7, A8, A9,
A10, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 10) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 10; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 11 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 11 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag](name: String, func: Function11[A1, A2, A3, A4, A5, A6,
A7, A8, A9, A10, A11, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 11) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 11; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 12 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 12 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag](name: String, func: Function12[A1, A2, A3,
A4, A5, A6, A7, A8, A9, A10, A11, A12, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 12) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 12; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 13 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 13 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag](name: String, func:
Function13[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, RT]):
UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 13) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 13; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 14 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 14 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag](name: String,
func: Function14[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14,
RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 14) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 14; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 15 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 15 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15:
TypeTag](name: String, func: Function15[A1, A2, A3, A4, A5, A6, A7, A8, A9,
A10, A11, A12, A13, A14, A15, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType ::
ScalaReflection.schemaFor[A15].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 15) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 15; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 16 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 16 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag,
A16: TypeTag](name: String, func: Function16[A1, A2, A3, A4, A5, A6, A7, A8,
A9, A10, A11, A12, A13, A14, A15, A16, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType ::
ScalaReflection.schemaFor[A15].dataType ::
ScalaReflection.schemaFor[A16].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 16) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 16; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 17 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 17 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag,
A16: TypeTag, A17: TypeTag](name: String, func: Function17[A1, A2, A3, A4, A5,
A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, RT]):
UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType ::
ScalaReflection.schemaFor[A15].dataType ::
ScalaReflection.schemaFor[A16].dataType ::
ScalaReflection.schemaFor[A17].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 17) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 17; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 18 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 18 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag,
A16: TypeTag, A17: TypeTag, A18: TypeTag](name: String, func: Function18[A1,
A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18,
RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType ::
ScalaReflection.schemaFor[A15].dataType ::
ScalaReflection.schemaFor[A16].dataType ::
ScalaReflection.schemaFor[A17].dataType ::
ScalaReflection.schemaFor[A18].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 18) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 18; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 19 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 19 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag,
A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag](name: String, func:
Function19[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12, A13, A14, A15,
A16, A17, A18, A19, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType ::
ScalaReflection.schemaFor[A15].dataType ::
ScalaReflection.schemaFor[A16].dataType ::
ScalaReflection.schemaFor[A17].dataType ::
ScalaReflection.schemaFor[A18].dataType ::
ScalaReflection.schemaFor[A19].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 19) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 19; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 20 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 20 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag,
A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag](name:
String, func: Function20[A1, A2, A3, A4, A5, A6, A7, A8, A9, A10, A11, A12,
A13, A14, A15, A16, A17, A18, A19, A20, RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType ::
ScalaReflection.schemaFor[A15].dataType ::
ScalaReflection.schemaFor[A16].dataType ::
ScalaReflection.schemaFor[A17].dataType ::
ScalaReflection.schemaFor[A18].dataType ::
ScalaReflection.schemaFor[A19].dataType ::
ScalaReflection.schemaFor[A20].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 20) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 20; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 21 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 21 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag,
A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21:
TypeTag](name: String, func: Function21[A1, A2, A3, A4, A5, A6, A7, A8, A9,
A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, RT]):
UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType ::
ScalaReflection.schemaFor[A15].dataType ::
ScalaReflection.schemaFor[A16].dataType ::
ScalaReflection.schemaFor[A17].dataType ::
ScalaReflection.schemaFor[A18].dataType ::
ScalaReflection.schemaFor[A19].dataType ::
ScalaReflection.schemaFor[A20].dataType ::
ScalaReflection.schemaFor[A21].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 21) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 21; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
/**
- * Register a Scala closure of 22 arguments as user-defined function
(UDF).
+ * Registers a deterministic Scala closure of 22 arguments as
user-defined function (UDF).
* @tparam RT return type of UDF.
* @since 1.3.0
*/
def register[RT: TypeTag, A1: TypeTag, A2: TypeTag, A3: TypeTag, A4:
TypeTag, A5: TypeTag, A6: TypeTag, A7: TypeTag, A8: TypeTag, A9: TypeTag, A10:
TypeTag, A11: TypeTag, A12: TypeTag, A13: TypeTag, A14: TypeTag, A15: TypeTag,
A16: TypeTag, A17: TypeTag, A18: TypeTag, A19: TypeTag, A20: TypeTag, A21:
TypeTag, A22: TypeTag](name: String, func: Function22[A1, A2, A3, A4, A5, A6,
A7, A8, A9, A10, A11, A12, A13, A14, A15, A16, A17, A18, A19, A20, A21, A22,
RT]): UserDefinedFunction = {
val ScalaReflection.Schema(dataType, nullable) =
ScalaReflection.schemaFor[RT]
val inputTypes = Try(ScalaReflection.schemaFor[A1].dataType ::
ScalaReflection.schemaFor[A2].dataType ::
ScalaReflection.schemaFor[A3].dataType ::
ScalaReflection.schemaFor[A4].dataType ::
ScalaReflection.schemaFor[A5].dataType ::
ScalaReflection.schemaFor[A6].dataType ::
ScalaReflection.schemaFor[A7].dataType ::
ScalaReflection.schemaFor[A8].dataType ::
ScalaReflection.schemaFor[A9].dataType ::
ScalaReflection.schemaFor[A10].dataType ::
ScalaReflection.schemaFor[A11].dataType ::
ScalaReflection.schemaFor[A12].dataType ::
ScalaReflection.schemaFor[A13].dataType ::
ScalaReflection.schemaFor[A14].dataType ::
ScalaReflection.schemaFor[A15].dataType ::
ScalaReflection.schemaFor[A16].dataType ::
ScalaReflection.schemaFor[A17].dataType ::
ScalaReflection.schemaFor[A18].dataType ::
ScalaReflection.schemaFor[A19].dataType ::
ScalaReflection.schemaFor[A20].dataType ::
ScalaReflection.schemaFor[A21].dataType ::
ScalaReflection.schemaFor[A22].dataType :: Nil).toOption
def builder(e: Seq[Expression]) = if (e.length == 22) {
- ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable)
+ ScalaUDF(func, dataType, e, inputTypes.getOrElse(Nil), Some(name),
nullable, udfDeterministic = true)
} else {
throw new AnalysisException("Invalid number of arguments for
function " + name +
". Expected: 22; Found: " + e.length)
}
functionRegistry.createOrReplaceTempFunction(name, builder)
- UserDefinedFunction(func, dataType,
inputTypes).withName(name).withNullability(nullable)
+ val udf = UserDefinedFunction(func, dataType,
inputTypes).withName(name)
+ if (nullable) udf else udf.asNonNullabe()
}
//////////////////////////////////////////////////////////////////////////////////////////////
//////////////////////////////////////////////////////////////////////////////////////////////
/**
- * Register a Java UDF class using reflection, for use from pyspark
+ * Registers a Java UDF class using reflection, for use from pyspark
*
* @param name udf name
* @param className fully qualified class name of udf
* @param returnDataType return type of udf. If it is null, spark would
try to infer
* via reflection.
*/
private[sql] def registerJava(name: String, className: String,
returnDataType: DataType): Unit = {
-
+ registerJava(name, className, returnDataType, deterministic = true)
+ }
+ /**
+ * Registers a Java UDF class using reflection, for use from pyspark
+ *
+ * @param name udf name
+ * @param className fully qualified class name of udf
+ * @param returnDataType return type of udf. If it is null, spark would
try to infer
+ * via reflection.
+ * @param deterministic True if the UDF is deterministic. Deterministic
UDF returns same result
+ * each time it is invoked with a particular input.
+ */
+ private[sql] def registerJava(
--- End diff --
Actually, our JAVA API directly uses it. To JAVA APIs, they are not private
at all.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
