Github user liancheng commented on a diff in the pull request:
https://github.com/apache/spark/pull/6710#discussion_r31991365
--- Diff:
sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/FunctionRegistry.scala
---
@@ -70,30 +76,89 @@ object EmptyFunctionRegistry extends FunctionRegistry {
override def lookupFunction(name: String, children: Seq[Expression]):
Expression = {
throw new UnsupportedOperationException
}
-
- override def conf: CatalystConf = throw new UnsupportedOperationException
}
-/**
- * Build a map with String type of key, and it also supports either key
case
- * sensitive or insensitive.
- * TODO move this into util folder?
- */
-object StringKeyHashMap {
- def apply[T](caseSensitive: Boolean): StringKeyHashMap[T] =
caseSensitive match {
- case false => new StringKeyHashMap[T](_.toLowerCase)
- case true => new StringKeyHashMap[T](identity)
- }
-}
-class StringKeyHashMap[T](normalizer: (String) => String) {
- private val base = new collection.mutable.HashMap[String, T]()
+object FunctionRegistry {
- def apply(key: String): T = base(normalizer(key))
+ type FunctionBuilder = Seq[Expression] => Expression
- def get(key: String): Option[T] = base.get(normalizer(key))
- def put(key: String, value: T): Option[T] = base.put(normalizer(key),
value)
- def remove(key: String): Option[T] = base.remove(normalizer(key))
- def iterator: Iterator[(String, T)] = base.toIterator
+ val expressions: Map[String, FunctionBuilder] = Map(
+ // Non aggregate functions
+ expression[Abs]("abs"),
+ expression[CreateArray]("array"),
+ expression[Coalesce]("coalesce"),
+ expression[Explode]("explode"),
+ expression[Lower]("lower"),
+ expression[Substring]("substr"),
+ expression[Substring]("substring"),
+ expression[Rand]("rand"),
+ expression[Randn]("randn"),
+ expression[CreateStruct]("struct"),
+ expression[Sqrt]("sqrt"),
+ expression[Upper]("upper"),
+
+ // Math functions
+ expression[Acos]("acos"),
+ expression[Asin]("asin"),
+ expression[Atan]("atan"),
+ expression[Atan2]("atan2"),
+ expression[Cbrt]("cbrt"),
+ expression[Ceil]("ceil"),
+ expression[Cos]("cos"),
+ expression[Exp]("exp"),
+ expression[Expm1]("expm1"),
+ expression[Floor]("floor"),
+ expression[Hypot]("hypot"),
+ expression[Log]("log"),
+ expression[Log10]("log10"),
+ expression[Log1p]("log1p"),
+ expression[Pow]("pow"),
+ expression[Rint]("rint"),
+ expression[Signum]("signum"),
+ expression[Sin]("sin"),
+ expression[Sinh]("sinh"),
+ expression[Tan]("tan"),
+ expression[Tanh]("tanh"),
+ expression[ToDegrees]("todegrees"),
+ expression[ToRadians]("toradians"),
+
+ // aggregate functions
+ expression[Average]("avg"),
+ expression[Count]("count"),
+ expression[First]("first"),
+ expression[Last]("last"),
+ expression[Max]("max"),
+ expression[Min]("min"),
+ expression[Sum]("sum")
+ )
+
+ /** See usage above. */
+ private def expression[T <: Expression](name: String)
+ (implicit tag: ClassTag[T]): (String, FunctionBuilder) = {
+ // Use the companion class to find apply methods.
+ val objectClass = Class.forName(tag.runtimeClass.getName + "$")
+ val companionObj = objectClass.getDeclaredField("MODULE$").get(null)
+
+ // See if we can find an apply that accepts Seq[Expression]
+ val varargApply = Try(objectClass.getDeclaredMethod("apply",
classOf[Seq[_]])).toOption
+
+ val builder = (expressions: Seq[Expression]) => {
+ if (varargApply.isDefined) {
+ // If there is an apply method that accepts Seq[Expression], use
that one.
+ varargApply.get.invoke(companionObj,
expressions).asInstanceOf[Expression]
+ } else {
+ // Otherwise, find an apply method that matches the number of
arguments, and use that.
+ val params = Seq.fill(expressions.size)(classOf[Expression])
+ val f = Try(objectClass.getDeclaredMethod("apply", params : _*))
match {
--- End diff --
OT: We can add this to `ScalaReflection` to cover Scrooge flavor code.
Scrooge generates a trait extending `Product` plus an `apply` method in its
companion object for a Thrift struct. Since `ScalaReflection` only looks for
case class constructor, Scala types generated by Scrooge can't be used to infer
Spark SQL schema. Details can be found in [PARQUET-293] [1].
[1]:
https://issues.apache.org/jira/browse/PARQUET-293?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=14577242#comment-14577242
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
