Repository: spark
Updated Branches:
refs/heads/master 424cb699e -> 417d1118c
[SPARK-5367][SQL] Support star expression in udfs
A follow up for #4163: support `select array(key, *) from src`
Since array(key, *) will not go into this case
```
case Alias(f UnresolvedFunction(_, args), name) if containsStar(args) =>
val expandedArgs = args.flatMap {
case s: Star => s.expand(child.output, resolver)
case o => o :: Nil
}
```
here added a case to cover the corner case of array.
/cc liancheng
Author: wangfei <[email protected]>
Author: scwf <[email protected]>
Closes #4353 from scwf/udf-star1 and squashes the following commits:
4350d17 [wangfei] minor fix
a7cd191 [wangfei] minor fix
0942fb1 [wangfei] follow up: support select array(key, *) from src
6ae00db [wangfei] also fix problem with array
da1da09 [scwf] minor fix
f87b5f9 [scwf] added test case
587bf7e [wangfei] compile fix
eb93c16 [wangfei] fix star resolve issue in udf
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/417d1118
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/417d1118
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/417d1118
Branch: refs/heads/master
Commit: 417d1118cd720b616bfd30e9c1a0bfb1838e8cbd
Parents: 424cb69
Author: wangfei <[email protected]>
Authored: Wed Feb 4 15:12:07 2015 -0800
Committer: Michael Armbrust <[email protected]>
Committed: Wed Feb 4 15:12:07 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/catalyst/analysis/Analyzer.scala | 6 ++++++
.../org/apache/spark/sql/hive/execution/HiveQuerySuite.scala | 2 ++
2 files changed, 8 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/417d1118/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
----------------------------------------------------------------------
diff --git
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
index ae7f7b9..ae1aee0 100644
---
a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
+++
b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/analysis/Analyzer.scala
@@ -257,6 +257,12 @@ class Analyzer(catalog: Catalog,
case o => o :: Nil
}
Alias(child = f.copy(children = expandedArgs), name)() :: Nil
+ case Alias(c @ CreateArray(args), name) if containsStar(args) =>
+ val expandedArgs = args.flatMap {
+ case s: Star => s.expand(child.output, resolver)
+ case o => o :: Nil
+ }
+ Alias(c.copy(children = expandedArgs), name)() :: Nil
case o => o :: Nil
},
child)
http://git-wip-us.apache.org/repos/asf/spark/blob/417d1118/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
----------------------------------------------------------------------
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
index 82efadb..a321452 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/execution/HiveQuerySuite.scala
@@ -620,6 +620,8 @@ class HiveQuerySuite extends HiveComparisonTest with
BeforeAndAfter {
test("SPARK-5367: resolve star expression in udf") {
assert(sql("select concat(*) from src limit 5").collect().size == 5)
assert(sql("select array(*) from src limit 5").collect().size == 5)
+ assert(sql("select concat(key, *) from src limit 5").collect().size == 5)
+ assert(sql("select array(key, *) from src limit 5").collect().size == 5)
}
test("Query Hive native command execution result") {
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
