Repository: spark Updated Branches: refs/heads/branch-1.6 b5e5812f9 -> f939c71b1
[SPARK-12242][SQL] Add DataFrame.transform method Author: Reynold Xin <[email protected]> Closes #10226 from rxin/df-transform. (cherry picked from commit 76540b6df5370b463277d3498097b2cc2d2e97a8) Signed-off-by: Reynold Xin <[email protected]> Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f939c71b Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f939c71b Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f939c71b Branch: refs/heads/branch-1.6 Commit: f939c71b187cff3a5bb63aa3659429b6efb0626d Parents: b5e5812 Author: Reynold Xin <[email protected]> Authored: Thu Dec 10 22:23:10 2015 +0800 Committer: Reynold Xin <[email protected]> Committed: Thu Dec 10 22:23:26 2015 +0800 ---------------------------------------------------------------------- .../src/main/scala/org/apache/spark/sql/Column.scala | 2 +- .../main/scala/org/apache/spark/sql/DataFrame.scala | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/f939c71b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala index d641fca..297ef22 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala @@ -84,7 +84,7 @@ class TypedColumn[-T, U]( * col("`a.column.with.dots`") // Escape `.` in column names. * $"columnName" // Scala short hand for a named column. * expr("a + 1") // A column that is constructed from a parsed SQL Expression. - * lit("1") // A column that produces a literal (constant) value. + * lit("abc") // A column that produces a literal (constant) value. * }}} * * [[Column]] objects can be composed to form complex expressions: http://git-wip-us.apache.org/repos/asf/spark/blob/f939c71b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala index eb87003..1acfe84 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala @@ -1414,6 +1414,19 @@ class DataFrame private[sql]( def first(): Row = head() /** + * Concise syntax for chaining custom transformations. + * {{{ + * def featurize(ds: DataFrame) = ... + * + * df + * .transform(featurize) + * .transform(...) + * }}} + * @since 1.6.0 + */ + def transform[U](t: DataFrame => DataFrame): DataFrame = t(this) + + /** * Returns a new RDD by applying a function to all rows of this DataFrame. * @group rdd * @since 1.3.0 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
