Repository: spark Updated Branches: refs/heads/master 21b3d2a75 -> 76540b6df
[SPARK-12242][SQL] Add DataFrame.transform method Author: Reynold Xin <r...@databricks.com> Closes #10226 from rxin/df-transform. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/76540b6d Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/76540b6d Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/76540b6d Branch: refs/heads/master Commit: 76540b6df5370b463277d3498097b2cc2d2e97a8 Parents: 21b3d2a Author: Reynold Xin <r...@databricks.com> Authored: Thu Dec 10 22:23:10 2015 +0800 Committer: Reynold Xin <r...@databricks.com> Committed: Thu Dec 10 22:23:10 2015 +0800 ---------------------------------------------------------------------- .../src/main/scala/org/apache/spark/sql/Column.scala | 2 +- .../main/scala/org/apache/spark/sql/DataFrame.scala | 13 +++++++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/76540b6d/sql/core/src/main/scala/org/apache/spark/sql/Column.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala index d641fca..297ef22 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/Column.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/Column.scala @@ -84,7 +84,7 @@ class TypedColumn[-T, U]( * col("`a.column.with.dots`") // Escape `.` in column names. * $"columnName" // Scala short hand for a named column. * expr("a + 1") // A column that is constructed from a parsed SQL Expression. - * lit("1") // A column that produces a literal (constant) value. + * lit("abc") // A column that produces a literal (constant) value. * }}} * * [[Column]] objects can be composed to form complex expressions: http://git-wip-us.apache.org/repos/asf/spark/blob/76540b6d/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala ---------------------------------------------------------------------- diff --git a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala index 243a8c8..da180a2 100644 --- a/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala +++ b/sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala @@ -1422,6 +1422,19 @@ class DataFrame private[sql]( def first(): Row = head() /** + * Concise syntax for chaining custom transformations. + * {{{ + * def featurize(ds: DataFrame) = ... + * + * df + * .transform(featurize) + * .transform(...) + * }}} + * @since 1.6.0 + */ + def transform[U](t: DataFrame => DataFrame): DataFrame = t(this) + + /** * Returns a new RDD by applying a function to all rows of this DataFrame. * @group rdd * @since 1.3.0 --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org
