Github user rxin commented on a diff in the pull request:
https://github.com/apache/spark/pull/4294#discussion_r23961981
--- Diff: sql/core/src/main/scala/org/apache/spark/sql/DataFrame.scala ---
@@ -618,8 +619,84 @@ class DataFrame protected[sql](
*/
@Experimental
override def saveAsTable(tableName: String): Unit = {
- sqlContext.executePlan(
- CreateTableAsSelect(None, tableName, logicalPlan, allowExisting =
false)).toRdd
+ val dataSourceName = sqlContext.conf.defaultDataSourceName
+ saveAsTable(tableName, dataSourceName, ("path",
sqlContext.defaultTableFilePath(tableName)))
+ }
+
+ /**
+ * :: Experimental ::
+ * Creates a table from the the contents of this DataFrame based on a
given data source and
+ * a set of options. This will fail if the table already exists.
+ *
+ * Note that this currently only works with DataFrames that are created
from a HiveContext as
+ * there is no notion of a persisted catalog in a standard SQL context.
Instead you can write
+ * an RDD out to a parquet file, and then register that file as a table.
This "table" can then
+ * be the target of an `insertInto`.
+ */
+ @Experimental
+ override def saveAsTable(
+ tableName: String,
+ dataSourceName: String,
+ options: (String, String)*): Unit = {
+ val cmd =
+ CreateTableUsingAsLogicalPlan(
+ tableName,
+ dataSourceName,
+ temporary = false,
+ options.toMap,
+ allowExisting = false,
+ logicalPlan)
+
+ sqlContext.executePlan(cmd).toRdd
+ }
+
+ /**
+ * :: Experimental ::
+ * Creates a table from the the contents of this DataFrame based on a
given data source and
+ * a set of options. This will fail if the table already exists.
+ *
+ * Note that this currently only works with DataFrames that are created
from a HiveContext as
+ * there is no notion of a persisted catalog in a standard SQL context.
Instead you can write
+ * an RDD out to a parquet file, and then register that file as a table.
This "table" can then
+ * be the target of an `insertInto`.
+ */
+ @Experimental
+ override def saveAsTable(
+ tableName: String,
+ dataSourceName: String,
+ options: java.util.Map[String, String]): Unit = {
+ saveAsTable(tableName, dataSourceName, options.toSeq:_*)
+ }
+
+ @Experimental
+ override def save(path: String): Unit = {
+ save(path, false)
+ }
+
+ @Experimental
+ override def save(path: String, overwrite: Boolean): Unit = {
+ val dataSourceName = sqlContext.conf.defaultDataSourceName
+ save(dataSourceName, overwrite, ("path" -> path))
+ }
+
+ @Experimental
+ override def save(
+ dataSourceName: String,
+ overwrite: Boolean,
+ options: (String, String)*): Unit = {
--- End diff --
here too. let's disambiguate it by having option: (String, String) followed
by options: (String, String)
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
