Sudarshan Lamkhede created SPARK-15595:
------------------------------------------
Summary: DataFrame.write.mode(SaveMode.Append).insertInto(TABLE)
overwrites non-partitioned table
Key: SPARK-15595
URL: https://issues.apache.org/jira/browse/SPARK-15595
Project: Spark
Issue Type: Bug
Components: SQL
Affects Versions: 1.5.2
Reporter: Sudarshan Lamkhede
See the examples below
<verbatim>
scala> sqlContext.sql("""CREATE TABLE IF NOT EXISTS noparts (model_name STRING,
dateint INT) STORED AS PARQUET""")
res0: org.apache.spark.sql.DataFrame = [result: string]
scala> sqlContext.sql("""CREATE TABLE IF NOT EXISTS parts (model_name STRING)
PARTITIONED BY (dateint INT) STORED AS PARQUET""")
res1: org.apache.spark.sql.DataFrame = [result: string]
scala> sqlContext.sql("select * from noparts").show()
+----------+-------+
|model_name|dateint|
+----------+-------+
+----------+-------+
scala> sqlContext.sql("select * from parts").show()
+----------+-------+
|model_name|dateint|
+----------+-------+
+----------+-------+
scala> import sqlContext.implicits._
import sqlContext.implicits._
scala> val df1 = sc.parallelize(Array(("before", 1)), 1).toDF("model_name",
"dateint")
df1: org.apache.spark.sql.DataFrame = [model_name: string, dateint: int]
scala> val df2 = sc.parallelize(Array(("after", 2)), 1).toDF("model_name",
"dateint")
df2: org.apache.spark.sql.DataFrame = [model_name: string, dateint: int]
scala> import org.apache.spark.sql.SaveMode
import org.apache.spark.sql.SaveMode
scala> df1.write.mode(SaveMode.Append).insertInto("noparts")
scala> sqlContext.sql("select * from noparts").show()
+----------+-------+
|model_name|dateint|
+----------+-------+
| before| 1|
+----------+-------+
scala> df2.write.mode(SaveMode.Append).insertInto("noparts")
scala> sqlContext.sql("select * from noparts").show()
+----------+-------+
|model_name|dateint|
+----------+-------+
| after| 2|
+----------+-------+
scala> df1.write.mode(SaveMode.Append).insertInto("parts")
scala> sqlContext.sql("select * from parts").show()
+----------+-------+
|model_name|dateint|
+----------+-------+
| before| 1|
+----------+-------+
scala> df2.write.mode(SaveMode.Append).insertInto("parts")
scala> sqlContext.sql("select * from parts").show()
+----------+-------+
|model_name|dateint|
+----------+-------+
| before| 1|
| after| 2|
+----------+-------+
</verbatim>
--
This message was sent by Atlassian JIRA
(v6.3.4#6332)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
