Github user vinodkc commented on a diff in the pull request:
https://github.com/apache/spark/pull/19779#discussion_r152446382
--- Diff:
sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala ---
@@ -841,6 +841,75 @@ class VersionsSuite extends SparkFunSuite with Logging
{
}
}
+ test(s"$version: SPARK-17920: Insert into/overwrite external avro
table") {
+ withTempDir { dir =>
+ val path = dir.getAbsolutePath
+ val schemaPath = s"""$path${File.separator}avroschemadir"""
+
+ new File(schemaPath).mkdir()
+ val avroSchema =
+ """{
+ | "name": "test_record",
+ | "type": "record",
+ | "fields": [ {
+ | "name": "f0",
+ | "type": [
+ | "null",
+ | {
+ | "precision": 38,
+ | "scale": 2,
+ | "type": "bytes",
+ | "logicalType": "decimal"
+ | }
+ | ]
+ | } ]
+ |}
+ """.stripMargin
+ val schemaurl = s"""$schemaPath${File.separator}avroDecimal.avsc"""
+ new java.io.PrintWriter(schemaurl) { write(avroSchema); close() }
+ val url =
Thread.currentThread().getContextClassLoader.getResource("avroDecimal")
+ val srcLocation = new File(url.getFile)
+ val destTableName = "tab1"
+ val srcTableName = "tab2"
+
+ withTable(srcTableName, destTableName) {
+ versionSpark.sql(
+ s"""
+ |CREATE TABLE $srcTableName
+ |ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+ |WITH SERDEPROPERTIES ('respectSparkSchema' = 'true')
+ |STORED AS
+ | INPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+ | OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+ |LOCATION '$srcLocation'
+ |TBLPROPERTIES ('avro.schema.url' = '$schemaurl')
+ """.stripMargin
+ )
+ val destLocation = s"""$path${File.separator}destTableLocation"""
+ new File(destLocation).mkdir()
+
+ versionSpark.sql(
+ s"""
+ |CREATE TABLE $destTableName
+ |ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+ |WITH SERDEPROPERTIES ('respectSparkSchema' = 'true')
+ |STORED AS
+ | INPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+ | OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+ |LOCATION '$destLocation'
--- End diff --
@cloud-fan , This bug is for both external and managed tables.
I've added a new test case for managed table too. However, to avoid code
duplication, should I include both test inside same test method?. Please
suggest.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
