Github user vinodkc commented on a diff in the pull request:
https://github.com/apache/spark/pull/19779#discussion_r152473900
--- Diff:
sql/hive/src/test/scala/org/apache/spark/sql/hive/client/VersionsSuite.scala ---
@@ -841,6 +841,76 @@ class VersionsSuite extends SparkFunSuite with Logging
{
}
}
+ test(s"$version: SPARK-17920: Insert into/overwrite avro table") {
+ withTempDir { dir =>
+ val path = dir.getAbsolutePath
+ val schemaPath = s"""$path${File.separator}avroschemadir"""
+
+ new File(schemaPath).mkdir()
+ val avroSchema =
+ """{
+ | "name": "test_record",
+ | "type": "record",
+ | "fields": [ {
+ | "name": "f0",
+ | "type": [
+ | "null",
+ | {
+ | "precision": 38,
+ | "scale": 2,
+ | "type": "bytes",
+ | "logicalType": "decimal"
+ | }
+ | ]
+ | } ]
+ |}
+ """.stripMargin
+ val schemaUrl = s"""$schemaPath${File.separator}avroDecimal.avsc"""
+ val schemaFile = new File(schemaPath, "avroDecimal.avsc")
+ val writer = new PrintWriter(schemaFile)
+ writer.write(avroSchema)
+ writer.close()
+
+ val url =
Thread.currentThread().getContextClassLoader.getResource("avroDecimal")
+ val srcLocation = new File(url.getFile)
+ val destTableName = "tab1"
+ val srcTableName = "tab2"
+
+ withTable(srcTableName, destTableName) {
+ versionSpark.sql(
+ s"""
+ |CREATE EXTERNAL TABLE $srcTableName
+ |ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+ |WITH SERDEPROPERTIES ('respectSparkSchema' = 'true')
+ |STORED AS
+ | INPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+ | OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+ |LOCATION '$srcLocation'
+ |TBLPROPERTIES ('avro.schema.url' = '$schemaUrl')
+ """.stripMargin
+ )
+
+ versionSpark.sql(
+ s"""
+ |CREATE TABLE $destTableName
+ |ROW FORMAT SERDE
'org.apache.hadoop.hive.serde2.avro.AvroSerDe'
+ |WITH SERDEPROPERTIES ('respectSparkSchema' = 'true')
+ |STORED AS
+ | INPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerInputFormat'
+ | OUTPUTFORMAT
'org.apache.hadoop.hive.ql.io.avro.AvroContainerOutputFormat'
+ |TBLPROPERTIES ('avro.schema.url' = '$schemaUrl')
+ """.stripMargin
+ )
+ versionSpark.sql(
+ s"""INSERT OVERWRITE TABLE $destTableName SELECT * FROM
$srcTableName""".stripMargin)
--- End diff --
Sure, I'll update it
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
