Repository: carbondata Updated Branches: refs/heads/master a9e405f47 -> 8c49e5b42
[HOTFIX] Fix SDV test case failure after PR #2645 **Changes: 1. #2645 has blocked the schema for external table. But SDV test cases were not updated. Hence updated the test cases 2. table properties was not blocked for external table, so even when a valid table property is passed, validation of table properties will fail for external table, as fields (schema) will be null. Hence blocked this with proper error message** This closes #2809 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/8c49e5b4 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/8c49e5b4 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/8c49e5b4 Branch: refs/heads/master Commit: 8c49e5b42fa786ad1df11d2b55c5a9e802bd579f Parents: a9e405f Author: ajantha-bhat <[email protected]> Authored: Wed Oct 10 16:51:12 2018 +0530 Committer: Jacky Li <[email protected]> Committed: Mon Oct 22 15:14:03 2018 +0800 ---------------------------------------------------------------------- .../sdv/generated/SDKwriterTestCase.scala | 32 ++++++++++---------- .../createTable/TestCreateExternalTable.scala | 2 +- .../sql/parser/CarbonSparkSqlParserUtil.scala | 9 +++++- 3 files changed, 25 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/8c49e5b4/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SDKwriterTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SDKwriterTestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SDKwriterTestCase.scala index 668d9d1..619bfb3 100644 --- a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SDKwriterTestCase.scala +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/SDKwriterTestCase.scala @@ -21,7 +21,7 @@ package org.apache.carbondata.cluster.sdv.generated import java.io.{ByteArrayInputStream, ByteArrayOutputStream, DataInputStream, InputStream} import java.util -import org.apache.spark.sql.Row +import org.apache.spark.sql.{AnalysisException, Row} import org.apache.spark.sql.common.util.QueryTest import org.scalatest.BeforeAndAfterEach import scala.collection.JavaConverters._ @@ -194,7 +194,7 @@ class SDKwriterTestCase extends QueryTest with BeforeAndAfterEach { sql("DROP TABLE IF EXISTS sdkTable") sql( - s"""CREATE EXTERNAL TABLE sdkTable(name string,age int) STORED BY + s"""CREATE EXTERNAL TABLE sdkTable STORED BY |'carbondata' LOCATION |'$writerPath' """.stripMargin) checkAnswer(sql("select * from sdkTable"), Seq(Row("abc0", 0, 0.0), @@ -228,7 +228,7 @@ class SDKwriterTestCase extends QueryTest with BeforeAndAfterEach { sql("DROP TABLE IF EXISTS sdkTable") sql( - s"""CREATE EXTERNAL TABLE sdkTable(name string,age int) STORED BY + s"""CREATE EXTERNAL TABLE sdkTable STORED BY |'carbondata' LOCATION |'$writerPath' """.stripMargin) @@ -266,11 +266,11 @@ class SDKwriterTestCase extends QueryTest with BeforeAndAfterEach { sql("DROP TABLE IF EXISTS sdkTable1") sql("DROP TABLE IF EXISTS sdkTable2") sql( - s"""CREATE EXTERNAL TABLE sdkTable1(name string,age int) STORED BY + s"""CREATE EXTERNAL TABLE sdkTable1 STORED BY |'carbondata' LOCATION |'$writerPath' """.stripMargin) sql( - s"""CREATE EXTERNAL TABLE sdkTable2(name string,age int) STORED BY + s"""CREATE EXTERNAL TABLE sdkTable2 STORED BY |'carbondata' LOCATION |'$writerPath' """.stripMargin) @@ -278,14 +278,13 @@ class SDKwriterTestCase extends QueryTest with BeforeAndAfterEach { checkAnswer(sql("select count(*) from sdkTable1"), Seq(Row(6))) } - test("test create External Table with Schema with partition, external table should " + - "ignore schema and partition") { + test("test create External Table without Schema") { buildTestDataSingleFile() assert(FileFactory.getCarbonFile(writerPath).exists()) sql("DROP TABLE IF EXISTS sdkTable") sql( - s"""CREATE EXTERNAL TABLE sdkTable(name string) PARTITIONED BY (age int) STORED BY + s"""CREATE EXTERNAL TABLE sdkTable STORED BY |'carbondata' LOCATION |'$writerPath' """.stripMargin) @@ -301,7 +300,7 @@ class SDKwriterTestCase extends QueryTest with BeforeAndAfterEach { sql("DROP TABLE IF EXISTS table1") sql( - s"""CREATE EXTERNAL TABLE sdkTable(name string) PARTITIONED BY (age int) STORED BY + s"""CREATE EXTERNAL TABLE sdkTable STORED BY |'carbondata' LOCATION |'$writerPath' """.stripMargin) @@ -323,17 +322,18 @@ class SDKwriterTestCase extends QueryTest with BeforeAndAfterEach { Seq(Row(0))) } - test("test create External Table with Table properties should ignore tblproperties") { + test("test create External Table with Table properties should fail") { buildTestDataSingleFile() assert(FileFactory.getCarbonFile(writerPath).exists()) sql("DROP TABLE IF EXISTS sdkTable") - sql( - s"""CREATE EXTERNAL TABLE sdkTable(name string,age int) STORED BY - |'carbondata' LOCATION - |'$writerPath' TBLPROPERTIES('sort_scope'='batch_sort') """.stripMargin) - - checkExistence(sql("Describe formatted sdkTable "), false, "batch_sort") + val ex = intercept[AnalysisException] { + sql( + s"""CREATE EXTERNAL TABLE sdkTable STORED BY + |'carbondata' LOCATION + |'$writerPath' TBLPROPERTIES('sort_scope'='batch_sort') """.stripMargin) + } + assert(ex.message.contains("table properties are not supported for external table")) } test("Read sdk writer output file and test without carbondata and carbonindex files should fail") http://git-wip-us.apache.org/repos/asf/carbondata/blob/8c49e5b4/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala index 56f038e..f0e7a1e 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/createTable/TestCreateExternalTable.scala @@ -101,7 +101,7 @@ class TestCreateExternalTable extends QueryTest with BeforeAndAfterAll { |LOCATION '$storeLocation/origin' """.stripMargin) } - assert(ex.message.contains("Schema may not be specified for external table")) + assert(ex.message.contains("Schema must not be specified for external table")) sql("DROP TABLE IF EXISTS source") http://git-wip-us.apache.org/repos/asf/carbondata/blob/8c49e5b4/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala index 1594c18..0378bf9 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParserUtil.scala @@ -123,6 +123,13 @@ object CarbonSparkSqlParserUtil { if (partitionFields.nonEmpty && options.isStreaming) { operationNotAllowed("Streaming is not allowed on partitioned table", partitionColumns) } + + if (external && fields.isEmpty && tableProperties.nonEmpty) { + // as fields are always zero for external table, cannot validate table properties. + operationNotAllowed( + "table properties are not supported for external table", tablePropertyList) + } + // validate tblProperties val bucketFields = parser.getBucketFields(tableProperties, fields, options) var isTransactionalTable: Boolean = true @@ -132,7 +139,7 @@ object CarbonSparkSqlParserUtil { // user provided schema for this external table, this is not allow currently // see CARBONDATA-2866 operationNotAllowed( - "Schema may not be specified for external table", columns) + "Schema must not be specified for external table", columns) } if (partitionByStructFields.nonEmpty) { operationNotAllowed(
