Repository: carbondata Updated Branches: refs/heads/master 2eb8f047c -> ee9df2e2c
[CARBONDATA-2262] Support the syntax of 'STORED AS CARBONDATA' to create table This closes #2078 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ee9df2e2 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ee9df2e2 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ee9df2e2 Branch: refs/heads/master Commit: ee9df2e2cab3d78e791cbcc8980211418ff00161 Parents: 2eb8f04 Author: root <[email protected]> Authored: Mon Mar 19 19:59:26 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Mon Mar 26 23:28:26 2018 +0800 ---------------------------------------------------------------------- .../sql/commands/StoredAsCarbondataSuite.scala | 102 +++++++++++++++++++ .../spark/sql/parser/CarbonSparkSqlParser.scala | 5 +- .../spark/sql/hive/CarbonSessionState.scala | 1 + .../spark/sql/hive/CarbonSessionState.scala | 1 + .../org/apache/spark/SparkCommandSuite.scala | 2 +- 5 files changed, 109 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/StoredAsCarbondataSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/StoredAsCarbondataSuite.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/StoredAsCarbondataSuite.scala new file mode 100644 index 0000000..2029e93 --- /dev/null +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/sql/commands/StoredAsCarbondataSuite.scala @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.sql.commands + +import org.apache.spark.sql.Row +import org.apache.spark.sql.test.util.QueryTest +import org.scalatest.BeforeAndAfterEach + +import org.apache.carbondata.core.constants.CarbonCommonConstants + +class StoredAsCarbondataSuite extends QueryTest with BeforeAndAfterEach { + override def beforeEach(): Unit = { + sql("DROP TABLE IF EXISTS carbon_table") + sql("DROP TABLE IF EXISTS tableSize3") + } + + override def afterEach(): Unit = { + sql("DROP TABLE IF EXISTS carbon_table") + sql("DROP TABLE IF EXISTS tableSize3") + } + + test("CARBONDATA-2262: Support the syntax of 'STORED AS CARBONDATA', upper case") { + sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS CARBONDATA") + sql("INSERT INTO carbon_table VALUES (28,'Bob')") + checkAnswer(sql("SELECT * FROM carbon_table"), Seq(Row(28, "Bob"))) + } + + test("CARBONDATA-2262: Support the syntax of 'STORED AS carbondata', low case") { + sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS carbondata") + sql("INSERT INTO carbon_table VALUES (28,'Bob')") + checkAnswer(sql("SELECT * FROM carbon_table"), Seq(Row(28, "Bob"))) + } + + test("CARBONDATA-2262: Support the syntax of 'STORED AS carbondata, get data size and index size after minor compaction") { + sql("CREATE TABLE tableSize3 (empno INT, workgroupcategory STRING, deptno INT, projectcode INT, attendance INT) STORED AS carbondata") + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""") + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""") + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""") + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""") + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""") + sql(s"""LOAD DATA LOCAL INPATH '$resourcesPath/data.csv' INTO TABLE tableSize3 OPTIONS ('DELIMITER'= ',', 'QUOTECHAR'= '\"', 'FILEHEADER'='')""") + sql("ALTER TABLE tableSize3 COMPACT 'minor'") + checkExistence(sql("DESCRIBE FORMATTED tableSize3"), true, CarbonCommonConstants.TABLE_DATA_SIZE) + checkExistence(sql("DESCRIBE FORMATTED tableSize3"), true, CarbonCommonConstants.TABLE_INDEX_SIZE) + val res3 = sql("DESCRIBE FORMATTED tableSize3").collect() + .filter(row => row.getString(0).contains(CarbonCommonConstants.TABLE_DATA_SIZE) || + row.getString(0).contains(CarbonCommonConstants.TABLE_INDEX_SIZE)) + assert(res3.length == 2) + res3.foreach(row => assert(row.getString(1).trim.toLong > 0)) + } + + test("CARBONDATA-2262: Don't Support the syntax of 'STORED AS 'carbondata''") { + try { + sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS 'carbondata'") + } catch { + case e: Exception => + assert(e.getMessage.contains("mismatched input")) + } + } + + test("CARBONDATA-2262: Don't Support the syntax of 'stored by carbondata'") { + try { + sql("CREATE TABLE carbon_table(key INT, value STRING) STORED BY carbondata") + } catch { + case e: Exception => + assert(e.getMessage.contains("mismatched input")) + } + } + + test("CARBONDATA-2262: Don't Support the syntax of 'STORED AS ', null format") { + try { + sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS ") + } catch { + case e: Exception => + assert(e.getMessage.contains("no viable alternative at input")) + } + } + + test("CARBONDATA-2262: Don't Support the syntax of 'STORED AS carbon'") { + try { + sql("CREATE TABLE carbon_table(key INT, value STRING) STORED AS carbon") + } catch { + case e: Exception => + assert(e.getMessage.contains("Operation not allowed: STORED AS with file format 'carbon'")) + } + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala index 610f9bf..659f069 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/parser/CarbonSparkSqlParser.scala @@ -91,8 +91,11 @@ class CarbonHelperSqlAstBuilder(conf: SQLConf, def getFileStorage(createFileFormat: CreateFileFormatContext): String = { Option(createFileFormat) match { case Some(value) => - if (value.children.get(1).getText.equalsIgnoreCase("by")) { + val result = value.children.get(1).getText + if (result.equalsIgnoreCase("by")) { value.storageHandler().STRING().getSymbol.getText + } else if (result.equalsIgnoreCase("as") && value.children.size() > 1) { + value.children.get(2).getText } else { // The case of "STORED AS PARQUET/ORC" "" http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala index e8ae67b..4c2304d 100644 --- a/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala +++ b/integration/spark2/src/main/spark2.1/org/apache/spark/sql/hive/CarbonSessionState.scala @@ -326,6 +326,7 @@ class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSes val fileStorage = helper.getFileStorage(ctx.createFileFormat) if (fileStorage.equalsIgnoreCase("'carbondata'") || + fileStorage.equalsIgnoreCase("carbondata") || fileStorage.equalsIgnoreCase("'carbonfile'") || fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) { val createTableTuple = (ctx.createTableHeader, ctx.skewSpec, ctx.bucketSpec, http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala index 3cb9bd6..2ffc7db 100644 --- a/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala +++ b/integration/spark2/src/main/spark2.2/org/apache/spark/sql/hive/CarbonSessionState.scala @@ -325,6 +325,7 @@ class CarbonSqlAstBuilder(conf: SQLConf, parser: CarbonSpark2SqlParser, sparkSes val fileStorage = helper.getFileStorage(ctx.createFileFormat) if (fileStorage.equalsIgnoreCase("'carbondata'") || + fileStorage.equalsIgnoreCase("carbondata") || fileStorage.equalsIgnoreCase("'carbonfile'") || fileStorage.equalsIgnoreCase("'org.apache.carbondata.format'")) { val createTableTuple = (ctx.createTableHeader, ctx.skewSpec, http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee9df2e2/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala index b7cb63d..a9db10a 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala @@ -31,7 +31,7 @@ class SparkCommandSuite extends Spark2QueryTest with BeforeAndAfterAll { sql("DROP TABLE IF EXISTS src_orc") } - test("CARBONDATA-734: Support the syntax of 'STORED BY PARQUET/ORC'") { + test("CARBONDATA-734: Support the syntax of 'STORED AS PARQUET/ORC'") { sql("CREATE TABLE src_pqt(key INT, value STRING) STORED AS PARQUET") sql("CREATE TABLE src_orc(key INT, value STRING) STORED AS ORC") }
