http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/PlanTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/PlanTest.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/PlanTest.scala new file mode 100644 index 0000000..9883607 --- /dev/null +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/PlanTest.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.test.util + +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, OneRowRelation} +import org.apache.spark.sql.catalyst.util._ + +/** + * Provides helper methods for comparing plans. + */ +class PlanTest extends CarbonFunSuite { + + /** Fails the test if the two expressions do not match */ + protected def compareExpressions(e1: Expression, e2: Expression): Unit = { + comparePlans(Filter(e1, OneRowRelation), Filter(e2, OneRowRelation)) + } + + /** Fails the test if the two plans do not match */ + protected def comparePlans(plan1: LogicalPlan, plan2: LogicalPlan) { + val normalized1 = normalizeExprIds(plan1) + val normalized2 = normalizeExprIds(plan2) + if (normalized1 != normalized2) { + fail( + s""" + |== FAIL: Plans do not match === + |${sideBySide(normalized1.treeString, normalized2.treeString).mkString("\n")} + """.stripMargin) + } + } + + /** + * Since attribute references are given globally unique ids during analysis, + * we must normalize them to check if two different queries are identical. + */ + protected def normalizeExprIds(plan: LogicalPlan) = { + plan transformAllExpressions { + case a: AttributeReference => + AttributeReference(a.name, a.dataType, a.nullable)(exprId = ExprId(0)) + case a: Alias => + Alias(a.child, a.name)(exprId = ExprId(0)) + } + } +}
http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala new file mode 100644 index 0000000..b4b7602 --- /dev/null +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/util/QueryTest.scala @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.test.util + +import java.util.{Locale, TimeZone} + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.{DataFrame, Row, SQLContext} +import org.apache.spark.sql.catalyst.plans._ +import org.apache.spark.sql.catalyst.util._ +import org.apache.spark.sql.test.TestQueryExecutor + +import org.apache.carbondata.common.logging.LogServiceFactory + +class QueryTest extends PlanTest { + + val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) + + // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*) + TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) + // Add Locale setting + Locale.setDefault(Locale.US) + + /** + * Runs the plan and makes sure the answer contains all of the keywords, or the + * none of keywords are listed in the answer + * @param df the [[DataFrame]] to be executed + * @param exists true for make sure the keywords are listed in the output, otherwise + * to make sure none of the keyword are not listed in the output + * @param keywords keyword in string array + */ + def checkExistence(df: DataFrame, exists: Boolean, keywords: String*) { + val outputs = df.collect().map(_.mkString).mkString + for (key <- keywords) { + if (exists) { + assert(outputs.contains(key), s"Failed for $df ($key doesn't exist in result)") + } else { + assert(!outputs.contains(key), s"Failed for $df ($key existed in the result)") + } + } + } + + def sqlTest(sqlString: String, expectedAnswer: Seq[Row])(implicit sqlContext: SQLContext) { + test(sqlString) { + checkAnswer(sqlContext.sql(sqlString), expectedAnswer) + } + } + + /** + * Runs the plan and makes sure the answer matches the expected result. + * @param df the [[DataFrame]] to be executed + * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. + */ + protected def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Unit = { + QueryTest.checkAnswer(df, expectedAnswer) match { + case Some(errorMessage) => fail(errorMessage) + case None => + } + } + + protected def checkAnswer(df: DataFrame, expectedAnswer: Row): Unit = { + checkAnswer(df, Seq(expectedAnswer)) + } + + protected def checkAnswer(df: DataFrame, expectedAnswer: DataFrame): Unit = { + checkAnswer(df, expectedAnswer.collect()) + } + + def sql(sqlText: String): DataFrame = TestQueryExecutor.INSTANCE.sql(sqlText) + + val sqlContext: SQLContext = TestQueryExecutor.INSTANCE.sqlContext + + val storeLocation = TestQueryExecutor.storeLocation + val resourcesPath = TestQueryExecutor.resourcesPath + val integrationPath = TestQueryExecutor.integrationPath +} + +object QueryTest { + + def checkAnswer(df: DataFrame, expectedAnswer: java.util.List[Row]): String = { + checkAnswer(df, expectedAnswer.asScala) match { + case Some(errorMessage) => errorMessage + case None => null + } + } + + /** + * Runs the plan and makes sure the answer matches the expected result. + * If there was exception during the execution or the contents of the DataFrame does not + * match the expected result, an error message will be returned. Otherwise, a [[None]] will + * be returned. + * @param df the [[DataFrame]] to be executed + * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. + */ + def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Option[String] = { + val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty + def prepareAnswer(answer: Seq[Row]): Seq[Row] = { + // Converts data to types that we can do equality comparison using Scala collections. + // For BigDecimal type, the Scala type has a better definition of equality test (similar to + // Java's java.math.BigDecimal.compareTo). + // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for + // equality test. + val converted: Seq[Row] = answer.map { s => + Row.fromSeq(s.toSeq.map { + case d: java.math.BigDecimal => BigDecimal(d) + case b: Array[Byte] => b.toSeq + case o => o + }) + } + if (!isSorted) converted.sortBy(_.toString()) else converted + } + val sparkAnswer = try df.collect().toSeq catch { + case e: Exception => + val errorMessage = + s""" + |Exception thrown while executing query: + |${df.queryExecution} + |== Exception == + |$e + |${org.apache.spark.sql.catalyst.util.stackTraceToString(e)} + """.stripMargin + return Some(errorMessage) + } + + if (prepareAnswer(expectedAnswer) != prepareAnswer(sparkAnswer)) { + val errorMessage = + s""" + |Results do not match for query: + |${df.queryExecution} + |== Results == + |${ + sideBySide( + s"== Correct Answer - ${expectedAnswer.size} ==" +: + prepareAnswer(expectedAnswer).map(_.toString()), + s"== Spark Answer - ${sparkAnswer.size} ==" +: + prepareAnswer(sparkAnswer).map(_.toString())).mkString("\n") + } + """.stripMargin + return Some(errorMessage) + } + + return None + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala index 721c6cc..aaaf66b 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/complexType/TestComplexPrimitiveTimestampDirectDictionary.scala @@ -17,7 +17,7 @@ package org.apache.carbondata.integration.spark.testsuite.complexType -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonCommonConstants http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala index d56f38a..98e4f18 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/SparkDatasourceSuite.scala @@ -18,7 +18,7 @@ package org.apache.carbondata.integration.spark.testsuite.dataload -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.{DataFrame, Row, SQLContext, SaveMode} import org.scalatest.BeforeAndAfterAll http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala index 8aade63..b61ecce 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/integration/spark/testsuite/dataload/TestLoadDataWithSingleQuotechar.scala @@ -18,7 +18,7 @@ package org.apache.carbondata.integration.spark.testsuite.dataload import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll /** * Test Class for data loading when there is single quote in fact data http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala index 8339c06..e2e7afc 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/AllDataTypesTestCaseAggregate.scala @@ -17,7 +17,7 @@ package org.apache.carbondata.spark.testsuite.allqueries -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.{Row, SaveMode} import org.scalatest.BeforeAndAfterAll http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala index 4261d9b..d762ec6 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark1TestCase.scala @@ -16,7 +16,7 @@ */ package org.apache.carbondata.spark.testsuite.allqueries -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonCommonConstants http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala index 851f7e9..7aee00d 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/badrecordloger/BadRecordLoggerSharedDictionaryTest.scala @@ -23,7 +23,7 @@ import java.io.File import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.hive.HiveContext import org.scalatest.BeforeAndAfterAll http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala index d0e9639..3bfe8d6 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/createtable/TestCreateTableSyntax.scala @@ -18,7 +18,7 @@ package org.apache.carbondata.spark.testsuite.createtable import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.spark.exception.MalformedCarbonCommandException http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala index ba0fc0a..ab75d6e 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CompactionSystemLockFeatureTest.scala @@ -18,7 +18,7 @@ package org.apache.carbondata.spark.testsuite.datacompaction import scala.collection.JavaConverters._ -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala index b715896..827b4f6 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala @@ -18,7 +18,7 @@ package org.apache.carbondata.spark.testsuite.datacompaction import scala.collection.JavaConverters._ -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala index 5897681..053ad44 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionNoDictionaryTest.scala @@ -19,7 +19,7 @@ package org.apache.carbondata.spark.testsuite.datacompaction import scala.collection.JavaConverters._ import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala index e9118ba..8a0279f 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionTest.scala @@ -19,7 +19,7 @@ package org.apache.carbondata.spark.testsuite.datacompaction import scala.collection.JavaConverters._ import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonTableIdentifier} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala index fa5189b..250ec06 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/filterexpr/GrtLtFilterProcessorTestCase.scala @@ -18,7 +18,7 @@ package org.apache.carbondata.spark.testsuite.filterexpr import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonCommonConstants http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala index ca8103f..1649875 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/hadooprelation/HadoopFSRelationTestCase.scala @@ -18,7 +18,7 @@ package org.apache.carbondata.spark.testsuite.hadooprelation import org.apache.spark.sql.DataFrame -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.scalatest.BeforeAndAfterAll /** http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala index a2d40b5..c149152 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala @@ -18,10 +18,9 @@ package org.apache.carbondata.spark.util import java.io.File -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.{CarbonEnv, CarbonRelation} import org.scalatest.BeforeAndAfterAll - import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.processing.constants.TableOptionConstant http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala index 598a90e..7842b11 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/AutoHighCardinalityIdentifyTestCase.scala @@ -19,10 +19,9 @@ package org.apache.carbondata.spark.util import java.io.{BufferedWriter, File, FileWriter} import java.util.Random -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.{CarbonEnv, CarbonRelation} import org.scalatest.BeforeAndAfterAll - import org.apache.carbondata.core.metadata.encoder.Encoding import org.apache.carbondata.core.metadata.schema.table.CarbonTable import org.apache.carbondata.core.util.path.CarbonStorePath http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala index 4505429..706d9d6 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala @@ -18,10 +18,9 @@ */ package org.apache.carbondata.spark.util -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.{CarbonEnv, CarbonRelation} import org.scalatest.BeforeAndAfterAll - import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.processing.constants.TableOptionConstant http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala index 9e0f851..e95b9cd 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilConcurrentTestCase.scala @@ -20,10 +20,10 @@ import java.util.concurrent.{Callable, Executors} import scala.collection.mutable.ListBuffer -import org.apache.spark.sql.common.util.QueryTest +import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.{CarbonEnv, CarbonRelation} import org.scalatest.BeforeAndAfterAll - import org.apache.carbondata.core.service.impl.PathFactory import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datastore.impl.FileFactory http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala index c4b213f..405322b 100644 --- a/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/util/GlobalDictionaryUtilTestCase.scala @@ -18,7 +18,7 @@ package org.apache.carbondata.spark.util import java.io.File -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.{CarbonEnv, CarbonRelation} import org.scalatest.BeforeAndAfterAll http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala index b8f0a7c..e79aa80 100644 --- a/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala +++ b/integration/spark/src/test/scala/org/apache/spark/sql/TestCarbonSqlParser.scala @@ -18,7 +18,7 @@ package org.apache.spark.sql import scala.collection.mutable.Map -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.test.util.QueryTest import org.apache.spark.sql.execution.command.Field import org.apache.carbondata.core.constants.CarbonCommonConstants http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala b/integration/spark/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala deleted file mode 100644 index 37803a8..0000000 --- a/integration/spark/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.common.util - -import org.apache.carbondata.common.logging.LogServiceFactory -import org.scalatest.{FunSuite, Outcome} - - -private[spark] abstract class CarbonFunSuite extends FunSuite { - - private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) - - /** - * Log the suite name and the test name before and after each test. - * - * Subclasses should never override this method. If they wish to run - * custom code before and after each test, they should should mix in - * the {{org.scalatest.BeforeAndAfter}} trait instead. - */ - final protected override def withFixture(test: NoArgTest): Outcome = { - val testName = test.text - val suiteName = this.getClass.getName - val shortSuiteName = suiteName.replaceAll("org.apache.spark", "o.a.s") - try { - LOGGER.info(s"\n\n===== TEST OUTPUT FOR $shortSuiteName: '$testName' =====\n") - test() - } finally { - LOGGER.info(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n") - } - } - -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala b/integration/spark/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala deleted file mode 100644 index cdd415f..0000000 --- a/integration/spark/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.common.util - -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, OneRowRelation} -import org.apache.spark.sql.catalyst.util._ - -/** - * Provides helper methods for comparing plans. - */ -class PlanTest extends CarbonFunSuite { - - /** Fails the test if the two expressions do not match */ - protected def compareExpressions(e1: Expression, e2: Expression): Unit = { - comparePlans(Filter(e1, OneRowRelation), Filter(e2, OneRowRelation)) - } - - /** Fails the test if the two plans do not match */ - protected def comparePlans(plan1: LogicalPlan, plan2: LogicalPlan) { - val normalized1 = normalizeExprIds(plan1) - val normalized2 = normalizeExprIds(plan2) - if (normalized1 != normalized2) { - fail( - s""" - |== FAIL: Plans do not match === - |${sideBySide(normalized1.treeString, normalized2.treeString).mkString("\n")} - """.stripMargin) - } - } - - /** - * Since attribute references are given globally unique ids during analysis, - * we must normalize them to check if two different queries are identical. - */ - protected def normalizeExprIds(plan: LogicalPlan) = { - plan transformAllExpressions { - case a: AttributeReference => - AttributeReference(a.name, a.dataType, a.nullable)(exprId = ExprId(0)) - case a: Alias => - Alias(a.child, a.name)(exprId = ExprId(0)) - } - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala b/integration/spark/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala deleted file mode 100644 index bd9b743..0000000 --- a/integration/spark/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala +++ /dev/null @@ -1,158 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.common.util - -import java.util.{Locale, TimeZone} - -import org.apache.carbondata.common.logging.LogServiceFactory -import scala.collection.JavaConversions._ - -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.util._ -import org.apache.spark.sql.test.TestQueryExecutor -import org.apache.spark.sql.{DataFrame, Row, SQLContext} - -class QueryTest extends PlanTest { - - val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) - - // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*) - TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) - // Add Locale setting - Locale.setDefault(Locale.US) - - /** - * Runs the plan and makes sure the answer contains all of the keywords, or the - * none of keywords are listed in the answer - * @param df the [[DataFrame]] to be executed - * @param exists true for make sure the keywords are listed in the output, otherwise - * to make sure none of the keyword are not listed in the output - * @param keywords keyword in string array - */ - def checkExistence(df: DataFrame, exists: Boolean, keywords: String*) { - val outputs = df.collect().map(_.mkString).mkString - for (key <- keywords) { - if (exists) { - assert(outputs.contains(key), s"Failed for $df ($key doesn't exist in result)") - } else { - assert(!outputs.contains(key), s"Failed for $df ($key existed in the result)") - } - } - } - - def sqlTest(sqlString: String, expectedAnswer: Seq[Row])(implicit sqlContext: SQLContext) { - test(sqlString) { - checkAnswer(sqlContext.sql(sqlString), expectedAnswer) - } - } - - /** - * Runs the plan and makes sure the answer matches the expected result. - * @param df the [[DataFrame]] to be executed - * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. - */ - protected def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Unit = { - QueryTest.checkAnswer(df, expectedAnswer) match { - case Some(errorMessage) => fail(errorMessage) - case None => - } - } - - protected def checkAnswer(df: DataFrame, expectedAnswer: Row): Unit = { - checkAnswer(df, Seq(expectedAnswer)) - } - - protected def checkAnswer(df: DataFrame, expectedAnswer: DataFrame): Unit = { - checkAnswer(df, expectedAnswer.collect()) - } - - def sql(sqlText: String): DataFrame = TestQueryExecutor.INSTANCE.sql(sqlText) - - val sqlContext: SQLContext = TestQueryExecutor.INSTANCE.sqlContext - val storeLocation = TestQueryExecutor.storeLocation - val resourcesPath = TestQueryExecutor.resourcesPath - val integrationPath = TestQueryExecutor.integrationPath -} - -object QueryTest { - - def checkAnswer(df: DataFrame, expectedAnswer: java.util.List[Row]): String = { - checkAnswer(df, expectedAnswer.toSeq) match { - case Some(errorMessage) => errorMessage - case None => null - } - } - - /** - * Runs the plan and makes sure the answer matches the expected result. - * If there was exception during the execution or the contents of the DataFrame does not - * match the expected result, an error message will be returned. Otherwise, a [[None]] will - * be returned. - * @param df the [[DataFrame]] to be executed - * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. - */ - def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Option[String] = { - val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty - def prepareAnswer(answer: Seq[Row]): Seq[Row] = { - // Converts data to types that we can do equality comparison using Scala collections. - // For BigDecimal type, the Scala type has a better definition of equality test (similar to - // Java's java.math.BigDecimal.compareTo). - // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for - // equality test. - val converted: Seq[Row] = answer.map { s => - Row.fromSeq(s.toSeq.map { - case d: java.math.BigDecimal => BigDecimal(d) - case b: Array[Byte] => b.toSeq - case o => o - }) - } - if (!isSorted) converted.sortBy(_.toString()) else converted - } - val sparkAnswer = try df.collect().toSeq catch { - case e: Exception => - val errorMessage = - s""" - |Exception thrown while executing query: - |${df.queryExecution} - |== Exception == - |$e - |${org.apache.spark.sql.catalyst.util.stackTraceToString(e)} - """.stripMargin - return Some(errorMessage) - } - - if (prepareAnswer(expectedAnswer) != prepareAnswer(sparkAnswer)) { - val errorMessage = - s""" - |Results do not match for query: - |${df.queryExecution} - |== Results == - |${ - sideBySide( - s"== Correct Answer - ${expectedAnswer.size} ==" +: - prepareAnswer(expectedAnswer).map(_.toString()), - s"== Spark Answer - ${sparkAnswer.size} ==" +: - prepareAnswer(sparkAnswer).map(_.toString())).mkString("\n") - } - """.stripMargin - return Some(errorMessage) - } - - return None - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark2TestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark2TestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark2TestCase.scala index 0dae268..d332261 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark2TestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/testsuite/allqueries/InsertIntoCarbonTableSpark2TestCase.scala @@ -17,13 +17,13 @@ package org.apache.carbondata.spark.testsuite.allqueries import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties -class InsertIntoCarbonTableSpark2TestCase extends QueryTest with BeforeAndAfterAll { +class InsertIntoCarbonTableSpark2TestCase extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll: Unit = { sql("drop table if exists OneRowTable") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala index 23800ee..89bf839 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/AllDictionaryTestCase.scala @@ -16,12 +16,11 @@ */ package org.apache.carbondata.spark.util -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.hive.CarbonRelation import org.apache.spark.sql.{CarbonEnv, SparkSession} import org.scalatest.BeforeAndAfterAll -import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.processing.constants.TableOptionConstant @@ -30,8 +29,7 @@ import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadM /** * Test Case for org.apache.carbondata.integration.spark.util.GlobalDictionaryUtil */ -class AllDictionaryTestCase extends QueryTest with BeforeAndAfterAll { - private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) +class AllDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll { var pwd: String = _ var sampleRelation: CarbonRelation = _ var complexRelation: CarbonRelation = _ http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala index 5f7ff2e..7350e2d 100644 --- a/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/carbondata/spark/util/ExternalColumnDictionaryTestCase.scala @@ -18,12 +18,11 @@ */ package org.apache.carbondata.spark.util -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.hive.CarbonRelation import org.apache.spark.sql.{CarbonEnv, SparkSession} import org.scalatest.BeforeAndAfterAll -import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.processing.constants.TableOptionConstant @@ -35,8 +34,7 @@ import org.apache.carbondata.spark.exception.MalformedCarbonCommandException * test case for external column dictionary generation * also support complicated type */ -class ExternalColumnDictionaryTestCase extends QueryTest with BeforeAndAfterAll { - private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) +class ExternalColumnDictionaryTestCase extends Spark2QueryTest with BeforeAndAfterAll { var extComplexRelation: CarbonRelation = _ var verticalDelimiteRelation: CarbonRelation = _ var loadSqlRelation: CarbonRelation = _ http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala index 13a04c5..b7cb63d 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/SparkCommandSuite.scala @@ -17,10 +17,10 @@ package org.apache.spark -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll -class SparkCommandSuite extends QueryTest with BeforeAndAfterAll { +class SparkCommandSuite extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll(): Unit = { sql("DROP TABLE IF EXISTS src_pqt") sql("DROP TABLE IF EXISTS src_orc") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/BadRecordPathLoadOptionTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/BadRecordPathLoadOptionTest.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/BadRecordPathLoadOptionTest.scala index 846c4b6..8bec6f6 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/BadRecordPathLoadOptionTest.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/BadRecordPathLoadOptionTest.scala @@ -19,7 +19,7 @@ package org.apache.spark.carbondata import java.io.File -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.hive.HiveContext import org.scalatest.BeforeAndAfterAll @@ -33,7 +33,7 @@ import org.apache.carbondata.core.util.CarbonProperties * * */ -class BadRecordPathLoadOptionTest extends QueryTest with BeforeAndAfterAll { +class BadRecordPathLoadOptionTest extends Spark2QueryTest with BeforeAndAfterAll { var hiveContext: HiveContext = _ var badRecordPath: String = null override def beforeAll { http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala index 2995e60..316881d 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/CarbonDataSourceSuite.scala @@ -19,14 +19,14 @@ package org.apache.spark.carbondata import scala.collection.mutable -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.types._ import org.apache.spark.sql.{Row, SaveMode} import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.util.CarbonProperties -class CarbonDataSourceSuite extends QueryTest with BeforeAndAfterAll { +class CarbonDataSourceSuite extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll(): Unit = { // Drop table sql("DROP TABLE IF EXISTS carbon_testtable") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala index 6f57cd6..cec7bbc 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/DataLoadFailAllTypeSortTest.scala @@ -19,7 +19,7 @@ package org.apache.spark.carbondata import java.io.File -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.hive.HiveContext import org.scalatest.BeforeAndAfterAll @@ -31,7 +31,7 @@ import org.apache.carbondata.core.util.CarbonProperties * * */ -class DataLoadFailAllTypeSortTest extends QueryTest with BeforeAndAfterAll { +class DataLoadFailAllTypeSortTest extends Spark2QueryTest with BeforeAndAfterAll { var hiveContext: HiveContext = _ override def beforeAll: Unit = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala index 2731812..cb926e1 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/bucketing/TableBucketingTestCase.scala @@ -17,7 +17,7 @@ package org.apache.spark.carbondata.bucketing -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.execution.command.LoadTable import org.apache.spark.sql.execution.exchange.ShuffleExchange import org.scalatest.BeforeAndAfterAll @@ -28,7 +28,7 @@ import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties import org.apache.carbondata.spark.exception.MalformedCarbonCommandException -class TableBucketingTestCase extends QueryTest with BeforeAndAfterAll { +class TableBucketingTestCase extends Spark2QueryTest with BeforeAndAfterAll { var threshold: Int = _ http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala index 18b4039..60cf121 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/commands/SetCommandTestCase.scala @@ -16,13 +16,13 @@ */ package org.apache.spark.carbondata.commands -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonLoadOptionConstants import org.apache.carbondata.core.exception.InvalidConfigurationException -class SetCommandTestCase extends QueryTest with BeforeAndAfterAll{ +class SetCommandTestCase extends Spark2QueryTest with BeforeAndAfterAll{ override def beforeAll: Unit = { sql("set carbon=true") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala index e8590dd..b1e0bde 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/datatype/NumericDimensionBadRecordTest.scala @@ -20,7 +20,7 @@ package org.apache.carbondata.spark.testsuite.badrecordloger import java.io.File import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.hive.HiveContext import org.scalatest.BeforeAndAfterAll @@ -32,7 +32,7 @@ import org.apache.carbondata.core.util.CarbonProperties * * */ -class NumericDimensionBadRecordTest extends QueryTest with BeforeAndAfterAll { +class NumericDimensionBadRecordTest extends Spark2QueryTest with BeforeAndAfterAll { var hiveContext: HiveContext = _ override def beforeAll { http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala index 1ab9409..31169fe 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/deletetable/DeleteTableTestCase.scala @@ -17,13 +17,13 @@ package org.apache.spark.carbondata.deletetable -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll /** * Test cases for drop table */ -class DeleteTableTestCase extends QueryTest with BeforeAndAfterAll { +class DeleteTableTestCase extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sql("drop table if exists IS") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/iud/DeleteCarbonTableSubqueryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/iud/DeleteCarbonTableSubqueryTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/iud/DeleteCarbonTableSubqueryTestCase.scala index ff6196c..e92d06d 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/iud/DeleteCarbonTableSubqueryTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/iud/DeleteCarbonTableSubqueryTestCase.scala @@ -17,13 +17,13 @@ package org.apache.spark.carbondata.iud import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties -class DeleteCarbonTableSubqueryTestCase extends QueryTest with BeforeAndAfterAll { +class DeleteCarbonTableSubqueryTestCase extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sql("use default") sql("drop database if exists iud_db_sub cascade") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala index ad56173..66cf675 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/SubQueryTestSuite.scala @@ -18,12 +18,12 @@ package org.apache.spark.carbondata.query import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.execution.exchange.ShuffleExchange import org.apache.spark.sql.execution.joins.BroadcastHashJoinExec import org.scalatest.BeforeAndAfterAll -class SubQueryTestSuite extends QueryTest with BeforeAndAfterAll { +class SubQueryTestSuite extends Spark2QueryTest with BeforeAndAfterAll { val tempDirPath = s"$resourcesPath/temp" http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala index fb0188c..d6c1d20 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/query/TestNotEqualToFilter.scala @@ -17,7 +17,7 @@ package org.apache.spark.carbondata.query -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonCommonConstants @@ -26,7 +26,7 @@ import org.apache.carbondata.core.util.CarbonProperties /** * Test cases for testing columns having \N or \null values for non numeric columns */ -class TestNotEqualToFilter extends QueryTest with BeforeAndAfterAll { +class TestNotEqualToFilter extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sql("drop table if exists test_not_equal_to_carbon") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala index b5a8071..c248e31 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableRevertTestCase.scala @@ -20,14 +20,13 @@ package org.apache.spark.carbondata.restructure import java.io.File import org.apache.spark.sql.AnalysisException -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.test.TestQueryExecutor import org.apache.spark.util.AlterTableUtil import org.scalatest.BeforeAndAfterAll - import org.apache.carbondata.core.metadata.CarbonMetadata -class AlterTableRevertTestCase extends QueryTest with BeforeAndAfterAll { +class AlterTableRevertTestCase extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll() { sql("drop table if exists reverttest") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala index ae64498..23270e3 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala @@ -21,13 +21,13 @@ import java.io.File import java.math.{BigDecimal, RoundingMode} import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties -class AlterTableValidationTestCase extends QueryTest with BeforeAndAfterAll { +class AlterTableValidationTestCase extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { CarbonProperties.getInstance() http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala index 7c94ebf..2a199ef 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/AddColumnTestCases.scala @@ -21,11 +21,11 @@ import java.io.{File, FileOutputStream, FileWriter} import java.math.{BigDecimal, RoundingMode} import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.test.TestQueryExecutor import org.scalatest.BeforeAndAfterAll -class AddColumnTestCases extends QueryTest with BeforeAndAfterAll { +class AddColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sqlContext.setConf("carbon.enable.vector.reader", "false") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/ChangeDataTypeTestCases.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/ChangeDataTypeTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/ChangeDataTypeTestCases.scala index e3fec55..fc2da21 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/ChangeDataTypeTestCases.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/ChangeDataTypeTestCases.scala @@ -20,10 +20,10 @@ package org.apache.spark.carbondata.restructure.rowreader import java.math.BigDecimal import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll -class ChangeDataTypeTestCases extends QueryTest with BeforeAndAfterAll { +class ChangeDataTypeTestCases extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sqlContext.setConf("carbon.enable.vector.reader", "false") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/DropColumnTestCases.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/DropColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/DropColumnTestCases.scala index cdb9a34..a812548 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/DropColumnTestCases.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/rowreader/DropColumnTestCases.scala @@ -20,12 +20,12 @@ package org.apache.spark.carbondata.restructure.rowreader import java.math.{BigDecimal, RoundingMode} import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.util.CarbonProperties -class DropColumnTestCases extends QueryTest with BeforeAndAfterAll { +class DropColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sqlContext.setConf("carbon.enable.vector.reader", "false") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala index 7a28365..5e2f729 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/AddColumnTestCases.scala @@ -21,12 +21,12 @@ import java.math.{BigDecimal, RoundingMode} import java.sql.{Date, Timestamp} import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.spark.exception.MalformedCarbonCommandException -class AddColumnTestCases extends QueryTest with BeforeAndAfterAll { +class AddColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sqlContext.setConf("carbon.enable.vector.reader", "true") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/ChangeDataTypeTestCases.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/ChangeDataTypeTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/ChangeDataTypeTestCases.scala index 56e4d2e..cbda10b 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/ChangeDataTypeTestCases.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/ChangeDataTypeTestCases.scala @@ -20,10 +20,10 @@ package org.apache.spark.carbondata.restructure.vectorreader import java.math.BigDecimal import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll -class ChangeDataTypeTestCases extends QueryTest with BeforeAndAfterAll { +class ChangeDataTypeTestCases extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sqlContext.setConf("carbon.enable.vector.reader", "true") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/DropColumnTestCases.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/DropColumnTestCases.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/DropColumnTestCases.scala index 7945c22..b757ad0 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/DropColumnTestCases.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/vectorreader/DropColumnTestCases.scala @@ -20,12 +20,12 @@ package org.apache.spark.carbondata.restructure.vectorreader import java.math.{BigDecimal, RoundingMode} import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.util.CarbonProperties -class DropColumnTestCases extends QueryTest with BeforeAndAfterAll { +class DropColumnTestCases extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { sqlContext.setConf("carbon.enable.vector.reader", "true") http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala index 55eaa20..dcc21f0 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/vectorreader/VectorReaderTestCase.scala @@ -18,7 +18,7 @@ package org.apache.spark.carbondata.vectorreader import org.apache.spark.sql.Row -import org.apache.spark.sql.common.util.QueryTest +import org.apache.spark.sql.common.util.Spark2QueryTest import org.apache.spark.sql.execution.command.LoadTable import org.apache.spark.sql.execution.{BatchedDataSourceScanExec, RowDataSourceScanExec} import org.scalatest.BeforeAndAfterAll @@ -26,7 +26,7 @@ import org.scalatest.BeforeAndAfterAll import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.util.CarbonProperties -class VectorReaderTestCase extends QueryTest with BeforeAndAfterAll { +class VectorReaderTestCase extends Spark2QueryTest with BeforeAndAfterAll { override def beforeAll { http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala deleted file mode 100644 index 37803a8..0000000 --- a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala +++ /dev/null @@ -1,47 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.common.util - -import org.apache.carbondata.common.logging.LogServiceFactory -import org.scalatest.{FunSuite, Outcome} - - -private[spark] abstract class CarbonFunSuite extends FunSuite { - - private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) - - /** - * Log the suite name and the test name before and after each test. - * - * Subclasses should never override this method. If they wish to run - * custom code before and after each test, they should should mix in - * the {{org.scalatest.BeforeAndAfter}} trait instead. - */ - final protected override def withFixture(test: NoArgTest): Outcome = { - val testName = test.text - val suiteName = this.getClass.getName - val shortSuiteName = suiteName.replaceAll("org.apache.spark", "o.a.s") - try { - LOGGER.info(s"\n\n===== TEST OUTPUT FOR $shortSuiteName: '$testName' =====\n") - test() - } finally { - LOGGER.info(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n") - } - } - -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala deleted file mode 100644 index cdd415f..0000000 --- a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.common.util - -import org.apache.spark.sql.catalyst.expressions._ -import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, OneRowRelation} -import org.apache.spark.sql.catalyst.util._ - -/** - * Provides helper methods for comparing plans. - */ -class PlanTest extends CarbonFunSuite { - - /** Fails the test if the two expressions do not match */ - protected def compareExpressions(e1: Expression, e2: Expression): Unit = { - comparePlans(Filter(e1, OneRowRelation), Filter(e2, OneRowRelation)) - } - - /** Fails the test if the two plans do not match */ - protected def comparePlans(plan1: LogicalPlan, plan2: LogicalPlan) { - val normalized1 = normalizeExprIds(plan1) - val normalized2 = normalizeExprIds(plan2) - if (normalized1 != normalized2) { - fail( - s""" - |== FAIL: Plans do not match === - |${sideBySide(normalized1.treeString, normalized2.treeString).mkString("\n")} - """.stripMargin) - } - } - - /** - * Since attribute references are given globally unique ids during analysis, - * we must normalize them to check if two different queries are identical. - */ - protected def normalizeExprIds(plan: LogicalPlan) = { - plan transformAllExpressions { - case a: AttributeReference => - AttributeReference(a.name, a.dataType, a.nullable)(exprId = ExprId(0)) - case a: Alias => - Alias(a.child, a.name)(exprId = ExprId(0)) - } - } -} http://git-wip-us.apache.org/repos/asf/carbondata/blob/770e42df/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala b/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala deleted file mode 100644 index c56c0c3..0000000 --- a/integration/spark2/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala +++ /dev/null @@ -1,218 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one or more - * contributor license agreements. See the NOTICE file distributed with - * this work for additional information regarding copyright ownership. - * The ASF licenses this file to You under the Apache License, Version 2.0 - * (the "License"); you may not use this file except in compliance with - * the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.spark.sql.common.util - -import java.util.{Locale, TimeZone} - -import scala.collection.JavaConversions._ - -import org.apache.spark.sql.catalyst.plans._ -import org.apache.spark.sql.catalyst.util._ -import org.apache.spark.sql.hive.CarbonSessionState -import org.apache.spark.sql.test.TestQueryExecutor -import org.apache.spark.sql.{DataFrame, Row} - -import org.apache.carbondata.common.logging.LogServiceFactory - -class QueryTest extends PlanTest { - - private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) - - // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*) - TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) - // Add Locale setting - Locale.setDefault(Locale.US) - - val sqlContext = TestQueryExecutor.INSTANCE.sqlContext - - val hiveClient = sqlContext.sparkSession.sessionState.asInstanceOf[CarbonSessionState] - .metadataHive - - val resourcesPath = TestQueryExecutor.resourcesPath - - def sql(sqlText: String): DataFrame = TestQueryExecutor.INSTANCE.sql(sqlText) - - /** - * Runs the plan and makes sure the answer contains all of the keywords, or the - * none of keywords are listed in the answer - * @param df the [[DataFrame]] to be executed - * @param exists true for make sure the keywords are listed in the output, otherwise - * to make sure none of the keyword are not listed in the output - * @param keywords keyword in string array - */ - def checkExistence(df: DataFrame, exists: Boolean, keywords: String*) { - val outputs = df.collect().map(_.mkString).mkString - for (key <- keywords) { - if (exists) { - assert(outputs.contains(key), s"Failed for $df ($key doesn't exist in result)") - } else { - assert(!outputs.contains(key), s"Failed for $df ($key existed in the result)") - } - } - } - - def sqlTest(sqlString: String, expectedAnswer: Seq[Row]) { - test(sqlString) { - checkAnswer(sql(sqlString), expectedAnswer) - } - } - - /** - * Runs the plan and makes sure the answer matches the expected result. - * @param df the [[DataFrame]] to be executed - * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. - */ - protected def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Unit = { - QueryTest.checkAnswer(df, expectedAnswer) match { - case Some(errorMessage) => fail(errorMessage) - case None => - } - } - - protected def checkAnswer(df: DataFrame, expectedAnswer: Row): Unit = { - checkAnswer(df, Seq(expectedAnswer)) - } - - protected def checkAnswer(df: DataFrame, expectedAnswer: DataFrame): Unit = { - checkAnswer(df, expectedAnswer.collect()) - } - - - protected def createAndLoadInputTable(inputTableName: String, inputPath: String): Unit = { - sql( - s""" - | CREATE TABLE $inputTableName - | ( shortField short, - | intField int, - | bigintField long, - | doubleField double, - | stringField string, - | timestampField string, - | decimalField decimal(18,2), - | dateField string, - | charField char(5) - | ) - | ROW FORMAT DELIMITED FIELDS TERMINATED BY ',' - """.stripMargin) - - sql( - s""" - | LOAD DATA LOCAL INPATH '$inputPath' - | INTO TABLE $inputTableName - """.stripMargin) - } - - protected def createAndLoadTestTable(tableName: String, inputTableName: String): Unit = { - sql( - s""" - | CREATE TABLE $tableName( - | shortField short, - | intField int, - | bigintField long, - | doubleField double, - | stringField string, - | timestampField timestamp, - | decimalField decimal(18,2), - | dateField date, - | charField char(5) - | ) - | USING org.apache.spark.sql.CarbonSource - | OPTIONS ('tableName' '$tableName') - """.stripMargin) - sql( - s""" - | INSERT INTO TABLE $tableName - | SELECT shortField, intField, bigintField, doubleField, stringField, - | from_unixtime(unix_timestamp(timestampField,'yyyy/M/dd')) timestampField, decimalField, - | cast(to_date(from_unixtime(unix_timestamp(dateField,'yyyy/M/dd'))) as date), charField - | FROM $inputTableName - """.stripMargin) - } - - protected def dropTable(tableName: String): Unit ={ - sql(s"DROP TABLE IF EXISTS $tableName") - } -} - -object QueryTest { - def checkAnswer(df: DataFrame, expectedAnswer: java.util.List[Row]): String = { - checkAnswer(df, expectedAnswer.toSeq) match { - case Some(errorMessage) => errorMessage - case None => null - } - } - - /** - * Runs the plan and makes sure the answer matches the expected result. - * If there was exception during the execution or the contents of the DataFrame does not - * match the expected result, an error message will be returned. Otherwise, a [[None]] will - * be returned. - * @param df the [[DataFrame]] to be executed - * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. - */ - def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Option[String] = { - val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty - def prepareAnswer(answer: Seq[Row]): Seq[Row] = { - // Converts data to types that we can do equality comparison using Scala collections. - // For BigDecimal type, the Scala type has a better definition of equality test (similar to - // Java's java.math.BigDecimal.compareTo). - // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for - // equality test. - val converted: Seq[Row] = answer.map { s => - Row.fromSeq(s.toSeq.map { - case d: java.math.BigDecimal => BigDecimal(d) - case b: Array[Byte] => b.toSeq - case o => o - }) - } - if (!isSorted) converted.sortBy(_.toString()) else converted - } - val sparkAnswer = try df.collect().toSeq catch { - case e: Exception => - val errorMessage = - s""" - |Exception thrown while executing query: - |${df.queryExecution} - |== Exception == - |$e - |${org.apache.spark.sql.catalyst.util.stackTraceToString(e)} - """.stripMargin - return Some(errorMessage) - } - - if (prepareAnswer(expectedAnswer) != prepareAnswer(sparkAnswer)) { - val errorMessage = - s""" - |Results do not match for query: - |${df.queryExecution} - |== Results == - |${ - sideBySide( - s"== Correct Answer - ${expectedAnswer.size} ==" +: - prepareAnswer(expectedAnswer).map(_.toString()), - s"== Spark Answer - ${sparkAnswer.size} ==" +: - prepareAnswer(sparkAnswer).map(_.toString())).mkString("\n") - } - """.stripMargin - return Some(errorMessage) - } - - None - } - -}
