[CARBONDATA-1482] fixed the failing test cases of presto integration 1.builds are passing 2.added dependency for spark guava 3.throw exception in case carbondatastore creator is unable to load the data 4.fixed the timestamp issue due to which test cases are failing
This closes #1360 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/d60d973d Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/d60d973d Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/d60d973d Branch: refs/heads/streaming_ingest Commit: d60d973dfbb69c1b740a6a00cab878bc168c14ae Parents: 6f20437 Author: anubhav100 <[email protected]> Authored: Fri Sep 15 15:12:10 2017 +0530 Committer: Ravindra Pesala <[email protected]> Committed: Fri Sep 15 19:25:10 2017 +0530 ---------------------------------------------------------------------- integration/presto/pom.xml | 7 ++++ .../integrationtest/PrestoAllDataTypeTest.scala | 26 +++++++------- .../presto/util/CarbonDataStoreCreator.scala | 36 ++++++++++++-------- 3 files changed, 42 insertions(+), 27 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/d60d973d/integration/presto/pom.xml ---------------------------------------------------------------------- diff --git a/integration/presto/pom.xml b/integration/presto/pom.xml index b23b1be..50e6349 100644 --- a/integration/presto/pom.xml +++ b/integration/presto/pom.xml @@ -438,6 +438,13 @@ </exclusion> </exclusions> </dependency> + + <dependency> + <groupId>org.apache.spark</groupId> + <artifactId>spark-network-common_2.11</artifactId> + <scope>test</scope> + <version>2.1.0</version> + </dependency> </dependencies> <build> http://git-wip-us.apache.org/repos/asf/carbondata/blob/d60d973d/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala index 1743be6..433ddd4 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/integrationtest/PrestoAllDataTypeTest.scala @@ -14,12 +14,12 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.carbondata.presto.integrationtest import java.io.File import org.scalatest.{BeforeAndAfterAll, FunSuiteLike} -import util.CarbonDataStoreCreator import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.presto.server.PrestoServer @@ -35,8 +35,10 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { private val storePath = s"$rootPath/integration/presto/target/store" override def beforeAll: Unit = { + import org.apache.carbondata.presto.util.CarbonDataStoreCreator CarbonDataStoreCreator - .createCarbonStore(storePath, s"$rootPath/integration/presto/src/test/resources/alldatatype.csv") + .createCarbonStore(storePath, + s"$rootPath/integration/presto/src/test/resources/alldatatype.csv") logger.info(s"\nCarbon store is created at location: $storePath") PrestoServer.startServer(storePath) } @@ -91,21 +93,21 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { test("test the result for count()clause with distinct operator on decimal column in presto") { val actualResult: List[Map[String, Any]] = PrestoServer .executeQuery("SELECT COUNT(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ") - val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 7)) - actualResult.equals(expectedResult) + val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10)) + assert(actualResult.equals(expectedResult)) } test("test the result for count()clause with out distinct operator on decimal column in presto") { val actualResult: List[Map[String, Any]] = PrestoServer .executeQuery("SELECT COUNT(BONUS) AS RESULT FROM TESTDB.TESTTABLE ") val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 10)) - actualResult.equals(expectedResult) + assert(actualResult.equals(expectedResult)) } test("test the result for sum()with out distinct operator for decimal column in presto") { val actualResult: List[Map[String, Any]] = PrestoServer .executeQuery("SELECT SUM(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ") - val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 54)) - actualResult.equals(expectedResult) + val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 20774.6475)) + assert(actualResult.toString().equals(expectedResult.toString())) } test("test the result for sum() with distinct operator for decimal column in presto") { val actualResult: List[Map[String, Any]] = PrestoServer @@ -115,11 +117,11 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { actualResult.head("RESULT").toString.toDouble == expectedResult.head("RESULT").toString.toDouble) } - test("test the result for avg() with distinct operator on decimal coin presto") { + test("test the result for avg() with distinct operator on decimal on presto") { val actualResult: List[Map[String, Any]] = PrestoServer .executeQuery("SELECT AVG(DISTINCT BONUS) AS RESULT FROM TESTDB.TESTTABLE ") - val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 8900)) - actualResult.equals(expectedResult) + val expectedResult: List[Map[String, Any]] = List(Map("RESULT" -> 2077.4648)) + assert(actualResult.toString.equals(expectedResult.toString)) } test("test the result for min() with distinct operator in decimalType of presto") { @@ -127,7 +129,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { .executeQuery("SELECT MIN(BONUS) AS RESULT FROM TESTDB.TESTTABLE ") val expectedResult: List[Map[String, Any]] = List(Map( "RESULT" -> java.math.BigDecimal.valueOf(500.414).setScale(4))) - actualResult.equals(expectedResult) + assert(actualResult.equals(expectedResult)) } test("test the result for max() with distinct operator in decimalType of presto") { @@ -135,7 +137,7 @@ class PrestoAllDataTypeTest extends FunSuiteLike with BeforeAndAfterAll { .executeQuery("SELECT MAX(BONUS) AS RESULT FROM TESTDB.TESTTABLE ") val expectedResult: List[Map[String, Any]] = List(Map( "RESULT" -> java.math.BigDecimal.valueOf(9999.999).setScale(4))) - actualResult.equals(expectedResult) + assert(actualResult.equals(expectedResult)) } test("select decimal data type with ORDER BY clause") { val actualResult: List[Map[String, Any]] = PrestoServer http://git-wip-us.apache.org/repos/asf/carbondata/blob/d60d973d/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala ---------------------------------------------------------------------- diff --git a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala index 6cb97f1..64f892d 100644 --- a/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala +++ b/integration/presto/src/test/scala/org/apache/carbondata/presto/util/CarbonDataStoreCreator.scala @@ -15,12 +15,12 @@ * limitations under the License. */ -package util +package org.apache.carbondata.presto.util +import java.util import java.io._ import java.nio.charset.Charset import java.text.SimpleDateFormat -import java.util import java.util.{ArrayList, Date, List, UUID} import scala.collection.JavaConversions._ @@ -30,34 +30,34 @@ import org.apache.hadoop.conf.Configuration import org.apache.hadoop.fs.Path import org.apache.hadoop.io.NullWritable import org.apache.hadoop.mapred.TaskAttemptID -import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl import org.apache.hadoop.mapreduce.{RecordReader, TaskType} +import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType} import org.apache.carbondata.core.cache.dictionary.{Dictionary, DictionaryColumnUniqueIdentifier, ReverseDictionary} -import org.apache.carbondata.core.cache.{Cache, CacheProvider, CacheType} import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.fileoperations.{AtomicFileOperations, AtomicFileOperationsImpl, FileWriteOperation} +import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, +CarbonTableIdentifier, ColumnIdentifier} import org.apache.carbondata.core.metadata.converter.{SchemaConverter, ThriftWrapperSchemaConverterImpl} import org.apache.carbondata.core.metadata.datatype.DataType import org.apache.carbondata.core.metadata.encoder.Encoding +import org.apache.carbondata.core.metadata.schema.{SchemaEvolution, SchemaEvolutionEntry} +import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema} import org.apache.carbondata.core.metadata.schema.table.column.{CarbonColumn, CarbonDimension, CarbonMeasure, ColumnSchema} -import org.apache.carbondata.core.metadata.schema.table.{CarbonTable, TableInfo, TableSchema} -import org.apache.carbondata.core.metadata.schema.{SchemaEvolution, SchemaEvolutionEntry} -import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, -CarbonTableIdentifier, ColumnIdentifier} import org.apache.carbondata.core.statusmanager.LoadMetadataDetails -import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath} import org.apache.carbondata.core.util.{CarbonProperties, CarbonUtil} -import org.apache.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter, -CarbonDictionarySortIndexWriterImpl, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator} +import org.apache.carbondata.core.util.path.{CarbonStorePath, CarbonTablePath} import org.apache.carbondata.core.writer.{CarbonDictionaryWriter, CarbonDictionaryWriterImpl, ThriftWriter} +import org.apache.carbondata.core.writer.sortindex.{CarbonDictionarySortIndexWriter, +CarbonDictionarySortIndexWriterImpl, CarbonDictionarySortInfo, CarbonDictionarySortInfoPreparator} import org.apache.carbondata.processing.api.dataloader.SchemaInfo import org.apache.carbondata.processing.constants.TableOptionConstant import org.apache.carbondata.processing.csvload.{BlockDetails, CSVInputFormat, @@ -65,6 +65,7 @@ CSVRecordReaderIterator, StringArrayWritable} import org.apache.carbondata.processing.model.{CarbonDataLoadSchema, CarbonLoadModel} import org.apache.carbondata.processing.newflow.DataLoadExecutor import org.apache.carbondata.processing.newflow.constants.DataLoadProcessorConstants +import org.apache.carbondata.processing.newflow.exception.CarbonDataLoadingException object CarbonDataStoreCreator { @@ -110,7 +111,7 @@ object CarbonDataStoreCreator { loadModel.setDefaultTimestampFormat( CarbonProperties.getInstance.getProperty( CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, - CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT)) + CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS)) loadModel.setDefaultDateFormat( CarbonProperties.getInstance.getProperty( CarbonCommonConstants.CARBON_DATE_FORMAT, @@ -397,7 +398,11 @@ object CarbonDataStoreCreator { dictionarySortInfo.getSortIndexInverted) } catch { - case exception: Exception => logger.error(s"exception occurs $exception") + case exception: Exception => + + + logger.error(s"exception occurs $exception") + throw new CarbonDataLoadingException("Data Loading Failed") } finally carbonDictionaryWriter.close() } @@ -545,13 +550,14 @@ object CarbonDataStoreCreator { writeOperation.close() } catch { - case exception: Exception => logger.error(s"Exception occurs $exception") + case exception: Exception => logger.error(s"exception occurs $exception") + throw new CarbonDataLoadingException("Data Loading Failed") } } private def readCurrentTime(): String = { val sdf: SimpleDateFormat = new SimpleDateFormat( - CarbonCommonConstants.CARBON_TIMESTAMP) + CarbonCommonConstants.CARBON_TIMESTAMP_MILLIS) sdf.format(new Date()) }
