http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/Vector2TestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/Vector2TestCase.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/Vector2TestCase.scala new file mode 100644 index 0000000..21423f1 --- /dev/null +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/generated/Vector2TestCase.scala @@ -0,0 +1,625 @@ + +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.carbondata.cluster.sdv.generated + +import org.apache.spark.sql.common.util._ +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties + +/** + * Test Class for Vector2TestCase to verify all scenerios + */ + +class Vector2TestCase extends QueryTest with BeforeAndAfterAll { + + + //To check select all records with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_071", Include) { + sql(s"""CREATE TABLE uniqdatavector2 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table uniqdatavector2 OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""select * from uniqdatavector2 """).collect + + + } + + + //To check random measure select query with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_072", Include) { + + sql(s"""select cust_name,DOB,DOJ from uniqdatavector2 where cust_id=10999""").collect + + sql(s"""drop table uniqdatavector2""").collect + } + + + //To check select random columns and order with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_073", Include) { + sql(s"""create table double1(id double, name string) STORED BY 'org.apache.carbondata.format' """).collect + sql(s"""load data inpath '$resourcesPath/Data/InsertData/maxrange_double.csv' into table double1""").collect + sql(s"""select id from double1 order by id""").collect + + + } + + + //To check the logs of executor with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_074", Include) { + + sql(s"""select id from double1 order by id""").collect + + + } + + + //To check for select random measures with group by and having clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_075", Include) { + + sql(s"""select id,count(*) from double1 group by id having count(*)=1""").collect + + + } + + + //To check for select count query with group by and having clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_076", Include) { + + sql(s"""select id,count(id) from double1 group by id having count(*)=1""").collect + + sql(s"""drop table double1""").collect + } + + + //To applied cast method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_077", Include) { + sql(s"""CREATE TABLE uniqdatavector22 (CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string, DOB timestamp, DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10), DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'carbondata'""").collect + sql(s"""LOAD DATA INPATH '$resourcesPath/Data/uniqdata/2000_UniqData.csv' into table uniqdatavector22 OPTIONS('DELIMITER'=',' , 'QUOTECHAR'='"','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='CUST_ID,CUST_NAME,ACTIVE_EMUI_VERSION,DOB,DOJ,BIGINT_COLUMN1,BIGINT_COLUMN2,DECIMAL_COLUMN1,DECIMAL_COLUMN2,Double_COLUMN1,Double_COLUMN2,INTEGER_COLUMN1')""").collect + sql(s"""select cast(Double_COLUMN1 as int) from uniqdatavector22""").collect + + + } + + + //To apply sum method on a column with select query with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_078", Include) { + + sql(s"""select sum(CUST_ID) from uniqdatavector22""").collect + + + } + + + //To apply the average method on a column with select query with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_079", Include) { + + sql(s"""select avg(CUST_ID) from uniqdatavector22""").collect + + + } + + + //To apply the percentile_approx method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_080", Include) { + + sql(s"""select percentile_approx(1, 0.5 ,500) from uniqdatavector22""").collect + + + } + + + //To apply the var_samp method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_081", Include) { + + sql(s"""select var_samp(cust_id) from uniqdatavector22""").collect + + + } + + + //To apply the stddev_pop method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_082", Include) { + + sql(s"""select stddev_pop(cust_id) from uniqdatavector22""").collect + + + } + + + //To apply the stddev_samp method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_083", Include) { + + sql(s"""select stddev_samp(cust_id) from uniqdatavector22""").collect + + + } + + + //To apply percentile method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_084", Include) { + + sql(s"""select percentile(0,1) from uniqdatavector22""").collect + + + } + + + //To apply min method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_085", Include) { + + sql(s"""select min(CUST_ID) from uniqdatavector22""").collect + + + } + + + //To applied max method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_086", Include) { + + sql(s"""select max(CUST_ID) from uniqdatavector22""").collect + + + } + + + //To apply sum method with plus operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_087", Include) { + + sql(s"""select sum(CUST_ID+1) from uniqdatavector22""").collect + + + } + + + //To apply sum method with minus operator with vectorized carbon reader enabled + + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_088", Include) { + + sql(s"""select sum(CUST_ID-1) from uniqdatavector22""").collect + + + } + + + //To apply count method with distinct operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_089", Include) { + + sql(s"""select count(DISTINCT CUST_ID) from uniqdatavector22""").collect + + + } + + + //To check random measure select query with AND operator and vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_090", Include) { + + sql(s"""select cust_name,DOB,DOJ from uniqdatavector22 where cust_id=10999 and INTEGER_COLUMN1=2000 """).collect + + + } + + + //To check random measure select query with OR operator and vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_091", Include) { + + sql(s"""select cust_name,DOB,DOJ from uniqdatavector22 where cust_id=10999 or INTEGER_COLUMN1=2000 """).collect + + + } + + + //To apply count method with if operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_092", Include) { + + sql(s"""select count(if(CUST_ID<1999,NULL,CUST_NAME)) from uniqdatavector22""").collect + + + } + + + //To apply in operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_093", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID IN(1,22)""").collect + + + } + + + //To apply not in operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_094", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID NOT IN(1,22)""").collect + + + } + + + //To apply between operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_095", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID BETWEEN 1 AND 11000""").collect + + + } + + + //To apply not between operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_096", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID NOT BETWEEN 1 AND 11000""").collect + + + } + + + //To apply between in operator with order by clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_097", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID in (1,10999)order by 'CUST_ID'""").collect + + + } + + + //To apply between in operator with group by clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_098", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID in (1,10999) group by CUST_NAME""").collect + + + } + + + //To apply null clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_099", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID is null""").collect + + + } + + + //To applied not null clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_100", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID is not null""").collect + + + } + + + //To apply > operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_101", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID>1""").collect + + + } + + + //To apply < operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_102", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID<1""").collect + + + } + + + //To apply != operator with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_103", Include) { + + sql(s"""select CUST_NAME from uniqdatavector22 where CUST_ID!=1""").collect + + + } + + + //To apply like clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_104", Include) { + + sql(s"""select CUST_ID from uniqdatavector22 where CUST_ID like 10999""").collect + + + } + + + //To apply like% clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_105", Include) { + + sql(s"""select CUST_ID from uniqdatavector22 where CUST_ID like '%10999%'""").collect + + + } + + + //To apply rlike clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_106", Include) { + + sql(s"""select CUST_ID from uniqdatavector22 where CUST_ID rlike 10999""").collect + + + } + + + //To apply rlike% clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_107", Include) { + + sql(s"""select CUST_ID from uniqdatavector22 where CUST_ID rlike '%10999'""").collect + + + } + + + //To apply alias clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_108", Include) { + + sql(s"""select count(cust_id)+10.364 as a from uniqdatavector22""").collect + + + } + + + //To apply aliase clause with group by clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_109", Include) { + + sql(s"""select count(cust_id)+10.364 as a from uniqdatavector22 group by CUST_ID""").collect + + + } + + + //To apply aliase clause with order by clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_110", Include) { + + sql(s"""select cust_id,count(cust_name) a from uniqdatavector22 group by cust_id order by cust_id""").collect + + + } + + + //To apply regexp_replace clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_111", Include) { + + sql(s"""select regexp_replace(cust_id, 'i', 'ment') from uniqdatavector22""").collect + + + } + + + //To apply date_add method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_118", Include) { + + sql(s"""SELECT date_add(DOB,1) FROM uniqdatavector22""").collect + + + } + + + //To apply date_sub method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_119", Include) { + + sql(s"""SELECT date_sub(DOB,1) FROM uniqdatavector22""").collect + + + } + + + //To apply current_date method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_120", Include) { + + sql(s"""SELECT current_date() FROM uniqdatavector22""").collect + + + } + + + //To apply add_month method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_121", Include) { + + sql(s"""SELECT add_months(dob,1) FROM uniqdatavector22""").collect + + + } + + + //To apply last_day method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_122", Include) { + + sql(s"""SELECT last_day(dob) FROM uniqdatavector22""").collect + + + } + + + //To apply next_day method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_123", Include) { + + sql(s"""SELECT next_day(dob,'monday') FROM uniqdatavector22""").collect + + + } + + + //To apply months_between method on carbon table + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_124", Include) { + + sql(s"""select months_between('2016-12-28', '2017-01-30') from uniqdatavector22""").collect + + + } + + + //Toapply date_diff method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_125", Include) { + + sql(s"""select datediff('2009-03-01', '2009-02-27') from uniqdatavector22""").collect + + + } + + + //To apply concat method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_126", Include) { + + sql(s"""SELECT concat('hi','hi') FROM uniqdatavector22""").collect + + + } + + + //To apply lower method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_127", Include) { + + sql(s"""SELECT lower('H') FROM uniqdatavector22""").collect + + + } + + + //To apply substr method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_128", Include) { + + sql(s"""select substr(cust_id,3) from uniqdatavector22""").collect + + + } + + + //To apply trim method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_129", Include) { + + sql(s"""select trim(cust_id) from uniqdatavector22""").collect + + + } + + + //To apply split method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_130", Include) { + + sql(s"""select split('knoldus','ol') from uniqdatavector22""").collect + + + } + + + //To apply split method limit clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_131", Include) { + + sql(s"""select split('knoldus','ol') from uniqdatavector22 limit 1""").collect + + + } + + + //To apply reverse on carbon table with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_132", Include) { + + sql(s"""select reverse('knoldus') from uniqdatavector22""").collect + + + } + + + //To apply replace on carbon table with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_133", Include) { + + sql(s"""select regexp_replace('Tester', 'T', 't') from uniqdatavector22""").collect + + + } + + + //To apply replace with limit clause with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_134", Include) { + + sql(s"""select regexp_replace('Tester', 'T', 't') from uniqdatavector22 limit 1""").collect + + + } + + + //To apply FORMAT_STRING on carbon table with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_135", Include) { + + sql(s"""select format_string('data', cust_name) from uniqdatavector22""").collect + + + } + + + //To apply sentences method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_136", Include) { + + sql(s"""select sentences(cust_name) from uniqdatavector22""").collect + + + } + + + //To apply space method on carbon table with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_137", Include) { + + sql(s"""select space(10) from uniqdatavector22""").collect + + + } + + + //To apply rtrim method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_138", Include) { + + sql(s"""select rtrim(" testing ") from uniqdatavector22""").collect + + + } + + + //To apply ascii method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_139", Include) { + + sql(s"""select ascii('A') from uniqdatavector22""").collect + + + } + + + //To apply utc_timestamp method with vectorized carbon reader enabled + test("PTS-AR-Productize-New-Features-Huawei-Spark2.1-007-01-01-01_001-TC_140", Include) { + + sql(s"""select from_utc_timestamp('2016-12-12 08:00:00','PST') from uniqdatavector22""").collect + + sql(s"""drop table uniqdatavector22""").collect + } + + val prop = CarbonProperties.getInstance() + val p1 = prop.getProperty("carbon.enable.vector.reader", CarbonCommonConstants.ENABLE_VECTOR_READER_DEFAULT) + + override protected def beforeAll() { + // Adding new properties + prop.addProperty("carbon.enable.vector.reader", "false") + } + + override def afterAll: Unit = { + //Reverting to old + prop.addProperty("carbon.enable.vector.reader", p1) + } +} \ No newline at end of file
http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala new file mode 100644 index 0000000..de946a0 --- /dev/null +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/carbondata/cluster/sdv/suite/SDVSuites.scala @@ -0,0 +1,125 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.carbondata.cluster.sdv.suite + +import org.apache.spark.sql.test.TestQueryExecutor +import org.scalatest.{BeforeAndAfterAll, Suites} + +import org.apache.carbondata.cluster.sdv.generated._ + +/** + * Suite class for all tests. + */ +class SDVSuites extends Suites with BeforeAndAfterAll { + + val suites = new AlterTableTestCase :: + new BadRecordTestCase :: + new BatchSortLoad1TestCase :: + new BatchSortLoad2TestCase :: + new BatchSortQueryTestCase :: + new ColumndictTestCase :: + new DataLoadingTestCase :: + new DataLoadingV3TestCase :: + new InvertedindexTestCase :: + new OffheapQuery1TestCase :: + new OffheapQuery2TestCase :: + new OffheapSort1TestCase :: + new OffheapSort2TestCase :: + new PartitionTestCase :: + new QueriesBasicTestCase :: + new QueriesBVATestCase :: + new QueriesCompactionTestCase :: + new QueriesExcludeDictionaryTestCase :: + new QueriesIncludeDictionaryTestCase :: + new QueriesNormalTestCase :: + new QueriesRangeFilterTestCase :: + new QueriesSparkBlockDistTestCase :: + new ShowLoadsTestCase :: + new SinglepassTestCase :: + new SortColumnTestCase :: + new TimestamptypesTestCase :: + new V3offheapvectorTestCase :: + new Vector1TestCase :: + new Vector2TestCase ::Nil + + override val nestedSuites = suites.toIndexedSeq + + override protected def afterAll() = { + println("---------------- Stopping spark -----------------") + TestQueryExecutor.INSTANCE.stop() + println("---------------- Stopped spark -----------------") + } +} + +/** + * Suite class for all tests. + */ +class SDVSuites1 extends Suites with BeforeAndAfterAll { + + val suites = new AlterTableTestCase :: + new BadRecordTestCase :: + new BatchSortLoad1TestCase :: + new BatchSortLoad2TestCase :: + new BatchSortQueryTestCase :: + new ColumndictTestCase :: + new DataLoadingTestCase :: + new DataLoadingV3TestCase :: + new InvertedindexTestCase :: + new OffheapQuery1TestCase :: + new OffheapQuery2TestCase :: + new OffheapSort1TestCase :: + new OffheapSort2TestCase :: + new PartitionTestCase :: + new QueriesBasicTestCase :: Nil + + override val nestedSuites = suites.toIndexedSeq + + override protected def afterAll() = { + println("---------------- Stopping spark -----------------") + TestQueryExecutor.INSTANCE.stop() + println("---------------- Stopped spark -----------------") + } +} + +/** + * Suite class for all tests. + */ +class SDVSuites2 extends Suites with BeforeAndAfterAll { + + val suites = new QueriesBVATestCase :: + new QueriesCompactionTestCase :: + new QueriesExcludeDictionaryTestCase :: + new QueriesIncludeDictionaryTestCase :: + new QueriesNormalTestCase :: + new QueriesRangeFilterTestCase :: + new QueriesSparkBlockDistTestCase :: + new ShowLoadsTestCase :: + new SinglepassTestCase :: + new SortColumnTestCase :: + new TimestamptypesTestCase :: + new V3offheapvectorTestCase :: + new Vector1TestCase :: + new Vector2TestCase :: Nil + + override val nestedSuites = suites.toIndexedSeq + + override protected def afterAll() = { + println("---------------- Stopping spark -----------------") + TestQueryExecutor.INSTANCE.stop() + println("---------------- Stopped spark -----------------") + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala new file mode 100644 index 0000000..37803a8 --- /dev/null +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/CarbonFunSuite.scala @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.common.util + +import org.apache.carbondata.common.logging.LogServiceFactory +import org.scalatest.{FunSuite, Outcome} + + +private[spark] abstract class CarbonFunSuite extends FunSuite { + + private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) + + /** + * Log the suite name and the test name before and after each test. + * + * Subclasses should never override this method. If they wish to run + * custom code before and after each test, they should should mix in + * the {{org.scalatest.BeforeAndAfter}} trait instead. + */ + final protected override def withFixture(test: NoArgTest): Outcome = { + val testName = test.text + val suiteName = this.getClass.getName + val shortSuiteName = suiteName.replaceAll("org.apache.spark", "o.a.s") + try { + LOGGER.info(s"\n\n===== TEST OUTPUT FOR $shortSuiteName: '$testName' =====\n") + test() + } finally { + LOGGER.info(s"\n\n===== FINISHED $shortSuiteName: '$testName' =====\n") + } + } + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala new file mode 100644 index 0000000..cdd415f --- /dev/null +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/PlanTest.scala @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.common.util + +import org.apache.spark.sql.catalyst.expressions._ +import org.apache.spark.sql.catalyst.plans.logical.{Filter, LogicalPlan, OneRowRelation} +import org.apache.spark.sql.catalyst.util._ + +/** + * Provides helper methods for comparing plans. + */ +class PlanTest extends CarbonFunSuite { + + /** Fails the test if the two expressions do not match */ + protected def compareExpressions(e1: Expression, e2: Expression): Unit = { + comparePlans(Filter(e1, OneRowRelation), Filter(e2, OneRowRelation)) + } + + /** Fails the test if the two plans do not match */ + protected def comparePlans(plan1: LogicalPlan, plan2: LogicalPlan) { + val normalized1 = normalizeExprIds(plan1) + val normalized2 = normalizeExprIds(plan2) + if (normalized1 != normalized2) { + fail( + s""" + |== FAIL: Plans do not match === + |${sideBySide(normalized1.treeString, normalized2.treeString).mkString("\n")} + """.stripMargin) + } + } + + /** + * Since attribute references are given globally unique ids during analysis, + * we must normalize them to check if two different queries are identical. + */ + protected def normalizeExprIds(plan: LogicalPlan) = { + plan transformAllExpressions { + case a: AttributeReference => + AttributeReference(a.name, a.dataType, a.nullable)(exprId = ExprId(0)) + case a: Alias => + Alias(a.child, a.name)(exprId = ExprId(0)) + } + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala new file mode 100644 index 0000000..292c160 --- /dev/null +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/QueryTest.scala @@ -0,0 +1,199 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.spark.sql.common.util + +import java.io.{FileInputStream, ObjectInputStream, ObjectOutputStream} +import java.math +import java.math.RoundingMode +import java.util.{Locale, TimeZone} + +import org.apache.carbondata.common.logging.LogServiceFactory +import scala.collection.JavaConversions._ + +import org.apache.spark.sql.catalyst.plans._ +import org.apache.spark.sql.catalyst.util._ +import org.apache.spark.sql.execution.command.LoadDataCommand +import org.apache.spark.sql.test.{ResourceRegisterAndCopier, TestQueryExecutor} +import org.apache.spark.sql.{DataFrame, Row, SQLContext} +import org.scalatest.Suite + +import org.apache.carbondata.core.datastore.impl.FileFactory + +class QueryTest extends PlanTest with Suite { + + val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) + + val DOLLAR = "$" + + // Timezone is fixed to America/Los_Angeles for those timezone sensitive tests (timestamp_*) + TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles")) + // Add Locale setting + Locale.setDefault(Locale.US) + + /** + * Runs the plan and makes sure the answer contains all of the keywords, or the + * none of keywords are listed in the answer + * @param df the [[DataFrame]] to be executed + * @param exists true for make sure the keywords are listed in the output, otherwise + * to make sure none of the keyword are not listed in the output + * @param keywords keyword in string array + */ + def checkExistence(df: DataFrame, exists: Boolean, keywords: String*) { + val outputs = df.collect().map(_.mkString).mkString + for (key <- keywords) { + if (exists) { + assert(outputs.contains(key), s"Failed for $df ($key doesn't exist in result)") + } else { + assert(!outputs.contains(key), s"Failed for $df ($key existed in the result)") + } + } + } + + /** + * Runs the plan and makes sure the answer matches the expected result. + * @param df the [[DataFrame]] to be executed + * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. + */ + protected def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Unit = { + QueryTest.checkAnswer(df, expectedAnswer) match { + case Some(errorMessage) => fail(errorMessage) + case None => + } + } + + protected def checkAnswer(df: DataFrame, expectedAnswer: Row): Unit = { + checkAnswer(df, Seq(expectedAnswer)) + } + + protected def checkAnswer(df: DataFrame, expectedAnswer: DataFrame): Unit = { + checkAnswer(df, expectedAnswer.collect()) + } + + protected def checkAnswer(carbon: String, hive: String, uniqueIdentifier:String): Unit = { + val path = TestQueryExecutor.hiveresultpath + "/"+uniqueIdentifier + if (FileFactory.isFileExist(path, FileFactory.getFileType(path))) { + val objinp = new ObjectInputStream(FileFactory.getDataInputStream(path, FileFactory.getFileType(path))) + val rows = objinp.readObject().asInstanceOf[Array[Row]] + objinp.close() + checkAnswer(sql(carbon), rows) + } else { + val rows = sql(hive).collect() + val obj = new ObjectOutputStream(FileFactory.getDataOutputStream(path, FileFactory.getFileType(path))) + obj.writeObject(rows) + obj.close() + checkAnswer(sql(carbon), rows) + } + } + + protected def checkAnswer(carbon: String, expectedAnswer: Seq[Row], uniqueIdentifier:String): Unit = { + checkAnswer(sql(carbon), expectedAnswer) + } + + def sql(sqlText: String): DataFrame = { + val frame = TestQueryExecutor.INSTANCE.sql(sqlText) + val plan = frame.queryExecution.logical + if (TestQueryExecutor.hdfsUrl.startsWith("hdfs")) { + plan match { + case l: LoadDataCommand => + val copyPath = TestQueryExecutor.warehouse + "/" + l.table.table.toLowerCase + + l.path.substring(l.path.lastIndexOf("/"), l.path.length) + ResourceRegisterAndCopier.copyLocalFile(l.path, copyPath) + case _ => + } + } + frame + } + + protected def dropTable(tableName: String): Unit ={ + sql(s"DROP TABLE IF EXISTS $tableName") + } + + val sqlContext: SQLContext = TestQueryExecutor.INSTANCE.sqlContext + + val storeLocation = TestQueryExecutor.storeLocation + val resourcesPath = TestQueryExecutor.resourcesPath + val integrationPath = TestQueryExecutor.integrationPath +} + +object QueryTest { + + def checkAnswer(df: DataFrame, expectedAnswer: java.util.List[Row]): String = { + checkAnswer(df, expectedAnswer.toSeq) match { + case Some(errorMessage) => errorMessage + case None => null + } + } + + import java.text.DecimalFormat + + /** + * Runs the plan and makes sure the answer matches the expected result. + * If there was exception during the execution or the contents of the DataFrame does not + * match the expected result, an error message will be returned. Otherwise, a [[None]] will + * be returned. + * @param df the [[DataFrame]] to be executed + * @param expectedAnswer the expected result in a [[Seq]] of [[Row]]s. + */ + def checkAnswer(df: DataFrame, expectedAnswer: Seq[Row]): Option[String] = { + val isSorted = df.logicalPlan.collect { case s: logical.Sort => s }.nonEmpty + def prepareAnswer(answer: Seq[Row]): Seq[Row] = { + // Converts data to types that we can do equality comparison using Scala collections. + // For BigDecimal type, the Scala type has a better definition of equality test (similar to + // Java's java.math.BigDecimal.compareTo). + // For binary arrays, we convert it to Seq to avoid of calling java.util.Arrays.equals for + // equality test. + val converted: Seq[Row] = answer.map { s => + Row.fromSeq(s.toSeq.map { + case d: java.math.BigDecimal => BigDecimal(d) + case b: Array[Byte] => b.toSeq + case o => o + }) + } + if (!isSorted) converted.sortBy(_.toString()) else converted + } + val sparkAnswer = try df.collect().toSeq catch { + case e: Exception => + val errorMessage = + s""" + |Exception thrown while executing query: + |== Exception == + |$e + |${org.apache.spark.sql.catalyst.util.stackTraceToString(e)} + """.stripMargin + return Some(errorMessage) + } + + if (prepareAnswer(expectedAnswer) != prepareAnswer(sparkAnswer)) { + val errorMessage = + s""" + |Results do not match for query: + |== Results == + |${ + sideBySide( + s"== Correct Answer - ${expectedAnswer.size} ==" +: + prepareAnswer(expectedAnswer).map(_.toString()), + s"== Spark Answer - ${sparkAnswer.size} ==" +: + prepareAnswer(sparkAnswer).map(_.toString())).mkString("\n") + } + """.stripMargin + return Some(errorMessage) + } + + return None + } +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala new file mode 100644 index 0000000..7d6770b --- /dev/null +++ b/integration/spark-common-cluster-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under onassemblye + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.common.util + +import org.scalatest.Tag + +object Include extends Tag("Include") + +object Exclude extends Tag("Exclude") http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/pom.xml ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/pom.xml b/integration/spark-common-test/pom.xml index 613d59f..fbb4f8d 100644 --- a/integration/spark-common-test/pom.xml +++ b/integration/spark-common-test/pom.xml @@ -137,6 +137,7 @@ <version>1.0</version> <!-- Note config is repeated in surefire config --> <configuration> + <skipTests>false</skipTests> <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory> <junitxml>.</junitxml> <filereports>CarbonTestSuite.txt</filereports> @@ -227,5 +228,11 @@ </dependency> </dependencies> </profile> + <profile> + <id>sdvtest</id> + <properties> + <maven.test.skip>true</maven.test.skip> + </properties> + </profile> </profiles> </project> http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala index adbc1b1..c09d285 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/dataload/TestDataLoadWithColumnsMoreThanSchema.scala @@ -128,7 +128,9 @@ class TestDataLoadWithColumnsMoreThanSchema extends QueryTest with BeforeAndAfte sql(s"LOAD DATA INPATH '$resourcesPath/seq_20Records.csv' into table smart_500_DE options('DELIMITER'=',','BAD_RECORDS_ACTION'='FORCE','FILEHEADER'='SID,PROBEID,INTERFACEID,GROUPID,GGSN_ID,SGSN_ID,dummy,SESSION_INDICATOR,BEGIN_TIME,BEGIN_TIME_MSEL,END_TIME,END_TIME_MSEL,PROT_CATEGORY,PROT_TYPE,L7_CARRIER_PROT,SUB_PROT_TYPE,MSISDN,IMSI,IMEI,ENCRYPT_VERSION,ROAMING_TYPE,ROAM_DIRECTION,MS_IP,SERVER_IP,MS_PORT,APN,SGSN_SIG_IP,GGSN_USER_IP,SGSN_USER_IP,MCC,MNC,RAT,LAC,RAC,SAC,CI,SERVER_DECIMAL,BROWSER_TIMESTAMP,TCP_CONN_STATES,GGSN_SIG_IP_BigInt_NEGATIVE,TCP_STATES_BIGINTPOSITIVE,dummy,TCP_WIN_SIZE,dummy,TCP_MSS,dummy,TCP_CONN_TIMES,dummy,TCP_CONN_2_FAILED_TIMES,dummy,TCP_CONN_3_FAILED_TIMES,HOST,STREAMING_URL,dummy,GET_STREAMING_FAILED_CODE,dummy,GET_STREAMING_FLAG,dummy,GET_NUM,dummy,GET_SUCCEED_NUM,dummy,GET_RETRANS_NUM,dummy,GET_TIMEOUT_NUM,INTBUFFER_FST_FLAG,INTBUFFER_FULL_FLAG,STALL_NUM,dummy,VIDEO_FRAME_RATE,dummy,VIDEO_CODEC_ID,dummy,VIDEO_WIDTH,dummy,VIDEO_HEIGHT,dummy,AUD IO_CODEC_ID,dummy,MEDIA_FILE_TYPE,dummy,PLAY_STATE,dummy,PLAY_STATE,dummy,STREAMING_FLAG,dummy,TCP_STATUS_INDICATOR,dummy,DISCONNECTION_FLAG,dummy,FAILURE_CODE,FLAG,TAC,ECI,dummy,TCP_SYN_TIME_MSEL,dummy,TCP_FST_SYN_DIRECTION,RAN_NE_USER_IP,HOMEMCC,HOMEMNC,dummy,CHARGE_FLAG,dummy,PREPAID_FLAG,dummy,USER_AGENT,dummy,MS_WIN_STAT_TOTAL_NUM,dummy,MS_WIN_STAT_SMALL_NUM,dummy,MS_ACK_TO_1STGET_DELAY,dummy,SERVER_ACK_TO_1STDATA_DELAY,dummy,STREAMING_TYPE,dummy,SOURCE_VIDEO_QUALITY,TETHERING_FLAG,CARRIER_ID,LAYER1ID,LAYER2ID,dummy,LAYER3ID,dummy,LAYER4ID,dummy,LAYER5ID,dummy,LAYER6ID,CHARGING_RULE_BASE_NAME,SP,dummy,EXTENDED_URL,SV,FIRST_SAI_CGI_ECGI,dummy,EXTENDED_URL_OTHER,SIGNALING_USE_FLAG,dummy,DNS_RETRANS_NUM,dummy,DNS_FAIL_CODE,FIRST_RAT,FIRST_RAT,MS_INDICATOR,LAST_SAI_CGI_ECGI,LAST_RAT,dummy,FIRST_LONGITUDE,dummy,FIRST_LATITUDE,dummy,FIRST_ALTITUDE,dummy,FIRST_RASTERLONGITUDE,dummy,FIRST_RASTERLATITUDE,dummy,FIRST_RASTERALTITUDE,dummy,FIRST_FREQUENCYSPOT,dummy,FIRST_CLUTTER,dummy,FIRS T_USERBEHAVIOR,dummy,FIRST_SPEED,dummy,FIRST_CREDIBILITY,dummy,LAST_LONGITUDE,dummy,LAST_LATITUDE,dummy,LAST_ALTITUDE,dummy,LAST_RASTERLONGITUDE,dummy,LAST_RASTERLATITUDE,dummy,LAST_RASTERALTITUDE,dummy,LAST_FREQUENCYSPOT,dummy,LAST_CLUTTER,dummy,LAST_USERBEHAVIOR,dummy,LAST_SPEED,dummy,LAST_CREDIBILITY,dummy,IMEI_CIPHERTEXT,APP_ID,dummy,DOMAIN_NAME,dummy,STREAMING_CACHE_IP,dummy,STOP_LONGER_THAN_MIN_THRESHOLD,dummy,STOP_LONGER_THAN_MAX_THRESHOLD,dummy,PLAY_END_STAT,dummy,STOP_START_TIME1,dummy,STOP_END_TIME1,dummy,STOP_START_TIME2,dummy,STOP_END_TIME2,dummy,STOP_START_TIME3,dummy,STOP_END_TIME3,dummy,STOP_START_TIME4,dummy,STOP_END_TIME4,dummy,STOP_START_TIME5,dummy,STOP_END_TIME5,dummy,STOP_START_TIME6,dummy,STOP_END_TIME6,dummy,STOP_START_TIME7,dummy,STOP_END_TIME7,dummy,STOP_START_TIME8,dummy,STOP_END_TIME8,dummy,STOP_START_TIME9,dummy,STOP_END_TIME9,dummy,STOP_START_TIME10,dummy,STOP_END_TIME10,dummy,FAIL_CLASS,RECORD_TYPE,dummy,NODATA_COUNT,dummy,VIDEO_NODATA_DURATION,dummy,VI DEO_SMOOTH_DURATION,dummy,VIDEO_SD_DURATION,dummy,VIDEO_HD_DURATION,dummy,VIDEO_UHD_DURATION,dummy,VIDEO_FHD_DURATION,dummy,FLUCTUATION,dummy,START_DOWNLOAD_THROUGHPUT,dummy,L7_UL_GOODPUT_FULL_MSS,dummy,SESSIONKEY,dummy,FIRST_UCELLID,dummy,LAST_UCELLID,dummy,UCELLID1,dummy,LONGITUDE1,dummy,LATITUDE1,dummy,UCELLID2,dummy,LONGITUDE2,dummy,LATITUDE2,dummy,UCELLID3,dummy,LONGITUDE3,dummy,LATITUDE3,dummy,UCELLID4,dummy,LONGITUDE4,dummy,LATITUDE4,dummy,UCELLID5,dummy,LONGITUDE5,dummy,LATITUDE5,dummy,UCELLID6,dummy,LONGITUDE6,dummy,LATITUDE6,dummy,UCELLID7,dummy,LONGITUDE7,dummy,LATITUDE7,dummy,UCELLID8,dummy,LONGITUDE8,dummy,LATITUDE8,dummy,UCELLID9,dummy,LONGITUDE9,dummy,LATITUDE9,dummy,UCELLID10,dummy,LONGITUDE10,dummy,LATITUDE10,dummy,INTBUFFER_FULL_DELAY,dummy,STALL_DURATION,dummy,STREAMING_DW_PACKETS,dummy,STREAMING_DOWNLOAD_DELAY,dummy,PLAY_DURATION,dummy,STREAMING_QUALITY,dummy,VIDEO_DATA_RATE,dummy,AUDIO_DATA_RATE,dummy,STREAMING_FILESIZE,dummy,STREAMING_DURATIOIN,dummy,TCP_SYN_TI ME,dummy,TCP_RTT_STEP1,CHARGE_ID,dummy,UL_REVERSE_TO_DL_DELAY,dummy,DL_REVERSE_TO_UL_DELAY,dummy,DATATRANS_DW_GOODPUT,dummy,DATATRANS_DW_TOTAL_DURATION,dummy,SUM_FRAGMENT_INTERVAL,dummy,TCP_FIN_TIMES,dummy,TCP_RESET_TIMES,dummy,URL_CLASSIFICATION,dummy,STREAMING_LQ_DURATIOIN,dummy,MAX_DNS_DELAY,dummy,MAX_DNS2SYN,dummy,MAX_LATANCY_OF_LINK_SETUP,dummy,MAX_SYNACK2FIRSTACK,dummy,MAX_SYNACK2LASTACK,dummy,MAX_ACK2GET_DELAY,dummy,MAX_FRAG_INTERVAL_PREDELAY,dummy,SUM_FRAG_INTERVAL_PREDELAY,dummy,SERVICE_DELAY_MSEC,dummy,HOMEPROVINCE,dummy,HOMECITY,dummy,SERVICE_ID,dummy,CHARGING_CLASS,dummy,DATATRANS_UL_DURATION,dummy,ASSOCIATED_ID,dummy,PACKET_LOSS_NUM,dummy,JITTER,dummy,MS_DNS_DELAY_MSEL,dummy,GET_STREAMING_DELAY,dummy,TCP_UL_RETRANS_WITHOUTPL,dummy,TCP_DW_RETRANS_WITHOUTPL,dummy,GET_MAX_UL_SIZE,dummy,GET_MIN_UL_SIZE,dummy,GET_MAX_DL_SIZE,dummy,GET_MIN_DL_SIZE,dummy,FLOW_SAMPLE_RATIO,dummy,UL_RTT_LONG_NUM,dummy,DW_RTT_LONG_NUM,dummy,UL_RTT_STAT_NUM,dummy,DW_RTT_STAT_NUM,dummy,USER_PROBE_U L_LOST_PKT,dummy,SERVER_PROBE_UL_LOST_PKT,dummy,SERVER_PROBE_DW_LOST_PKT,dummy,USER_PROBE_DW_LOST_PKT,dummy,AVG_DW_RTT_MICRO_SEC,dummy,AVG_UL_RTT_MICRO_SEC,dummy,RAN_NE_ID,dummy,AVG_UL_RTT,dummy,AVG_DW_RTT,dummy,CHARGING_CHARACTERISTICS,dummy,DL_SERIOUS_OUT_OF_ORDER_NUM,dummy,DL_SLIGHT_OUT_OF_ORDER_NUM,dummy,DL_FLIGHT_TOTAL_SIZE,dummy,DL_FLIGHT_TOTAL_NUM,dummy,DL_MAX_FLIGHT_SIZE,dummy,VALID_TRANS_DURATION,dummy,AIR_PORT_DURATION,dummy,RADIO_CONN_TIMES,dummy,UL_SERIOUS_OUT_OF_ORDER_NUM,dummy,UL_SLIGHT_OUT_OF_ORDER_NUM,dummy,UL_FLIGHT_TOTAL_SIZE,dummy,UL_FLIGHT_TOTAL_NUM,dummy,UL_MAX_FLIGHT_SIZE,dummy,USER_DL_SLIGHT_OUT_OF_ORDER_PACKETS,dummy,SERVER_UL_SLIGHT_OUT_OF_ORDER_PACKETS,dummy,DL_CONTINUOUS_RETRANSMISSION_DELAY,dummy,USER_HUNGRY_DELAY,dummy,SERVER_HUNGRY_DELAY,dummy,UPPERLAYER_IP_UL_FRAGMENTS,dummy,UPPERLAYER_IP_DL_FRAGMENTS,dummy,DOWNLAYER_IP_UL_FRAGMENTS,dummy,DOWNLAYER_IP_DL_FRAGMENTS,dummy,UPPERLAYER_IP_UL_PACKETS,dummy,UPPERLAYER_IP_DL_PACKETS,dummy,DOWNLAYER_IP_UL_PACKE TS,dummy,DOWNLAYER_IP_DL_PACKETS,dummy,TCP_UL_PACKAGES_WITHPL,dummy,TCP_DW_PACKAGES_WITHPL,dummy,TCP_UL_PACKAGES_WITHOUTPL,dummy,TCP_DW_PACKAGES_WITHOUTPL,dummy,TCP_UL_RETRANS_WITHPL,dummy,TCP_DW_RETRANS_WITHPL,L4_UL_THROUGHPUT,L4_DW_THROUGHPUT,L4_UL_GOODPUT,L4_DW_GOODPUT,NETWORK_UL_TRAFFIC,NETWORK_DL_TRAFFIC,L4_UL_PACKETS,L4_DW_PACKETS,TCP_RTT,TCP_UL_OUTOFSEQU,TCP_DW_OUTOFSEQU,TCP_UL_RETRANS,TCP_DW_RETRANS')") assert(true) } catch { - case _: Throwable => assert(false) + case e: Throwable => + e.printStackTrace() + assert(false) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala index 59f7a98..9018ec0 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryTest.scala @@ -53,7 +53,8 @@ class DateDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfterAll case x: Throwable => x.printStackTrace() CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) } } @@ -131,7 +132,8 @@ class DateDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfterAll override def afterAll { sql("drop table directDictionaryTable") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala index 463489b..195b107 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeDirectDictionaryWithNoDictTestCase.scala @@ -56,7 +56,8 @@ class DateDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with Befo case x: Throwable => x.printStackTrace() CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) } } @@ -94,7 +95,8 @@ class DateDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with Befo override def afterAll { sql("drop table directDictionaryTable") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala index d00a108..20e1d2d 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/DateDataTypeNullDataTest.scala @@ -52,7 +52,8 @@ class DateDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll { case x: Throwable => x.printStackTrace() CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) } } @@ -73,7 +74,8 @@ class DateDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll { override def afterAll { sql("drop table timestampTyeNullData") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala index ca337cc..b8d057c 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryTestCase.scala @@ -64,7 +64,8 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte sql("LOAD DATA local inpath '" + csvFilePath + "' INTO TABLE directDictionaryTable_hive") } catch { case x: Throwable => CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) } } @@ -144,7 +145,8 @@ class TimestampDataTypeDirectDictionaryTest extends QueryTest with BeforeAndAfte sql("drop table directDictionaryTable") sql("drop table directDictionaryTable_hive") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala index 26f9ff7..4878c86 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeDirectDictionaryWithNoDictTestCase.scala @@ -56,7 +56,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')"); } catch { case x: Throwable => CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) } } @@ -91,7 +92,8 @@ class TimestampDataTypeDirectDictionaryWithNoDictTestCase extends QueryTest with override def afterAll { sql("drop table directDictionaryTable") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala index 9b07b1e..bc3d6dd 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/directdictionary/TimestampDataTypeNullDataTest.scala @@ -58,7 +58,8 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll { } catch { case x: Throwable => CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) } } @@ -79,7 +80,8 @@ class TimestampDataTypeNullDataTest extends QueryTest with BeforeAndAfterAll { override def afterAll { sql("drop table timestampTyeNullData") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, + CarbonCommonConstants.CARBON_TIMESTAMP_DEFAULT_FORMAT) CarbonProperties.getInstance().addProperty("carbon.direct.dictionary", "false") } http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala index 4aa6954..54586c2 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestAllDataTypeForPartitionTable.scala @@ -39,8 +39,8 @@ class TestAllDataTypeForPartitionTable extends QueryTest with BeforeAndAfterAll override def afterAll = { CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat) - .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT) + .addProperty(CarbonCommonConstants.CARBON_DATE_FORMAT, + CarbonCommonConstants.CARBON_DATE_DEFAULT_FORMAT) dropTable } http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala index a05d908..0217486 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestCompactionForPartitionTable.scala @@ -71,8 +71,6 @@ class TestCompactionForPartitionTable extends QueryTest with BeforeAndAfterAll { override def afterAll = { dropTable - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat) } def dropTable = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala index 41289b1..4ae5ece 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDDLForPartitionTable.scala @@ -356,8 +356,6 @@ class TestDDLForPartitionTable extends QueryTest with BeforeAndAfterAll { override def afterAll = { dropTable - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat) } def dropTable = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala index 4bfc4ef..f7ac87c 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestDataLoadingForPartitionTable.scala @@ -371,8 +371,6 @@ class TestDataLoadingForPartitionTable extends QueryTest with BeforeAndAfterAll override def afterAll = { dropTable - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat) } def dropTable = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala index e02c6bf..df0eea3 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/partition/TestQueryForPartitionTable.scala @@ -272,8 +272,6 @@ class TestQueryForPartitionTable extends QueryTest with BeforeAndAfterAll { override def afterAll = { dropTable - CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat) } def dropTable = { http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala b/integration/spark-common-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala new file mode 100644 index 0000000..7d6770b --- /dev/null +++ b/integration/spark-common-test/src/test/scala/org/apache/spark/sql/common/util/Tags.scala @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under onassemblye + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.spark.sql.common.util + +import org.scalatest.Tag + +object Include extends Tag("Include") + +object Exclude extends Tag("Exclude") http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common/pom.xml ---------------------------------------------------------------------- diff --git a/integration/spark-common/pom.xml b/integration/spark-common/pom.xml index ac42bf3..4e75334 100644 --- a/integration/spark-common/pom.xml +++ b/integration/spark-common/pom.xml @@ -129,6 +129,41 @@ <target>1.7</target> </configuration> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <version>3.0.0</version> + <executions> + <execution> + <id>compile</id> + <phase>compile</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <artifactItems> + <artifactItem> + <groupId>org.apache.carbondata</groupId> + <artifactId>carbondata-format</artifactId> + <version>${project.version}</version> + </artifactItem> + <artifactItem> + <groupId>com.google.code.gson</groupId> + <artifactId>gson</artifactId> + <version>2.3.1</version> + </artifactItem> + <artifactItem> + <groupId>org.xerial.snappy</groupId> + <artifactId>snappy-java</artifactId> + <version>${snappy.version}</version> + </artifactItem> + </artifactItems> + <outputDirectory>${project.build.directory}/jars</outputDirectory> + <!-- other configurations here --> + </configuration> + </execution> + </executions> + </plugin> </plugins> </build> http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala index c908c08..b137d6d 100644 --- a/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala +++ b/integration/spark-common/src/main/scala/org/apache/carbondata/spark/rdd/CarbonRDD.scala @@ -19,13 +19,14 @@ package org.apache.carbondata.spark.rdd import java.io.{ByteArrayInputStream, DataInputStream} +import scala.collection.JavaConverters._ import scala.reflect.ClassTag import org.apache.spark.{Dependency, OneToOneDependency, Partition, SparkContext, TaskContext} import org.apache.spark.rdd.RDD import org.apache.carbondata.core.metadata.schema.table.TableInfo -import org.apache.carbondata.core.util.{CarbonSessionInfo, CarbonTaskInfo, SessionParams, ThreadLocalSessionInfo, ThreadLocalTaskInfo} +import org.apache.carbondata.core.util.{CarbonProperties, CarbonSessionInfo, CarbonTaskInfo, SessionParams, ThreadLocalSessionInfo, ThreadLocalTaskInfo} /** * This RDD maintains session level ThreadLocal @@ -35,6 +36,8 @@ abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext, val carbonSessionInfo: CarbonSessionInfo = ThreadLocalSessionInfo.getCarbonSessionInfo +// val addedProperty = CarbonProperties.getInstance().getAddedProperty + /** Construct an RDD with just a one-to-one dependency on one parent */ def this(@transient oneParent: RDD[_]) = this (oneParent.context, List(new OneToOneDependency(oneParent))) @@ -47,6 +50,7 @@ abstract class CarbonRDD[T: ClassTag](@transient sc: SparkContext, val carbonTaskInfo = new CarbonTaskInfo carbonTaskInfo.setTaskId(System.nanoTime) ThreadLocalTaskInfo.setCarbonTaskInfo(carbonTaskInfo) +// addedProperty.asScala.map(f => CarbonProperties.getInstance().addProperty(f._1, f._2)) internalCompute(split, context) } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common/src/main/scala/org/apache/spark/sql/test/ResourceRegisterAndCopier.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/ResourceRegisterAndCopier.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/ResourceRegisterAndCopier.scala new file mode 100644 index 0000000..6364630 --- /dev/null +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/ResourceRegisterAndCopier.scala @@ -0,0 +1,113 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.spark.sql.test + +import java.io.{BufferedReader, File, FileReader} +import java.net.URL + +import scala.collection.mutable.ArrayBuffer + +import org.apache.hadoop.io.IOUtils + +import org.apache.carbondata.common.logging.LogServiceFactory +import org.apache.carbondata.core.datastore.impl.FileFactory +import org.apache.carbondata.core.util.CarbonUtil + +/** + * All the registered resources will be checked into hdfs and copies to it if not exists. + */ +object ResourceRegisterAndCopier { + + private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) + + val link = "https://raw.githubusercontent"; + + ".com/ravipesala/incubator-carbondata/sdv-test_data/integration/spark-common-test" + + "/src/test/resources" + + def copyResourcesifNotExists(hdfsPath: String, + resourcePath: String, + dataFilesPath: String): Unit = { + val fileType = FileFactory.getFileType(hdfsPath) + val file = FileFactory.getCarbonFile(hdfsPath, fileType) + if (!file.exists()) { + sys.error(s"""Provided path $hdfsPath does not exist""") + } + val resources = readDataFiles(dataFilesPath) + resources.foreach { file => + val hdfsDataPath = hdfsPath + "/" + file + val rsFile = FileFactory.getCarbonFile(hdfsDataPath, fileType) + if (!rsFile.exists()) { + val target = resourcePath + "/" + file + new File(resourcePath + "/" + file.substring(0, file.lastIndexOf("/"))).mkdirs() + downloadFile(link, file, target) + // copy it + copyLocalFile(hdfsDataPath, target) + new File(target).delete() + } + } + } + + def readDataFiles(dataFilesPath: String): Seq[String] = { + val buffer = new ArrayBuffer[String]() + val reader = new BufferedReader(new FileReader(dataFilesPath)) + var line = reader.readLine() + while (line != null) { + buffer += line + line = reader.readLine() + } + reader.close() + buffer + } + + def copyLocalFile(dst: String, + src: String): Unit = { + LOGGER.info(s"Copying file : $src to $dst") + if (FileFactory.isFileExist(src, FileFactory.getFileType(src))) { + val dataOutputStream = FileFactory.getDataOutputStream(dst, + FileFactory.getFileType(dst)) + val dataInputStream = FileFactory.getDataInputStream(src, + FileFactory.getFileType(src)) + IOUtils.copyBytes(dataInputStream, dataOutputStream, 8 * 1024) + CarbonUtil.closeStream(dataInputStream) + CarbonUtil.closeStream(dataOutputStream) + } + } + + def downloadFile(relativeLink: String, fileToDownLoad: String, targetFile: String): Unit = { + import java.io.FileOutputStream + val link = relativeLink + "/" + fileToDownLoad + LOGGER.info(s"Downloading file $link") + val url = new URL(link) + val c = url.openConnection + c.setRequestProperty("User-Agent", + "Mozilla/4.0 (compatible; MSIE 6.0; Windows NT 5.1; .NET CLR 1.0.3705; .NET CLR 1.1.4322;" + + " .NET CLR 1.2.30703)") + + var input = c.getInputStream + val buffer = new Array[Byte](4096) + var n = input.read(buffer) + + val output = new FileOutputStream(new File(targetFile)) + while (n != -1) { + output.write(buffer, 0, n) + n = input.read(buffer) + } + output.close() + input.close() + } + +} http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala index 149e3b1..cf90912 100644 --- a/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala +++ b/integration/spark-common/src/main/scala/org/apache/spark/sql/test/TestQueryExecutor.scala @@ -17,14 +17,18 @@ package org.apache.spark.sql.test -import java.io.File +import java.io.{File, FilenameFilter} import java.util.ServiceLoader +import scala.collection.mutable.ArrayBuffer +import scala.util.Random + import org.apache.spark.sql.{DataFrame, SQLContext} import org.apache.spark.util.Utils import org.apache.carbondata.common.logging.LogServiceFactory import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.util.CarbonProperties /** @@ -33,6 +37,8 @@ import org.apache.carbondata.core.util.CarbonProperties trait TestQueryExecutorRegister { def sql(sqlText: String): DataFrame + def stop() + def sqlContext: SQLContext } @@ -44,16 +50,99 @@ object TestQueryExecutor { .getCanonicalPath LOGGER.info(s"project path: $projectPath") val integrationPath = s"$projectPath/integration" - val resourcesPath = s"$integrationPath/spark-common-test/src/test/resources" - val storeLocation = s"$integrationPath/spark-common/target/store" - val warehouse = s"$integrationPath/spark-common/target/warehouse" val metastoredb = s"$integrationPath/spark-common/target" - val timestampFormat = "dd-MM-yyyy" + val masterUrl = { + val property = System.getProperty("spark.master.url") + if (property == null) { + "local[2]" + } else { + property + } + } + + val hdfsUrl = { + val property = System.getProperty("hdfs.url") + if (property == null) { + "local" + } else { + LOGGER.info("HDFS PATH given : " + property) + property + } + } + + val resourcesPath = if (hdfsUrl.startsWith("hdfs://")) { + ResourceRegisterAndCopier. + copyResourcesifNotExists(hdfsUrl, s"$integrationPath/spark-common-test/src/test/resources", + s"$integrationPath//spark-common-cluster-test/src/test/resources/testdatafileslist.txt") + hdfsUrl + } else { + s"$integrationPath/spark-common-test/src/test/resources" + } + + val storeLocation = if (hdfsUrl.startsWith("hdfs://")) { + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE, + CarbonCommonConstants.CARBON_LOCK_TYPE_HDFS) + val carbonFile = FileFactory. + getCarbonFile(s"$hdfsUrl/store", FileFactory.getFileType(s"$hdfsUrl/store")) + FileFactory.deleteAllCarbonFilesOfDir(carbonFile) + s"$hdfsUrl/store_" + System.nanoTime() + } else { + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.LOCK_TYPE, + CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL) + s"$integrationPath/spark-common/target/store" + } + val warehouse = if (hdfsUrl.startsWith("hdfs://")) { + val carbonFile = FileFactory. + getCarbonFile(s"$hdfsUrl/warehouse", FileFactory.getFileType(s"$hdfsUrl/warehouse")) + FileFactory.deleteAllCarbonFilesOfDir(carbonFile) + s"$hdfsUrl/warehouse_" + System.nanoTime() + } else { + s"$integrationPath/spark-common/target/warehouse" + } + + val hiveresultpath = if (hdfsUrl.startsWith("hdfs://")) { + val p = s"$hdfsUrl/hiveresultpath" + FileFactory.mkdirs(p, FileFactory.getFileType(p)) + p + } else { + val p = s"$integrationPath/spark-common/target/hiveresultpath" + new File(p).mkdirs() + p + } + + LOGGER.info(s"""Store path taken $storeLocation""") + LOGGER.info(s"""Warehouse path taken $warehouse""") + LOGGER.info(s"""Resource path taken $resourcesPath""") + + lazy val modules = Seq(TestQueryExecutor.projectPath + "/common/target", + TestQueryExecutor.projectPath + "/core/target", + TestQueryExecutor.projectPath + "/hadoop/target", + TestQueryExecutor.projectPath + "/processing/target", + TestQueryExecutor.projectPath + "/integration/spark-common/target", + TestQueryExecutor.projectPath + "/integration/spark2/target", + TestQueryExecutor.projectPath + "/integration/spark-common/target/jars") + lazy val jars = { + val jarsLocal = new ArrayBuffer[String]() + modules.foreach { path => + val files = new File(path).listFiles(new FilenameFilter { + override def accept(dir: File, name: String) = { + name.endsWith(".jar") + } + }) + files.foreach(jarsLocal += _.getAbsolutePath) + } + jarsLocal + } val INSTANCE = lookupQueryExecutor.newInstance().asInstanceOf[TestQueryExecutorRegister] CarbonProperties.getInstance() .addProperty(CarbonCommonConstants.CARBON_BAD_RECORDS_ACTION, "FORCE") .addProperty(CarbonCommonConstants.CARBON_BADRECORDS_LOC, "/tmp/carbon/badrecords") + .addProperty(CarbonCommonConstants.DICTIONARY_SERVER_PORT, + (CarbonCommonConstants.DICTIONARY_SERVER_PORT_DEFAULT.toInt + Random.nextInt(100)) + "") + .addProperty(CarbonCommonConstants.CARBON_MAX_DRIVER_LRU_CACHE_SIZE, "1024") + .addProperty(CarbonCommonConstants.CARBON_MAX_EXECUTOR_LRU_CACHE_SIZE, "1024") + private def lookupQueryExecutor: Class[_] = { ServiceLoader.load(classOf[TestQueryExecutorRegister], Utils.getContextOrSparkClassLoader) .iterator().next().getClass http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark/pom.xml ---------------------------------------------------------------------- diff --git a/integration/spark/pom.xml b/integration/spark/pom.xml index 04b5aa7..fc13c7a 100644 --- a/integration/spark/pom.xml +++ b/integration/spark/pom.xml @@ -188,5 +188,13 @@ </plugin> </plugins> </build> + <profiles> + <profile> + <id>sdvtest</id> + <properties> + <maven.test.skip>true</maven.test.skip> + </properties> + </profile> + </profiles> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala b/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala index 27df623..bb00126 100644 --- a/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala +++ b/integration/spark/src/main/scala/org/apache/spark/sql/test/SparkTestQueryExecutor.scala @@ -32,13 +32,14 @@ class SparkTestQueryExecutor extends TestQueryExecutorRegister { override def sql(sqlText: String): DataFrame = SparkTestQueryExecutor.cc.sql(sqlText) override def sqlContext: SQLContext = SparkTestQueryExecutor.cc + + override def stop(): Unit = SparkTestQueryExecutor.cc.sparkContext.stop() } object SparkTestQueryExecutor { private val LOGGER = LogServiceFactory.getLogService(this.getClass.getCanonicalName) LOGGER.info("use TestQueryExecutorImplV1") CarbonProperties.getInstance() - .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, TestQueryExecutor.timestampFormat) .addProperty(CarbonCommonConstants.STORE_LOCATION_TEMP_PATH, System.getProperty("java.io.tmpdir")) .addProperty(CarbonCommonConstants.LOCK_TYPE, CarbonCommonConstants.CARBON_LOCK_TYPE_LOCAL) http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark2/pom.xml ---------------------------------------------------------------------- diff --git a/integration/spark2/pom.xml b/integration/spark2/pom.xml index 8279c5d..2f34c34 100644 --- a/integration/spark2/pom.xml +++ b/integration/spark2/pom.xml @@ -143,6 +143,7 @@ <version>1.0</version> <!-- Note config is repeated in surefire config --> <configuration> + <skipTests>false</skipTests> <reportsDirectory>${project.build.directory}/surefire-reports</reportsDirectory> <junitxml>.</junitxml> <filereports>CarbonTestSuite.txt</filereports> @@ -166,5 +167,13 @@ </plugin> </plugins> </build> + <profiles> + <profile> + <id>sdvtest</id> + <properties> + <maven.test.skip>true</maven.test.skip> + </properties> + </profile> + </profiles> </project> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/carbondata/blob/14624953/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala index c9eaf6d..a32146a 100644 --- a/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala +++ b/integration/spark2/src/main/scala/org/apache/spark/sql/hive/CarbonFileMetastore.scala @@ -33,6 +33,7 @@ import org.apache.carbondata.core.cache.dictionary.ManageDictionaryAndBTree import org.apache.carbondata.core.constants.CarbonCommonConstants import org.apache.carbondata.core.datastore.impl.FileFactory import org.apache.carbondata.core.fileoperations.FileWriteOperation +import org.apache.carbondata.core.indexstore.DataMapStoreManager import org.apache.carbondata.core.metadata.{AbsoluteTableIdentifier, CarbonMetadata, CarbonTableIdentifier} import org.apache.carbondata.core.metadata.converter.ThriftWrapperSchemaConverterImpl import org.apache.carbondata.core.metadata.schema @@ -425,6 +426,7 @@ class CarbonFileMetastore(conf: RuntimeConfig) extends CarbonMetaStore { CarbonHiveMetadataUtil.invalidateAndDropTable(dbName, tableName, sparkSession) // discard cached table info in cachedDataSourceTables sparkSession.sessionState.catalog.refreshTable(tableIdentifier) + DataMapStoreManager.getInstance().clearDataMap(identifier, "blocklet") } }
