Repository: carbondata Updated Branches: refs/heads/master 44ffaf57e -> ee11bb1eb
[CARBONDATA-2070]fix create preaggregate on decimal column in hive metastore Problem: when hive metastore is enabled and aggregate table is tried to create on the decimal column of main table, cast exception is thrown for Decimal datatype solution:During creation of TableInfo from hivemetastore the DataMapSchemas and the columns dataTypes are not converted to the appropriate child classes. convert to actual datatype This closes #1852 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/ee11bb1e Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/ee11bb1e Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/ee11bb1e Branch: refs/heads/master Commit: ee11bb1eb7790687b685b4af35ec9f1ab7fa6eff Parents: 44ffaf5 Author: akashrn5 <[email protected]> Authored: Tue Jan 23 18:59:51 2018 +0530 Committer: ravipesala <[email protected]> Committed: Wed Jan 24 23:51:30 2018 +0530 ---------------------------------------------------------------------- .../core/metadata/schema/table/CarbonTable.java | 11 +++++++++++ .../spark/testsuite/datamap/TestDataMapCommand.scala | 12 ++++++++++++ 2 files changed, 23 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee11bb1e/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java index dd65b84..4bb0d20 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/schema/table/CarbonTable.java @@ -156,6 +156,17 @@ public class CarbonTable implements Serializable { columnSchema.setDataType(DataTypeUtil.valueOf(columnSchema.getDataType(), columnSchema.getPrecision(), columnSchema.getScale())); } + List<DataMapSchema> childSchema = tableInfo.getDataMapSchemaList(); + for (DataMapSchema dataMapSchema : childSchema) { + if (dataMapSchema.childSchema != null + && dataMapSchema.childSchema.getListOfColumns().size() > 0) { + for (ColumnSchema columnSchema : dataMapSchema.childSchema.getListOfColumns()) { + columnSchema.setDataType(DataTypeUtil + .valueOf(columnSchema.getDataType(), columnSchema.getPrecision(), + columnSchema.getScale())); + } + } + } if (tableInfo.getFactTable().getBucketingInfo() != null) { for (ColumnSchema columnSchema : tableInfo.getFactTable() .getBucketingInfo().getListOfColumns()) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/ee11bb1e/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala index f3458e2..a0ea317 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datamap/TestDataMapCommand.scala @@ -35,6 +35,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll { override def beforeAll { sql("drop table if exists datamaptest") sql("drop table if exists datamapshowtest") + sql("drop table if exists uniqdata") sql("create table datamaptest (a string, b string, c string) stored by 'carbondata'") } @@ -210,6 +211,16 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll { Seq(Row(1, 31), Row(2, 27), Row(3, 70), Row(4, 55))) } + test("test preaggregate load for decimal column for hivemetastore") { + CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, "true") + sql("drop datamap if exists uniqdata_agg on table uniqdata") + sql("CREATE TABLE uniqdata(CUST_ID int,CUST_NAME String,ACTIVE_EMUI_VERSION string,DOB timestamp,DOJ timestamp, BIGINT_COLUMN1 bigint,BIGINT_COLUMN2 bigint,DECIMAL_COLUMN1 decimal(30,10),DECIMAL_COLUMN2 decimal(36,10),Double_COLUMN1 double, Double_COLUMN2 double,INTEGER_COLUMN1 int) STORED BY 'org.apache.carbondata.format'") + sql("insert into uniqdata select 9000,'CUST_NAME_00000','ACTIVE_EMUI_VERSION_00000','1970-01-01 01:00:03','1970-01-01 02:00:03',123372036854,-223372036854,12345678901.1234000000,22345678901.1234000000,11234567489.7976000000,-11234567489.7976000000,1") + sql("create datamap uniqdata_agg on table uniqdata using 'preaggregate' as select min(DECIMAL_COLUMN1) from uniqdata group by DECIMAL_COLUMN1") + checkAnswer(sql("select * from uniqdata_uniqdata_agg"),Seq(Row(12345678901.1234000000, 12345678901.1234000000))) + sql("drop datamap if exists uniqdata_agg on table uniqdata") + } + test("create pre-agg table with path") { sql("drop table if exists main_preagg") sql("drop table if exists main ") @@ -237,6 +248,7 @@ class TestDataMapCommand extends QueryTest with BeforeAndAfterAll { override def afterAll { sql("DROP TABLE IF EXISTS maintable") + sql("drop table if exists uniqdata") CarbonProperties.getInstance().addProperty(CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE, CarbonCommonConstants.ENABLE_HIVE_SCHEMA_META_STORE_DEFAULT) sql("drop table if exists datamaptest")
