Repository: carbondata Updated Branches: refs/heads/master 729286919 -> a7926ea13
[CARBONDATA-2436] Block pruning problem post the carbon schema restructure. Currently DataMap is pruning with segmentproperties from the 0th block of BlockletDataMap is not correct. As post restructure if the table is updated then all the block will not have symmetric schema within the same segments. Solution : It must be ensured the block could be pruned with the same schema. This closes #2263 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/a7926ea1 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/a7926ea1 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/a7926ea1 Branch: refs/heads/master Commit: a7926ea133170bd027756969410e043eb9a4b7a3 Parents: 7292869 Author: mohammadshahidkhan <mohdshahidkhan1...@gmail.com> Authored: Fri May 4 16:36:57 2018 +0530 Committer: manishgupta88 <tomanishgupt...@gmail.com> Committed: Mon May 7 11:32:18 2018 +0530 ---------------------------------------------------------------------- .../blockletindex/BlockletDataMap.java | 6 ++++- .../AlterTableValidationTestCase.scala | 23 ++++++++++++++++++-- 2 files changed, 26 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/a7926ea1/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java index 906291a..f72dc06 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/blockletindex/BlockletDataMap.java @@ -685,7 +685,11 @@ public class BlockletDataMap extends CoarseGrainDataMap implements Cacheable { } } // Prune with filters if the partitions are existed in this datamap - return prune(filterExp, segmentProperties); + // changed segmentProperties to this.segmentProperties to make sure the pruning with its own + // segmentProperties. + // Its a temporary fix. The Interface DataMap.prune(FilterResolverIntf filterExp, + // SegmentProperties segmentProperties, List<PartitionSpec> partitions) should be corrected + return prune(filterExp, this.segmentProperties); } private boolean isCorrectUUID(String[] fileDetails, PartitionSpec spec) { http://git-wip-us.apache.org/repos/asf/carbondata/blob/a7926ea1/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala index a86bf17..648ed11 100644 --- a/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala +++ b/integration/spark2/src/test/scala/org/apache/spark/carbondata/restructure/AlterTableValidationTestCase.scala @@ -43,7 +43,7 @@ class AlterTableValidationTestCase extends Spark2QueryTest with BeforeAndAfterAl sql("drop table if exists testalterwithboolean") sql("drop table if exists testalterwithbooleanwithoutdefaultvalue") sql("drop table if exists test") - + sql("drop table if exists retructure_iud") // clean data folder CarbonProperties.getInstance() @@ -690,7 +690,25 @@ test("test alter command for boolean data type with correct default measure valu testFilterWithDefaultValue(true) testFilterWithDefaultValue(false) } - + test("Filter query on Restructure and updated table") { + sql( + """ + CREATE TABLE retructure_iud(id int, name string, city string, age int) + STORED BY 'org.apache.carbondata.format' + """) + val testData = s"$resourcesPath/sample.csv" + sql(s"LOAD DATA LOCAL INPATH '$testData' into table retructure_iud") + sql("ALTER TABLE retructure_iud ADD COLUMNS (newField STRING) " + + "TBLPROPERTIES ('DEFAULT.VALUE.newField'='def', 'DICTIONARY_INCLUDE'='newField')").show() + sql("ALTER TABLE retructure_iud ADD COLUMNS (newField1 STRING) " + + "TBLPROPERTIES ('DEFAULT.VALUE.newField1'='def', 'DICTIONARY_EXCLUDE'='newField1')").show() + // update operation + sql("""update retructure_iud d set (d.id) = (d.id + 1) where d.id > 2""").show() + checkAnswer( + sql("select count(*) from retructure_iud where id = 2 and newfield1='def'"), + Seq(Row(1)) + ) + } override def afterAll { sql("DROP TABLE IF EXISTS restructure") sql("drop table if exists table1") @@ -707,5 +725,6 @@ test("test alter command for boolean data type with correct default measure valu sql("drop table if exists testalterwithboolean") sql("drop table if exists testalterwithbooleanwithoutdefaultvalue") sql("drop table if exists test") + sql("drop table if exists retructure_iud") } }