[CARBONDATA-1947]fix select * issue after compaction, delete and clean files operation
When updating the metadata details for merged segment, get the load starttime from carbonloadModel facttime stamp This closes #1735 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/83e92715 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/83e92715 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/83e92715 Branch: refs/heads/branch-1.3 Commit: 83e92715a6819d2ff5c3e5cd50fe073be5ee01a6 Parents: 1937a21 Author: akashrn5 <[email protected]> Authored: Thu Dec 28 13:43:15 2017 +0530 Committer: Jacky Li <[email protected]> Committed: Fri Dec 29 14:41:30 2017 +0800 ---------------------------------------------------------------------- .../testsuite/iud/DeleteCarbonTableTestCase.scala | 15 +++++++++++++++ .../carbondata/spark/rdd/CarbonTableCompactor.scala | 3 +-- .../processing/merger/CarbonDataMergerUtil.java | 4 ++-- 3 files changed, 18 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala index deaad20..6521657 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/iud/DeleteCarbonTableTestCase.scala @@ -165,6 +165,21 @@ class DeleteCarbonTableTestCase extends QueryTest with BeforeAndAfterAll { sql("drop table if exists preaggmain_preagg1") } + test("test select query after compaction, delete and clean files") { + sql("drop table if exists select_after_clean") + sql("create table select_after_clean(id int, name string) stored by 'carbondata'") + sql("insert into select_after_clean select 1,'abc'") + sql("insert into select_after_clean select 2,'def'") + sql("insert into select_after_clean select 3,'uhj'") + sql("insert into select_after_clean select 4,'frg'") + sql("alter table select_after_clean compact 'minor'") + sql("clean files for table select_after_clean") + sql("delete from select_after_clean where name='def'") + sql("clean files for table select_after_clean") + checkAnswer(sql("""select * from select_after_clean"""), + Seq(Row(1, "abc"), Row(3, "uhj"), Row(4, "frg"))) + } + override def afterAll { sql("use default") http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala ---------------------------------------------------------------------- diff --git a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala index 9768625..0dc856d 100644 --- a/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala +++ b/integration/spark2/src/main/scala/org/apache/carbondata/spark/rdd/CarbonTableCompactor.scala @@ -135,7 +135,6 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel, val factTableName = carbonLoadModel.getTableName val validSegments: Array[String] = CarbonDataMergerUtil .getValidSegments(loadsToMerge).split(',') - val mergeLoadStartTime = CarbonUpdateUtil.readCurrentTime() val partitionMapper = if (carbonTable.isHivePartitionTable) { var partitionMap: util.Map[String, util.List[String]] = null validSegments.foreach { segmentId => @@ -245,7 +244,7 @@ class CarbonTableCompactor(carbonLoadModel: CarbonLoadModel, carbonLoadModel)) || CarbonDataMergerUtil .updateLoadMetadataWithMergeStatus(loadsToMerge, carbonTable.getMetaDataFilepath, - mergedLoadNumber, carbonLoadModel, mergeLoadStartTime, compactionType) + mergedLoadNumber, carbonLoadModel, compactionType) if (!statusFileUpdation) { LOGGER.audit(s"Compaction request failed for table ${ carbonLoadModel.getDatabaseName }." + http://git-wip-us.apache.org/repos/asf/carbondata/blob/83e92715/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java index e245927..26648d0 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java @@ -280,7 +280,7 @@ public final class CarbonDataMergerUtil { */ public static boolean updateLoadMetadataWithMergeStatus(List<LoadMetadataDetails> loadsToMerge, String metaDataFilepath, String mergedLoadNumber, CarbonLoadModel carbonLoadModel, - long mergeLoadStartTime, CompactionType compactionType) throws IOException { + CompactionType compactionType) throws IOException { boolean tableStatusUpdationStatus = false; AbsoluteTableIdentifier absoluteTableIdentifier = carbonLoadModel.getCarbonDataLoadSchema().getCarbonTable().getAbsoluteTableIdentifier(); @@ -327,7 +327,7 @@ public final class CarbonDataMergerUtil { loadMetadataDetails.setLoadName(mergedLoadNumber); CarbonLoaderUtil .addDataIndexSizeIntoMetaEntry(loadMetadataDetails, mergedLoadNumber, carbonTable); - loadMetadataDetails.setLoadStartTime(mergeLoadStartTime); + loadMetadataDetails.setLoadStartTime(carbonLoadModel.getFactTimeStamp()); loadMetadataDetails.setPartitionCount("0"); // if this is a major compaction then set the segment as major compaction. if (CompactionType.MAJOR == compactionType) {
