Repository: carbondata Updated Branches: refs/heads/master 7e0803fec -> 5daae9515
[CARBONDATA-2289] If carbon merge index is enabled then after IUD operation if some blocks of a segment is deleted, then during query and IUD operation the driver is throwing FileNotFoundException while preparing BlockMetaInfo. Solution: Skiping the invalid(deleted) blocks and logig the same as warning. This closes #2110 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5daae951 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5daae951 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5daae951 Branch: refs/heads/master Commit: 5daae951500967a910b863e4e57742b51f63f591 Parents: 7e0803f Author: mohammadshahidkhan <[email protected]> Authored: Wed Mar 28 16:38:21 2018 +0530 Committer: ravipesala <[email protected]> Committed: Fri Mar 30 11:34:27 2018 +0530 ---------------------------------------------------------------------- .../core/indexstore/BlockletDataMapIndexStore.java | 8 +++++++- .../datacompaction/CarbonIndexFileMergeTestCase.scala | 11 +++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/5daae951/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java index befa121..167a04e 100644 --- a/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java +++ b/core/src/main/java/org/apache/carbondata/core/indexstore/BlockletDataMapIndexStore.java @@ -118,7 +118,13 @@ public class BlockletDataMapIndexStore .getIndexFileName(), indexFileStore.getFileData(identifier.getIndexFileName())); for (DataFileFooter footer : indexInfo) { String blockPath = footer.getBlockInfo().getTableBlockInfo().getFilePath(); - blockMetaInfoMap.put(blockPath, createBlockMetaInfo(blockPath)); + if (FileFactory.isFileExist(blockPath)) { + blockMetaInfoMap.put(blockPath, createBlockMetaInfo(blockPath)); + } else { + LOGGER.warn("Skipping invalid block " + footer.getBlockInfo().getBlockUniqueName() + + " The block does not exist. The block might be got deleted due to clean up post" + + " update/delete operation over table."); + } } return blockMetaInfoMap; } http://git-wip-us.apache.org/repos/asf/carbondata/blob/5daae951/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala ---------------------------------------------------------------------- diff --git a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala index c7912cf..9cb30b9 100644 --- a/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala +++ b/integration/spark-common-test/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/CarbonIndexFileMergeTestCase.scala @@ -178,6 +178,17 @@ class CarbonIndexFileMergeTestCase checkAnswer(sql("""Select count(*) from nonindexmerge"""), rows) } + test("Query should not fail after iud operation on a table having merge indexes") { + sql("drop table if exists mitable") + sql("create table mitable(id int, issue date) stored by 'carbondata'") + sql("insert into table mitable select '1','2000-02-01'") + val table = CarbonMetadata.getInstance().getCarbonTable("default", "mitable") + new CarbonIndexFileMergeWriter() + .mergeCarbonIndexFilesOfSegment("0", table.getTablePath, false) + sql("update mitable set(id)=(2) where issue = '2000-02-01'").show() + sql("clean files for table mitable") + sql("select * from mitable").show() + } private def getIndexFileCount(tableName: String, segment: String): Int = { val table = CarbonMetadata.getInstance().getCarbonTable(tableName) val path = CarbonTablePath
