Repository: carbondata Updated Branches: refs/heads/master 45960f4a8 -> 7ab670652
[CARBONDATA-2753] Fix Compatibility issues on index Files with 1.3 store Problem: Currently,in the segmentFile we are writing the index files list in files field, only if it exists, otherwise it will be empty(in case if it is merged to merge index file). But in the old store, we were writing both the files and mergeFileName fields even if the index files are merged. Solution: While querying we have to check the physical existence of the index files listed in the files field. If it physically exists, then we have to consider that. This closes #2534 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/7ab67065 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/7ab67065 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/7ab67065 Branch: refs/heads/master Commit: 7ab6706523d2da008585b41f8587762f94c1bdd4 Parents: 45960f4 Author: dhatchayani <[email protected]> Authored: Fri Jul 20 20:06:00 2018 +0530 Committer: manishgupta88 <[email protected]> Committed: Mon Jul 23 18:27:51 2018 +0530 ---------------------------------------------------------------------- .../carbondata/core/metadata/SegmentFileStore.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/7ab67065/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java index 9681e37..28ac47e 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java @@ -580,7 +580,7 @@ public class SegmentFileStore { * Gets all index files from this segment * @return */ - public Map<String, String> getIndexOrMergeFiles() { + public Map<String, String> getIndexOrMergeFiles() throws IOException { Map<String, String> indexFiles = new HashMap<>(); if (segmentFile != null) { for (Map.Entry<String, FolderDetails> entry : getLocationMap().entrySet()) { @@ -597,7 +597,14 @@ public class SegmentFileStore { Set<String> files = entry.getValue().getFiles(); if (null != files && !files.isEmpty()) { for (String indexFile : files) { - indexFiles.put(location + CarbonCommonConstants.FILE_SEPARATOR + indexFile, null); + String indexFilePath = location + CarbonCommonConstants.FILE_SEPARATOR + indexFile; + // In the 1.3 store, files field contain the carbonindex files names + // even if they are merged to a carbonindexmerge file. In that case we have to check + // for the physical existence of the file to decide + // on whether it is already merged or not. + if (FileFactory.isFileExist(indexFilePath)) { + indexFiles.put(indexFilePath, null); + } } } }
