[CARBONDATA-2753] Fix Compatibility issues on index Files with 1.3 store Problem: Currently,in the segmentFile we are writing the index files list in files field, only if it exists, otherwise it will be empty(in case if it is merged to merge index file). But in the old store, we were writing both the files and mergeFileName fields even if the index files are merged.
Solution: While querying we have to check the physical existence of the index files listed in the files field. If it physically exists, then we have to consider that. This closes #2534 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/5d055a41 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/5d055a41 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/5d055a41 Branch: refs/heads/branch-1.4 Commit: 5d055a417d6b764628bcb8724bf56ce751586116 Parents: 5491446 Author: dhatchayani <[email protected]> Authored: Fri Jul 20 20:06:00 2018 +0530 Committer: ravipesala <[email protected]> Committed: Tue Jul 31 00:10:41 2018 +0530 ---------------------------------------------------------------------- .../carbondata/core/metadata/SegmentFileStore.java | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/5d055a41/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java index 9681e37..28ac47e 100644 --- a/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java +++ b/core/src/main/java/org/apache/carbondata/core/metadata/SegmentFileStore.java @@ -580,7 +580,7 @@ public class SegmentFileStore { * Gets all index files from this segment * @return */ - public Map<String, String> getIndexOrMergeFiles() { + public Map<String, String> getIndexOrMergeFiles() throws IOException { Map<String, String> indexFiles = new HashMap<>(); if (segmentFile != null) { for (Map.Entry<String, FolderDetails> entry : getLocationMap().entrySet()) { @@ -597,7 +597,14 @@ public class SegmentFileStore { Set<String> files = entry.getValue().getFiles(); if (null != files && !files.isEmpty()) { for (String indexFile : files) { - indexFiles.put(location + CarbonCommonConstants.FILE_SEPARATOR + indexFile, null); + String indexFilePath = location + CarbonCommonConstants.FILE_SEPARATOR + indexFile; + // In the 1.3 store, files field contain the carbonindex files names + // even if they are merged to a carbonindexmerge file. In that case we have to check + // for the physical existence of the file to decide + // on whether it is already merged or not. + if (FileFactory.isFileExist(indexFilePath)) { + indexFiles.put(indexFilePath, null); + } } } }
