[CARBONDATA-2571] Calculating the carbonindex and carbondata file size of a table is wrong
Problem: While calculating the carbonindex files size, we are checking either index file or merge file. But in PR#2333, implementation is changed to fill both the file name and the merge file name. So, we have to consider both fields. Solution: While calculating the carbonindex files size, we have to consider both the files and mergeFileName fields. We should get the list of index files from these 2 fields and then calculate the size of the files. This closes #2358 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/27d70599 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/27d70599 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/27d70599 Branch: refs/heads/carbonstore Commit: 27d7059984962b97bcaf576fed496653932ea743 Parents: 92d9b92 Author: dhatchayani <dhatcha.offic...@gmail.com> Authored: Fri Jun 1 15:13:38 2018 +0530 Committer: manishgupta88 <tomanishgupt...@gmail.com> Committed: Tue Jun 5 11:51:52 2018 +0530 ---------------------------------------------------------------------- .../apache/carbondata/core/util/CarbonUtil.java | 37 +++++++++++--------- 1 file changed, 20 insertions(+), 17 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/27d70599/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java ---------------------------------------------------------------------- diff --git a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java index 1526047..5a7bce3 100644 --- a/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java +++ b/core/src/main/java/org/apache/carbondata/core/util/CarbonUtil.java @@ -2688,27 +2688,30 @@ public final class CarbonUtil { throws IOException { long carbonDataSize = 0L; long carbonIndexSize = 0L; - List<String> listOfFilesRead = new ArrayList<>(); HashMap<String, Long> dataAndIndexSize = new HashMap<String, Long>(); - if (fileStore.getLocationMap() != null) { + Map<String, SegmentFileStore.FolderDetails> locationMap = fileStore.getLocationMap(); + if (locationMap != null) { fileStore.readIndexFiles(); - Map<String, String> indexFiles = fileStore.getIndexFiles(); Map<String, List<String>> indexFilesMap = fileStore.getIndexFilesMap(); - for (Map.Entry<String, List<String>> entry : indexFilesMap.entrySet()) { - // get the size of carbonindex file - String indexFile = entry.getKey(); - String mergeIndexFile = indexFiles.get(indexFile); - if (null != mergeIndexFile) { - String mergeIndexPath = indexFile - .substring(0, indexFile.lastIndexOf(CarbonCommonConstants.FILE_SEPARATOR) + 1) - + mergeIndexFile; - if (!listOfFilesRead.contains(mergeIndexPath)) { - carbonIndexSize += FileFactory.getCarbonFile(mergeIndexPath).getSize(); - listOfFilesRead.add(mergeIndexPath); - } - } else { - carbonIndexSize += FileFactory.getCarbonFile(indexFile).getSize(); + // get the size of carbonindex file + for (Map.Entry<String, SegmentFileStore.FolderDetails> entry : locationMap.entrySet()) { + SegmentFileStore.FolderDetails folderDetails = entry.getValue(); + Set<String> carbonindexFiles = folderDetails.getFiles(); + String mergeFileName = folderDetails.getMergeFileName(); + if (null != mergeFileName) { + String mergeIndexPath = + fileStore.getTablePath() + entry.getKey() + CarbonCommonConstants.FILE_SEPARATOR + + mergeFileName; + carbonIndexSize += FileFactory.getCarbonFile(mergeIndexPath).getSize(); } + for (String indexFile : carbonindexFiles) { + String indexPath = + fileStore.getTablePath() + entry.getKey() + CarbonCommonConstants.FILE_SEPARATOR + + indexFile; + carbonIndexSize += FileFactory.getCarbonFile(indexPath).getSize(); + } + } + for (Map.Entry<String, List<String>> entry : indexFilesMap.entrySet()) { // get the size of carbondata files for (String blockFile : entry.getValue()) { carbonDataSize += FileFactory.getCarbonFile(blockFile).getSize();