[CARBONDATA-2813] Fixed code to get data size from LoadDetails if size is written there
Problem: In 1.3.x when index files are merged to form mergeindex file a mapping of which index files if merged to which mergeindex is kept in the segments file. In 1.4.x both the index and merge index files are scanned to calculate the size of segments for major compaction. As the index file was deleted in the 1.3.x store therefore in 1.4.x it was throwing "Unable to get File status exception". Solution: Try to the size of the segments from LoadMetadataDetails. If not present then try to read the size from index files. This closes #2600 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/38384cb9 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/38384cb9 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/38384cb9 Branch: refs/heads/external-format Commit: 38384cb9f309cc7eb83e61e85c48dd8583921004 Parents: f2e898a Author: kunal642 <kunalkapoor...@gmail.com> Authored: Thu Aug 2 11:44:20 2018 +0530 Committer: manishgupta88 <tomanishgupt...@gmail.com> Committed: Thu Aug 2 18:14:56 2018 +0530 ---------------------------------------------------------------------- .../processing/merger/CarbonDataMergerUtil.java | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/38384cb9/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java index 1162fc2..e3da86d 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java +++ b/processing/src/main/java/org/apache/carbondata/processing/merger/CarbonDataMergerUtil.java @@ -49,6 +49,8 @@ import org.apache.carbondata.core.writer.CarbonDeleteDeltaWriterImpl; import org.apache.carbondata.processing.loading.model.CarbonLoadModel; import org.apache.carbondata.processing.util.CarbonLoaderUtil; +import org.apache.commons.lang.StringUtils; + /** * utility class for load merging. */ @@ -649,8 +651,14 @@ public final class CarbonDataMergerUtil { // variable to store one segment size across partition. long sizeOfOneSegmentAcrossPartition; if (segment.getSegmentFile() != null) { - sizeOfOneSegmentAcrossPartition = CarbonUtil.getSizeOfSegment( - carbonTable.getTablePath(), new Segment(segId, segment.getSegmentFile())); + // If LoadMetaDataDetail already has data size no need to calculate the data size from + // index files. If not there then read the index file and calculate size. + if (!StringUtils.isEmpty(segment.getDataSize())) { + sizeOfOneSegmentAcrossPartition = Long.parseLong(segment.getDataSize()); + } else { + sizeOfOneSegmentAcrossPartition = CarbonUtil.getSizeOfSegment(carbonTable.getTablePath(), + new Segment(segId, segment.getSegmentFile())); + } } else { sizeOfOneSegmentAcrossPartition = getSizeOfSegment(carbonTable.getTablePath(), segId); }
