Repository: carbondata Updated Branches: refs/heads/master c2c5b18ba -> c3bc1ba10
[CARBONDATA-2658][DataLoad] Fix bugs in spilling in-memory pages the parameter carbon.load.sortMemory.spill.percentage configured the value range 0-100,according to configuration merge and spill in-memory pages to disk This closes #2414 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/c3bc1ba1 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/c3bc1ba1 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/c3bc1ba1 Branch: refs/heads/master Commit: c3bc1ba10ef4c468f199fdea6e38d8b197908fcd Parents: c2c5b18 Author: ndwangsen <[email protected]> Authored: Fri Jun 22 12:02:36 2018 +0800 Committer: Jacky Li <[email protected]> Committed: Mon Jul 9 15:18:12 2018 +0800 ---------------------------------------------------------------------- .../processing/loading/sort/unsafe/UnsafeSortDataRows.java | 1 - .../sort/unsafe/merger/UnsafeIntermediateMerger.java | 9 +++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3bc1ba1/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java index fb0bcc3..76532f0 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/UnsafeSortDataRows.java @@ -149,7 +149,6 @@ public class UnsafeSortDataRows { if (isMemoryAvailable) { UnsafeSortMemoryManager.INSTANCE.allocateDummyMemory(baseBlock.size()); } else { - LOGGER.info("trigger in-memory merge and spill for table " + parameters.getTableName()); // merge and spill in-memory pages to disk if memory is not enough unsafeInMemoryIntermediateFileMerger.tryTriggerInmemoryMerging(true); } http://git-wip-us.apache.org/repos/asf/carbondata/blob/c3bc1ba1/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java index bd2f5d9..e6e24ec 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java +++ b/processing/src/main/java/org/apache/carbondata/processing/loading/sort/unsafe/merger/UnsafeIntermediateMerger.java @@ -84,6 +84,14 @@ public class UnsafeIntermediateMerger { Integer spillPercentage = CarbonProperties.getInstance().getSortMemorySpillPercentage(); this.spillSizeInSortMemory = UnsafeSortMemoryManager.INSTANCE.getUsableMemory() * spillPercentage / 100; + // get memory chunk size + long inMemoryChunkSizeInMB = CarbonProperties.getInstance().getSortMemoryChunkSizeInMB(); + if (spillSizeInSortMemory < inMemoryChunkSizeInMB * 1024 * 1024) { + LOGGER.warn("the configure spill size is " + spillSizeInSortMemory + + " less than the page size " + inMemoryChunkSizeInMB * 1024 * 1024 + + ",so no merge and spill in-memory pages to disk"); + } + } public void addDataChunkToMerge(UnsafeCarbonRowPage rowPage) { @@ -147,6 +155,7 @@ public class UnsafeIntermediateMerger { UnsafeCarbonRowPage page = iter.next(); if (!spillDisk || sizeAdded + page.getDataBlock().size() < this.spillSizeInSortMemory) { pages2Merge.add(page); + sizeAdded += page.getDataBlock().size(); totalRows2Merge += page.getBuffer().getActualSize(); iter.remove(); } else {
