Repository: carbondata Updated Branches: refs/heads/branch-1.3 aff822ccf -> 87de69729
[CARBONDATA-2265] [DFX]-Load]: Load job fails if 1 folder contains 1000 files Problem : We are keeping the rawTablePages also in memory, but we are no where using it Solution : Removed storing rawTablePages This closes #2082 Project: http://git-wip-us.apache.org/repos/asf/carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/carbondata/commit/87de6972 Tree: http://git-wip-us.apache.org/repos/asf/carbondata/tree/87de6972 Diff: http://git-wip-us.apache.org/repos/asf/carbondata/diff/87de6972 Branch: refs/heads/branch-1.3 Commit: 87de69729e916e1ca5ad1be00a091ccbe65a990d Parents: aff822c Author: dhatchayani <[email protected]> Authored: Tue Mar 20 18:11:58 2018 +0530 Committer: kunal642 <[email protected]> Committed: Wed Mar 21 15:45:04 2018 +0530 ---------------------------------------------------------------------- .../processing/store/writer/v3/BlockletDataHolder.java | 7 ------- .../store/writer/v3/CarbonFactDataWriterImplV3.java | 5 +++-- 2 files changed, 3 insertions(+), 9 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/carbondata/blob/87de6972/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java index 68aee95..36fda3c 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/BlockletDataHolder.java @@ -24,24 +24,20 @@ import org.apache.carbondata.processing.store.TablePage; public class BlockletDataHolder { private List<EncodedTablePage> encodedTablePage; - private List<TablePage> rawTablePages; private long currentSize; public BlockletDataHolder() { this.encodedTablePage = new ArrayList<>(); - this.rawTablePages = new ArrayList<>(); } public void clear() { encodedTablePage.clear(); - rawTablePages.clear(); currentSize = 0; } public void addPage(TablePage rawTablePage) { EncodedTablePage encodedTablePage = rawTablePage.getEncodedTablePage(); this.encodedTablePage.add(encodedTablePage); - this.rawTablePages.add(rawTablePage); currentSize += encodedTablePage.getEncodedSize(); } @@ -66,7 +62,4 @@ public class BlockletDataHolder { return encodedTablePage; } - public List<TablePage> getRawTablePages() { - return rawTablePages; - } } http://git-wip-us.apache.org/repos/asf/carbondata/blob/87de6972/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java ---------------------------------------------------------------------- diff --git a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java index ddf444d..fc1bd7d 100644 --- a/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java +++ b/processing/src/main/java/org/apache/carbondata/processing/store/writer/v3/CarbonFactDataWriterImplV3.java @@ -187,9 +187,10 @@ public class CarbonFactDataWriterImplV3 extends AbstractFactDataWriter { } catch (IOException e) { LOGGER.error(e, "Problem while writing file"); throw new CarbonDataWriterException("Problem while writing file", e); + } finally { + // clear the data holder + blockletDataHolder.clear(); } - // clear the data holder - blockletDataHolder.clear(); }
