Repository: parquet-mr Updated Branches: refs/heads/master 1dd5cec08 -> 83406b73e
PARQUET-340: MemoryManager: max memory can be truncated Using float will cause the max heap limit to be limited to 2147483647 due to math.round(float) if used with a large heap. This should be a double precision to prevent rounding to an int32 before storing into a long. Author: Chris Bannister <[email protected]> Closes #246 from Zariel/default-mem-truncated and squashes the following commits: bf375f6 [Chris Bannister] MemoryManager: ensure max memory is not truncated Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/83406b73 Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/83406b73 Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/83406b73 Branch: refs/heads/master Commit: 83406b73e70a251eec5daae34f1bd8d554cdddec Parents: 1dd5cec Author: Chris Bannister <[email protected]> Authored: Mon Jul 20 09:59:29 2015 -0700 Committer: julien <[email protected]> Committed: Mon Jul 20 09:59:29 2015 -0700 ---------------------------------------------------------------------- .../src/main/java/org/apache/parquet/hadoop/MemoryManager.java | 2 +- .../src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/83406b73/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java ---------------------------------------------------------------------- diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java index f997d08..0c56bb2 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/MemoryManager.java @@ -57,7 +57,7 @@ public class MemoryManager { memoryPoolRatio = ratio; minMemoryAllocation = minAllocation; - totalMemoryPool = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax + totalMemoryPool = Math.round((double) ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax () * ratio); LOG.debug(String.format("Allocated total memory pool is: %,d", totalMemoryPool)); } http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/83406b73/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java ---------------------------------------------------------------------- diff --git a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java index bbe7443..fb71b86 100644 --- a/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java +++ b/parquet-hadoop/src/test/java/org/apache/parquet/hadoop/TestMemoryManager.java @@ -54,7 +54,7 @@ public class TestMemoryManager { @Before public void setUp() { GroupWriteSupport.setSchema(MessageTypeParser.parseMessageType(writeSchema),conf); - expectPoolSize = Math.round(ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax + expectPoolSize = Math.round((double) ManagementFactory.getMemoryMXBean().getHeapMemoryUsage().getMax () * MemoryManager.DEFAULT_MEMORY_POOL_RATIO); rowGroupSize = (int) Math.floor(expectPoolSize / 2); conf.setInt(ParquetOutputFormat.BLOCK_SIZE, rowGroupSize);
