Repository: parquet-mr Updated Branches: refs/heads/master 6c9ca4d4c -> 944291b74
PARQUET-431: Make ParquetOutputFormat.memoryManager volatile Currently ParquetOutputFormat.getRecordWriter() contains an unsynchronized lazy initialization of the non-volatile static field *memoryManager*. Because the compiler or processor may reorder instructions, threads are not guaranteed to see a completely initialized object, when ParquetOutputFormat.getRecordWriter() is called by multiple threads. This PR makes *memoryManager* volatile to correct the problem. Author: Liwei Lin <[email protected]> Author: proflin <[email protected]> Closes #313 from proflin/PARQUET-431 and squashes the following commits: 1aa4a44 [Liwei Lin] empty commit to trigger CI 5e94fa3 [Liwei Lin] Remove the volatile modifier for memoryManager d54bb99 [Liwei Lin] Undo the Deprecated anotation fd1df4e [Liwei Lin] Adds synchronization around the creation of memoryManager as well as getMemoryManager() 615aa5a [proflin] PARQUET-431 Project: http://git-wip-us.apache.org/repos/asf/parquet-mr/repo Commit: http://git-wip-us.apache.org/repos/asf/parquet-mr/commit/944291b7 Tree: http://git-wip-us.apache.org/repos/asf/parquet-mr/tree/944291b7 Diff: http://git-wip-us.apache.org/repos/asf/parquet-mr/diff/944291b7 Branch: refs/heads/master Commit: 944291b748bcfec4e2f3c17623884db7a17b9f21 Parents: 6c9ca4d Author: Liwei Lin <[email protected]> Authored: Mon Feb 15 16:37:04 2016 -0800 Committer: Julien Le Dem <[email protected]> Committed: Mon Feb 15 16:37:04 2016 -0800 ---------------------------------------------------------------------- .../org/apache/parquet/hadoop/ParquetOutputFormat.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/parquet-mr/blob/944291b7/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java ---------------------------------------------------------------------- diff --git a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java index 31cc96b..6accce1 100644 --- a/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java +++ b/parquet-hadoop/src/main/java/org/apache/parquet/hadoop/ParquetOutputFormat.java @@ -394,9 +394,12 @@ public class ParquetOutputFormat<T> extends FileOutputFormat<Void, T> { MemoryManager.DEFAULT_MEMORY_POOL_RATIO); long minAllocation = conf.getLong(ParquetOutputFormat.MIN_MEMORY_ALLOCATION, MemoryManager.DEFAULT_MIN_MEMORY_ALLOCATION); - if (memoryManager == null) { - memoryManager = new MemoryManager(maxLoad, minAllocation); - } else if (memoryManager.getMemoryPoolRatio() != maxLoad) { + synchronized (ParquetOutputFormat.class) { + if (memoryManager == null) { + memoryManager = new MemoryManager(maxLoad, minAllocation); + } + } + if (memoryManager.getMemoryPoolRatio() != maxLoad) { LOG.warn("The configuration " + MEMORY_POOL_RATIO + " has been set. It should not " + "be reset by the new value: " + maxLoad); } @@ -441,13 +444,12 @@ public class ParquetOutputFormat<T> extends FileOutputFormat<Void, T> { return committer; } - /** * This memory manager is for all the real writers (InternalParquetRecordWriter) in one task. */ private static MemoryManager memoryManager; - public static MemoryManager getMemoryManager() { + public synchronized static MemoryManager getMemoryManager() { return memoryManager; } }
