Repository: spark Updated Branches: refs/heads/master 366cac6fb -> e1f684539
[SPARK-12181] Check Cached unaligned-access capability before using Unsafe ## What changes were proposed in this pull request? For MemoryMode.OFF_HEAP, Unsafe.getInt etc. are used with no restriction. However, the Oracle implementation uses these methods only if the class variable unaligned (commented as "Cached unaligned-access capability") is true, which seems to be calculated whether the architecture is i386, x86, amd64, or x86_64. I think we should perform similar check for the use of Unsafe. Reference: https://github.com/netty/netty/blob/4.1/common/src/main/java/io/netty/util/internal/PlatformDependent0.java#L112 ## How was this patch tested? Unit test suite Author: tedyu <[email protected]> Closes #11943 from tedyu/master. Project: http://git-wip-us.apache.org/repos/asf/spark/repo Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e1f68453 Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e1f68453 Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e1f68453 Branch: refs/heads/master Commit: e1f6845391078726f60e760f0ea68ccf81f9eca9 Parents: 366cac6 Author: tedyu <[email protected]> Authored: Tue Mar 29 17:16:53 2016 -0700 Committer: Shixiong Zhu <[email protected]> Committed: Tue Mar 29 17:16:53 2016 -0700 ---------------------------------------------------------------------- .../java/org/apache/spark/unsafe/Platform.java | 28 ++++++++++++++++++++ .../org/apache/spark/memory/MemoryManager.scala | 3 +++ 2 files changed, 31 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/spark/blob/e1f68453/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java ---------------------------------------------------------------------- diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java index 18761bf..672552c 100644 --- a/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java +++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/Platform.java @@ -18,6 +18,7 @@ package org.apache.spark.unsafe; import java.lang.reflect.Field; +import java.lang.reflect.Method; import sun.misc.Unsafe; @@ -37,6 +38,33 @@ public final class Platform { public static final int DOUBLE_ARRAY_OFFSET; + private static final boolean unaligned; + static { + boolean _unaligned; + // use reflection to access unaligned field + try { + Class<?> bitsClass = + Class.forName("java.nio.Bits", false, ClassLoader.getSystemClassLoader()); + Method unalignedMethod = bitsClass.getDeclaredMethod("unaligned"); + unalignedMethod.setAccessible(true); + _unaligned = Boolean.TRUE.equals(unalignedMethod.invoke(null)); + } catch (Throwable t) { + // We at least know x86 and x64 support unaligned access. + String arch = System.getProperty("os.arch", ""); + //noinspection DynamicRegexReplaceableByCompiledPattern + _unaligned = arch.matches("^(i[3-6]86|x86(_64)?|x64|amd64)$"); + } + unaligned = _unaligned; + } + + /** + * @return true when running JVM is having sun's Unsafe package available in it and underlying + * system having unaligned-access capability. + */ + public static boolean unaligned() { + return unaligned; + } + public static int getInt(Object object, long offset) { return _UNSAFE.getInt(object, offset); } http://git-wip-us.apache.org/repos/asf/spark/blob/e1f68453/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala ---------------------------------------------------------------------- diff --git a/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala b/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala index 10656bc..0210217 100644 --- a/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala +++ b/core/src/main/scala/org/apache/spark/memory/MemoryManager.scala @@ -23,6 +23,7 @@ import org.apache.spark.SparkConf import org.apache.spark.internal.Logging import org.apache.spark.storage.BlockId import org.apache.spark.storage.memory.MemoryStore +import org.apache.spark.unsafe.Platform import org.apache.spark.unsafe.array.ByteArrayMethods import org.apache.spark.unsafe.memory.MemoryAllocator @@ -190,6 +191,8 @@ private[spark] abstract class MemoryManager( if (conf.getBoolean("spark.memory.offHeap.enabled", false)) { require(conf.getSizeAsBytes("spark.memory.offHeap.size", 0) > 0, "spark.memory.offHeap.size must be > 0 when spark.memory.offHeap.enabled == true") + require(Platform.unaligned(), + "No support for unaligned Unsafe. Set spark.memory.offHeap.enabled to false.") MemoryMode.OFF_HEAP } else { MemoryMode.ON_HEAP --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
