[
https://issues.apache.org/jira/browse/ARROW-186?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=16214534#comment-16214534
]
ASF GitHub Bot commented on ARROW-186:
--------------------------------------
wesm closed pull request #98: ARROW-186 - Make sure alignment and memory
padding conform to spec
URL: https://github.com/apache/arrow/pull/98
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
index b6de2e3aa..b22edcfbc 100644
--- a/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
+++ b/java/memory/src/main/java/io/netty/buffer/PooledByteBufAllocatorL.java
@@ -18,7 +18,9 @@
package io.netty.buffer;
+import io.netty.util.internal.PlatformDependent;
import io.netty.util.internal.StringUtil;
+import io.netty.util.internal.SystemPropertyUtil;
import org.apache.arrow.memory.OutOfMemoryException;
@@ -37,7 +39,109 @@
private static final org.slf4j.Logger memoryLogger =
org.slf4j.LoggerFactory.getLogger("arrow" +
".allocator");
+ private static final int DEFAULT_NUM_HEAP_ARENA;
+ private static final int DEFAULT_NUM_DIRECT_ARENA;
+
+ private static final int DEFAULT_PAGE_SIZE;
+ private static final int DEFAULT_MAX_ORDER; // 8192 << 11 = 16 MiB per chunk
+ private static final int DEFAULT_TINY_CACHE_SIZE;
+ private static final int DEFAULT_SMALL_CACHE_SIZE;
+ private static final int DEFAULT_NORMAL_CACHE_SIZE;
+ private static final int DEFAULT_MAX_CACHED_BUFFER_CAPACITY;
+ private static final int DEFAULT_CACHE_TRIM_INTERVAL;
+ private static final boolean DEFAULT_USE_CACHE_FOR_ALL_THREADS;
+ private static final int DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT = 64;
+
+ private static final int MIN_PAGE_SIZE = 4096;
+ private static final int MAX_CHUNK_SIZE = (int) (((long) Integer.MAX_VALUE +
1) / 2);
+
+ static {
+ int defaultPageSize =
SystemPropertyUtil.getInt("io.netty.allocator.pageSize", 8192);
+ Throwable pageSizeFallbackCause = null;
+ try {
+ validateAndCalculatePageShifts(defaultPageSize);
+ } catch (Throwable t) {
+ pageSizeFallbackCause = t;
+ defaultPageSize = 8192;
+ }
+ DEFAULT_PAGE_SIZE = defaultPageSize;
+
+ int defaultMaxOrder =
SystemPropertyUtil.getInt("io.netty.allocator.maxOrder", 11);
+ Throwable maxOrderFallbackCause = null;
+ try {
+ validateAndCalculateChunkSize(DEFAULT_PAGE_SIZE, defaultMaxOrder);
+ } catch (Throwable t) {
+ maxOrderFallbackCause = t;
+ defaultMaxOrder = 11;
+ }
+ DEFAULT_MAX_ORDER = defaultMaxOrder;
+
+ // Determine reasonable default for nHeapArena and nDirectArena.
+ // Assuming each arena has 3 chunks, the pool should not consume more
than 50% of max memory.
+ final Runtime runtime = Runtime.getRuntime();
+
+ // Use 2 * cores by default to reduce condition as we use 2 * cores for
the number of EventLoops
+ // in NIO and EPOLL as well. If we choose a smaller number we will run
into hotspots as allocation and
+ // deallocation needs to be synchronized on the PoolArena.
+ // See https://github.com/netty/netty/issues/3888
+ final int defaultMinNumArena = runtime.availableProcessors() * 2;
+ final int defaultChunkSize = DEFAULT_PAGE_SIZE << DEFAULT_MAX_ORDER;
+ DEFAULT_NUM_HEAP_ARENA = Math.max(0,
+ SystemPropertyUtil.getInt(
+ "io.netty.allocator.numHeapArenas",
+ (int) Math.min(
+ defaultMinNumArena,
+ runtime.maxMemory() / defaultChunkSize / 2 /
3)));
+ DEFAULT_NUM_DIRECT_ARENA = Math.max(0,
+ SystemPropertyUtil.getInt(
+ "io.netty.allocator.numDirectArenas",
+ (int) Math.min(
+ defaultMinNumArena,
+ PlatformDependent.maxDirectMemory() /
defaultChunkSize / 2 / 3)));
+
+ // cache sizes
+ DEFAULT_TINY_CACHE_SIZE =
SystemPropertyUtil.getInt("io.netty.allocator.tinyCacheSize", 512);
+ DEFAULT_SMALL_CACHE_SIZE =
SystemPropertyUtil.getInt("io.netty.allocator.smallCacheSize", 256);
+ DEFAULT_NORMAL_CACHE_SIZE =
SystemPropertyUtil.getInt("io.netty.allocator.normalCacheSize", 64);
+
+ // 32 kb is the default maximum capacity of the cached buffer. Similar
to what is explained in
+ // 'Scalable memory allocation using jemalloc'
+ DEFAULT_MAX_CACHED_BUFFER_CAPACITY = SystemPropertyUtil.getInt(
+ "io.netty.allocator.maxCachedBufferCapacity", 32 * 1024);
+
+ // the number of threshold of allocations when cached entries will be
freed up if not frequently used
+ DEFAULT_CACHE_TRIM_INTERVAL = SystemPropertyUtil.getInt(
+ "io.netty.allocator.cacheTrimInterval", 8192);
+
+ DEFAULT_USE_CACHE_FOR_ALL_THREADS = SystemPropertyUtil.getBoolean(
+ "io.netty.allocator.useCacheForAllThreads", true);
+
+
+ if (memoryLogger.isDebugEnabled()) {
+ memoryLogger.debug("-Dio.netty.allocator.numHeapArenas: {}",
DEFAULT_NUM_HEAP_ARENA);
+ memoryLogger.debug("-Dio.netty.allocator.numDirectArenas: {}",
DEFAULT_NUM_DIRECT_ARENA);
+ if (pageSizeFallbackCause == null) {
+ memoryLogger.debug("-Dio.netty.allocator.pageSize: {}",
DEFAULT_PAGE_SIZE);
+ } else {
+ memoryLogger.debug("-Dio.netty.allocator.pageSize: {}",
DEFAULT_PAGE_SIZE, pageSizeFallbackCause);
+ }
+ if (maxOrderFallbackCause == null) {
+ memoryLogger.debug("-Dio.netty.allocator.maxOrder: {}",
DEFAULT_MAX_ORDER);
+ } else {
+ memoryLogger.debug("-Dio.netty.allocator.maxOrder: {}",
DEFAULT_MAX_ORDER, maxOrderFallbackCause);
+ }
+ memoryLogger.debug("-Dio.netty.allocator.chunkSize: {}",
DEFAULT_PAGE_SIZE << DEFAULT_MAX_ORDER);
+ memoryLogger.debug("-Dio.netty.allocator.tinyCacheSize: {}",
DEFAULT_TINY_CACHE_SIZE);
+ memoryLogger.debug("-Dio.netty.allocator.smallCacheSize: {}",
DEFAULT_SMALL_CACHE_SIZE);
+ memoryLogger.debug("-Dio.netty.allocator.normalCacheSize: {}",
DEFAULT_NORMAL_CACHE_SIZE);
+ memoryLogger.debug("-Dio.netty.allocator.maxCachedBufferCapacity:
{}", DEFAULT_MAX_CACHED_BUFFER_CAPACITY);
+ memoryLogger.debug("-Dio.netty.allocator.cacheTrimInterval: {}",
DEFAULT_CACHE_TRIM_INTERVAL);
+ memoryLogger.debug("-Dio.netty.allocator.useCacheForAllThreads: {}",
DEFAULT_USE_CACHE_FOR_ALL_THREADS);
+ }
+ }
+
private static final int MEMORY_LOGGER_FREQUENCY_SECONDS = 60;
+
public final UnsafeDirectLittleEndian empty;
private final AtomicLong hugeBufferSize = new AtomicLong(0);
private final AtomicLong hugeBufferCount = new AtomicLong(0);
@@ -79,6 +183,38 @@ public long getNormalBufferCount() {
return normalBufferSize.get();
}
+ private static int validateAndCalculatePageShifts(int pageSize) {
+ if (pageSize < MIN_PAGE_SIZE) {
+ throw new IllegalArgumentException(
+ "pageSize: " + pageSize + " (expected: " + MIN_PAGE_SIZE + ")");
+ }
+
+ if ((pageSize & pageSize - 1) != 0) {
+ throw new IllegalArgumentException("pageSize: " + pageSize + "
(expected: power of 2)");
+ }
+
+ // Logarithm base 2. At this point we know that pageSize is a power of two.
+ return Integer.SIZE - 1 - Integer.numberOfLeadingZeros(pageSize);
+ }
+
+ private static int validateAndCalculateChunkSize(int pageSize, int maxOrder)
{
+ if (maxOrder > 14) {
+ throw new IllegalArgumentException("maxOrder: " + maxOrder + "
(expected: 0-14)");
+ }
+
+ // Ensure the resulting chunkSize does not overflow.
+ int chunkSize = pageSize;
+ for (int i = maxOrder; i > 0; i--) {
+ if (chunkSize > MAX_CHUNK_SIZE / 2) {
+ throw new IllegalArgumentException(
+ String.format("pageSize (%d) << maxOrder (%d) must not exceed %d",
pageSize, maxOrder,
+ MAX_CHUNK_SIZE));
+ }
+ chunkSize <<= 1;
+ }
+ return chunkSize;
+ }
+
private static class AccountedUnsafeDirectLittleEndian extends
UnsafeDirectLittleEndian {
private final long initialCapacity;
@@ -129,7 +265,7 @@ public boolean release(int decrement) {
private final int chunkSize;
public InnerAllocator() {
- super(true);
+ super(true, DEFAULT_NUM_HEAP_ARENA, DEFAULT_NUM_DIRECT_ARENA,
DEFAULT_PAGE_SIZE, DEFAULT_MAX_ORDER, DEFAULT_TINY_CACHE_SIZE,
DEFAULT_SMALL_CACHE_SIZE, DEFAULT_NORMAL_CACHE_SIZE,
DEFAULT_USE_CACHE_FOR_ALL_THREADS, DEFAULT_DIRECT_MEMORY_CACHE_ALIGNMENT);
try {
Field f =
PooledByteBufAllocator.class.getDeclaredField("directArenas");
diff --git
a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
index 59b7be87e..2615b9929 100644
--- a/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
+++ b/java/memory/src/test/java/org/apache/arrow/memory/TestBaseAllocator.java
@@ -33,6 +33,7 @@
// private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(TestBaseAllocator.class);
private final static int MAX_ALLOCATION = 8 * 1024;
+ private static final long CACHE_ALIGN = 64;
/*
// ---------------------------------------- DEBUG
-----------------------------------
@@ -657,4 +658,29 @@ public void assertEquiv(ArrowBuf origBuf, ArrowBuf newBuf)
{
assertEquals(origBuf.readerIndex(), newBuf.readerIndex());
assertEquals(origBuf.writerIndex(), newBuf.writerIndex());
}
+
+ @Test
+ public void testMemoryOffset() {
+ try (final BufferAllocator allocator = new RootAllocator(10000)) {
+ int[] reqCapacities = { 0, 5, 15, 67, 130, 510, 1024, 1023, 1025 };
+ int[] expectedResult = { 0, 8, 16, 128, 256, 512, 1024, 1024, 2048 };
+
+ for (int i = 0; i < reqCapacities.length; i++) {
+ ArrowBuf buf = allocator.buffer(reqCapacities[i]);
+ if (buf.capacity() > 0) {
+ long memory = buf.memoryAddress();
+ int delta = (int) (memory % CACHE_ALIGN);
+ assertTrue("memory is not align to 64-bytes", delta == 0);
+ assertEquals(expectedResult[i], buf.capacity());
+ int size = buf.capacity();
+ if (size >= CACHE_ALIGN) {
+ delta = (int) (size % CACHE_ALIGN);
+ assertTrue(" size is not align to 64-bytes ", delta == 0);
+ }
+ }
+ buf.release();
+ }
+ }
+ }
+
}
diff --git a/java/pom.xml b/java/pom.xml
index 5edd605e8..7504ea4c3 100644
--- a/java/pom.xml
+++ b/java/pom.xml
@@ -454,7 +454,7 @@
<dependency>
<groupId>io.netty</groupId>
<artifactId>netty-handler</artifactId>
- <version>4.0.41.Final</version>
+ <version>4.0.45.Final</version>
</dependency>
<dependency>
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
> [Java] Make sure alignment and memory padding conform to spec
> -------------------------------------------------------------
>
> Key: ARROW-186
> URL: https://issues.apache.org/jira/browse/ARROW-186
> Project: Apache Arrow
> Issue Type: Bug
> Components: Java - Memory
> Reporter: Micah Kornfield
> Labels: pull-request-available
>
> Per spec 64 byte alignment and padding for buffers.
--
This message was sent by Atlassian JIRA
(v6.4.14#64029)
