This is an automated email from the ASF dual-hosted git repository.
jacques pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/master by this push:
new 5115f10 ARROW-9016: [Java] Remove direct references to Netty/Unsafe
Allocators
5115f10 is described below
commit 5115f106925dbe6977717df1531b0cc46d8682d3
Author: Ryan Murray <[email protected]>
AuthorDate: Wed Jun 3 20:10:38 2020 -0700
ARROW-9016: [Java] Remove direct references to Netty/Unsafe Allocators
As part of ARROW-8230 this removes direct references to Netty and Unsafe
Allocation
managers in the `DefaultAllocationManagerOption`.
This maintains Netty as the default allocator until ARROW-8230 is closed.
Closes #7329 from rymurr/ARROW-9016
Authored-by: Ryan Murray <[email protected]>
Signed-off-by: Jacques Nadeau <[email protected]>
---
.../org/apache/arrow/memory/CheckAllocator.java | 86 ++++++++++++++++++++++
.../memory/DefaultAllocationManagerFactory.java | 35 +++++++++
.../memory/DefaultAllocationManagerOption.java | 26 ++++++-
.../arrow/memory/NettyAllocationManager.java | 13 +---
.../arrow/memory/UnsafeAllocationManager.java | 13 +---
5 files changed, 146 insertions(+), 27 deletions(-)
diff --git
a/java/memory/src/main/java/org/apache/arrow/memory/CheckAllocator.java
b/java/memory/src/main/java/org/apache/arrow/memory/CheckAllocator.java
new file mode 100644
index 0000000..8421057
--- /dev/null
+++ b/java/memory/src/main/java/org/apache/arrow/memory/CheckAllocator.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.memory;
+
+import java.io.IOException;
+import java.net.URL;
+import java.util.Enumeration;
+import java.util.LinkedHashSet;
+import java.util.Set;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Static method to ensure we have a RootAllocator on the classpath and report
which one is used.
+ */
+final class CheckAllocator {
+ private static final Logger logger =
LoggerFactory.getLogger(CheckAllocator.class);
+ private static final String ALLOCATOR_PATH =
"org/apache/arrow/memory/DefaultAllocationManagerFactory.class";
+
+ private CheckAllocator() {
+
+ }
+
+ static String check() {
+ Set<URL> urls = scanClasspath();
+ URL rootAllocator = assertOnlyOne(urls);
+ reportResult(rootAllocator);
+ return "org.apache.arrow.memory.DefaultAllocationManagerFactory";
+ }
+
+
+ private static Set<URL> scanClasspath() {
+ // LinkedHashSet appropriate here because it preserves insertion order
+ // during iteration
+ Set<URL> allocatorPathSet = new LinkedHashSet<>();
+ try {
+ ClassLoader allocatorClassLoader = CheckAllocator.class.getClassLoader();
+ Enumeration<URL> paths;
+ if (allocatorClassLoader == null) {
+ paths = ClassLoader.getSystemResources(ALLOCATOR_PATH);
+ } else {
+ paths = allocatorClassLoader.getResources(ALLOCATOR_PATH);
+ }
+ while (paths.hasMoreElements()) {
+ URL path = paths.nextElement();
+ allocatorPathSet.add(path);
+ }
+ } catch (IOException ioe) {
+ logger.error("Error getting resources from path", ioe);
+ }
+ return allocatorPathSet;
+ }
+
+ private static void reportResult(URL rootAllocator) {
+ String path = rootAllocator.getPath();
+ String subPath = path.substring(path.indexOf("memory"));
+ logger.info("Using DefaultAllocationManager at {}", subPath);
+ }
+
+ private static URL assertOnlyOne(Set<URL> urls) {
+ if (urls.size() > 1) {
+ logger.warn("More than one DefaultAllocationManager on classpath.
Choosing first found");
+ }
+ if (urls.isEmpty()) {
+ throw new RuntimeException("No DefaultAllocationManager found on
classpath. Can't allocate Arrow buffers.");
+ }
+ return urls.iterator().next();
+ }
+
+}
diff --git
a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
new file mode 100644
index 0000000..a7fe795
--- /dev/null
+++
b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerFactory.java
@@ -0,0 +1,35 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.arrow.memory;
+
+/**
+ * The default Allocation Manager Factory for a module.
+ *
+ * This will be split out to the arrow-memory-netty/arrow-memory-unsafe modules
+ * as part of ARROW-8230. This is currently a placeholder which defaults to
Netty.
+ *
+ */
+public class DefaultAllocationManagerFactory implements
AllocationManager.Factory {
+
+ public static final AllocationManager.Factory FACTORY = new
DefaultAllocationManagerFactory();
+
+ @Override
+ public AllocationManager create(BaseAllocator accountingAllocator, long
size) {
+ return new NettyAllocationManager(accountingAllocator, size);
+ }
+}
diff --git
a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java
b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java
index 66f0ec6..a652527 100644
---
a/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java
+++
b/java/memory/src/main/java/org/apache/arrow/memory/DefaultAllocationManagerOption.java
@@ -17,6 +17,8 @@
package org.apache.arrow.memory;
+import java.lang.reflect.Field;
+
/**
* A class for choosing the default allocation manager.
*/
@@ -85,14 +87,32 @@ public class DefaultAllocationManagerOption {
switch (type) {
case Netty:
- return NettyAllocationManager.FACTORY;
+ return getNettyFactory();
case Unsafe:
- return UnsafeAllocationManager.FACTORY;
+ return getUnsafeFactory();
case Unknown:
LOGGER.info("allocation manager type not specified, using netty as the
default type");
- return NettyAllocationManager.FACTORY;
+ return getFactory(CheckAllocator.check());
default:
throw new IllegalStateException("Unknown allocation manager type: " +
type);
}
}
+
+ private static AllocationManager.Factory getFactory(String clazzName) {
+ try {
+ Field field = Class.forName(clazzName).getDeclaredField("FACTORY");
+ field.setAccessible(true);
+ return (AllocationManager.Factory) field.get(null);
+ } catch (Exception e) {
+ throw new RuntimeException("Unable to instantiate Allocation Manager for
" + clazzName, e);
+ }
+ }
+
+ private static AllocationManager.Factory getUnsafeFactory() {
+ return getFactory("org.apache.arrow.memory.UnsafeAllocationManager");
+ }
+
+ private static AllocationManager.Factory getNettyFactory() {
+ return getFactory("org.apache.arrow.memory.NettyAllocationManager");
+ }
}
diff --git
a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java
b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java
index 3414017..0f713a8 100644
---
a/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java
+++
b/java/memory/src/main/java/org/apache/arrow/memory/NettyAllocationManager.java
@@ -27,7 +27,7 @@ import io.netty.util.internal.PlatformDependent;
*/
public class NettyAllocationManager extends AllocationManager {
- public static final Factory FACTORY = new Factory();
+ public static final AllocationManager.Factory FACTORY =
NettyAllocationManager::new;
/**
* The default cut-off value for switching allocation strategies.
@@ -105,15 +105,4 @@ public class NettyAllocationManager extends
AllocationManager {
return allocatedSize;
}
- /**
- * Factory for creating {@link NettyAllocationManager}.
- */
- public static class Factory implements AllocationManager.Factory {
- private Factory() {}
-
- @Override
- public AllocationManager create(BaseAllocator accountingAllocator, long
size) {
- return new NettyAllocationManager(accountingAllocator, size);
- }
- }
}
diff --git
a/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java
b/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java
index d450444..976f890 100644
---
a/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java
+++
b/java/memory/src/main/java/org/apache/arrow/memory/UnsafeAllocationManager.java
@@ -24,7 +24,7 @@ import org.apache.arrow.memory.util.MemoryUtil;
*/
public final class UnsafeAllocationManager extends AllocationManager {
- public static final Factory FACTORY = new Factory();
+ public static final AllocationManager.Factory FACTORY =
UnsafeAllocationManager::new;
private final long allocatedSize;
@@ -51,15 +51,4 @@ public final class UnsafeAllocationManager extends
AllocationManager {
MemoryUtil.UNSAFE.freeMemory(allocatedAddress);
}
- /**
- * Factory for creating {@link UnsafeAllocationManager}.
- */
- public static class Factory implements AllocationManager.Factory {
- private Factory() {}
-
- @Override
- public AllocationManager create(BaseAllocator accountingAllocator, long
size) {
- return new UnsafeAllocationManager(accountingAllocator, size);
- }
- }
}
