This is an automated email from the ASF dual-hosted git repository.
sankarh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hive.git
The following commit(s) were added to refs/heads/master by this push:
new 16730f5 HIVE-25784: Upgrade arrow version to 2.0.0 (Adesh Rao,
reviewed by Sankar Hariappan)
16730f5 is described below
commit 16730f5595577a74542ca5ced90e747ffc2229f7
Author: Adesh Kumar Rao <[email protected]>
AuthorDate: Wed Dec 22 17:27:16 2021 +0530
HIVE-25784: Upgrade arrow version to 2.0.0 (Adesh Rao, reviewed by Sankar
Hariappan)
Signed-off-by: Sankar Hariappan <[email protected]>
Closes (#2896)
---
llap-server/pom.xml | 16 +++++++++++++++-
.../hive/llap/cli/service/AsyncTaskCopyLocalJars.java | 5 ++++-
pom.xml | 2 +-
ql/pom.xml | 16 ++++++++++++++++
.../hadoop/hive/llap/WritableByteChannelAdapter.java | 4 +++-
.../org/apache/hadoop/hive/ql/io/arrow/Deserializer.java | 2 +-
.../org/apache/hadoop/hive/ql/io/arrow/Serializer.java | 2 +-
7 files changed, 41 insertions(+), 6 deletions(-)
diff --git a/llap-server/pom.xml b/llap-server/pom.xml
index 4e940dd..28e08b8 100644
--- a/llap-server/pom.xml
+++ b/llap-server/pom.xml
@@ -233,7 +233,21 @@
<artifactId>disruptor</artifactId>
<version>${disruptor.version}</version>
</dependency>
-
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ <version>${arrow.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
<!-- test intra-project -->
<dependency>
<groupId>org.apache.hive</groupId>
diff --git
a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java
index 40d5db0..5f3e869 100644
---
a/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java
+++
b/llap-server/src/java/org/apache/hadoop/hive/llap/cli/service/AsyncTaskCopyLocalJars.java
@@ -63,8 +63,11 @@ class AsyncTaskCopyLocalJars implements Callable<Void> {
// log4j-1.2-API needed for NDC
org.apache.log4j.config.Log4j1ConfigurationFactory.class,
io.netty.util.NetUtil.class, // netty4
+ io.netty.handler.codec.http.HttpObjectAggregator.class, //
org.apache.arrow.vector.types.pojo.ArrowType.class, //arrow-vector
- org.apache.arrow.memory.BaseAllocator.class, //arrow-memory
+ org.apache.arrow.memory.RootAllocator.class, //arrow-memory
+ org.apache.arrow.memory.NettyAllocationManager.class,
//arrow-memory-netty
+ io.netty.handler.codec.http.HttpObjectAggregator.class, // netty-all
org.apache.arrow.flatbuf.Schema.class, //arrow-format
com.google.flatbuffers.Table.class, //flatbuffers
com.carrotsearch.hppc.ByteArrayDeque.class, //hppc
diff --git a/pom.xml b/pom.xml
index 2fb29f6..57ca0a6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -109,7 +109,7 @@
<antlr.version>3.5.2</antlr.version>
<apache-directory-server.version>1.5.7</apache-directory-server.version>
<!-- Include arrow for LlapOutputFormatService -->
- <arrow.version>0.15.1</arrow.version>
+ <arrow.version>2.0.0</arrow.version>
<avatica.version>1.12.0</avatica.version>
<avro.version>1.8.2</avro.version>
<bcprov-jdk15on.version>1.64</bcprov-jdk15on.version>
diff --git a/ql/pom.xml b/ql/pom.xml
index 4054bb3..1de2f62 100644
--- a/ql/pom.xml
+++ b/ql/pom.xml
@@ -535,6 +535,22 @@
<artifactId>hive-standalone-metastore-server</artifactId>
<version>${project.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.arrow</groupId>
+ <artifactId>arrow-memory-netty</artifactId>
+ <version>${arrow.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-buffer</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>io.netty</groupId>
+ <artifactId>netty-common</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+
<!-- test intra-project -->
<dependency>
<groupId>org.apache.hive</groupId>
diff --git
a/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
b/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
index b931ee5..0ed2aff 100644
--- a/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
+++ b/ql/src/java/org/apache/hadoop/hive/llap/WritableByteChannelAdapter.java
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.nio.channels.WritableByteChannel;
import java.util.concurrent.Semaphore;
+import org.apache.arrow.memory.ArrowByteBufAllocator;
import org.apache.arrow.memory.BufferAllocator;
import org.slf4j.Logger;
@@ -93,7 +94,8 @@ public class WritableByteChannelAdapter implements
WritableByteChannel {
int size = src.remaining();
//Down the semaphore or block until available
takeWriteResources(1);
- ByteBuf buf = allocator.getAsByteBufAllocator().buffer(size);
+ ArrowByteBufAllocator abba = new ArrowByteBufAllocator(allocator);
+ ByteBuf buf = abba.buffer(size);
buf.writeBytes(src);
chc.writeAndFlush(buf).addListener(writeListener);
return size;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Deserializer.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Deserializer.java
index ce8488f..85b4ec6 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Deserializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Deserializer.java
@@ -17,7 +17,7 @@
*/
package org.apache.hadoop.hive.ql.io.arrow;
-import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.DateDayVector;
diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
index c00885e..1a45600 100644
--- a/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
+++ b/ql/src/java/org/apache/hadoop/hive/ql/io/arrow/Serializer.java
@@ -18,7 +18,7 @@
package org.apache.hadoop.hive.ql.io.arrow;
import com.google.common.annotations.VisibleForTesting;
-import io.netty.buffer.ArrowBuf;
+import org.apache.arrow.memory.ArrowBuf;
import org.apache.arrow.vector.BigIntVector;
import org.apache.arrow.vector.BitVector;
import org.apache.arrow.vector.BitVectorHelper;
