This is an automated email from the ASF dual-hosted git repository.
myui pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-hivemall.git
The following commit(s) were added to refs/heads/master by this push:
new 2621ca9 [HIVEMALL-311] Upgrade Kryo version from 2.21 to 2.24.0
2621ca9 is described below
commit 2621ca984eb4724be9eea2b87ded8a98e4e3391a
Author: Makoto Yui <[email protected]>
AuthorDate: Sun May 2 15:55:46 2021 +0900
[HIVEMALL-311] Upgrade Kryo version from 2.21 to 2.24.0
## What changes were proposed in this pull request?
xgboost4j and xgboost module used Kryo version 2.21 but it has a bug in
serializing generic collections. So, update Kryo version to 2.24.0 just in case.
## What type of PR is it?
Bug Fix
## What is the Jira issue?
https://issues.apache.org/jira/browse/HIVEMALL-311
## How was this patch tested?
unit tests
## Checklist
(Please remove this section if not needed; check `x` for YES, blank for NO)
- [x] Did you apply source code formatter, i.e., `./bin/format_code.sh`,
for your commit?
- [ ] Did you run system tests on Hive (or Spark)?
Author: Makoto Yui <[email protected]>
Closes #241 from myui/kryo_update.
---
core/pom.xml | 6 +++
core/src/test/java/hivemall/TestUtils.java | 64 ++++++++++++++++++++++++++----
pom.xml | 1 +
xgboost/pom.xml | 2 +-
4 files changed, 65 insertions(+), 8 deletions(-)
diff --git a/core/pom.xml b/core/pom.xml
index ee65e63..20eba13 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -163,6 +163,12 @@
<artifactId>mockito-core</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>com.esotericsoftware.kryo</groupId>
+ <artifactId>kryo</artifactId>
+ <version>${kryo.version}</version>
+ <scope>test</scope>
+ </dependency>
</dependencies>
</project>
diff --git a/core/src/test/java/hivemall/TestUtils.java
b/core/src/test/java/hivemall/TestUtils.java
index fd8d189..3fb61dc 100644
--- a/core/src/test/java/hivemall/TestUtils.java
+++ b/core/src/test/java/hivemall/TestUtils.java
@@ -32,6 +32,7 @@ import
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hive.com.esotericsoftware.kryo.Kryo;
import org.apache.hive.com.esotericsoftware.kryo.io.Input;
import org.apache.hive.com.esotericsoftware.kryo.io.Output;
+import org.objenesis.strategy.StdInstantiatorStrategy;
public final class TestUtils {
@@ -48,8 +49,11 @@ public final class TestUtils {
udf.initialize(ois);
// serialization after initialization
- byte[] serialized = serializeObjectByKryo(udf);
- deserializeObjectByKryo(serialized, clazz);
+ byte[] serialized1 = serializeObjectByKryo(udf);
+ deserializeObjectByKryo(serialized1, clazz);
+
+ byte[] serialized2 = serializeObjectByOriginalKryo(udf);
+ deserializeObjectByOriginalKryo(serialized2, clazz);
int size = row.length;
GenericUDF.DeferredObject[] rowDeferred = new
GenericUDF.DeferredObject[size];
@@ -60,8 +64,11 @@ public final class TestUtils {
udf.evaluate(rowDeferred);
// serialization after evaluating row
- serialized = serializeObjectByKryo(udf);
- TestUtils.deserializeObjectByKryo(serialized, clazz);
+ serialized1 = serializeObjectByKryo(udf);
+ TestUtils.deserializeObjectByKryo(serialized1, clazz);
+
+ serialized2 = serializeObjectByOriginalKryo(udf);
+ TestUtils.deserializeObjectByOriginalKryo(serialized2, clazz);
udf.close();
}
@@ -82,6 +89,9 @@ public final class TestUtils {
byte[] serialized = serializeObjectByKryo(udtf);
deserializeObjectByKryo(serialized, clazz);
+ byte[] serialized2 = serializeObjectByOriginalKryo(udtf);
+ deserializeObjectByOriginalKryo(serialized2, clazz);
+
udtf.setCollector(new Collector() {
public void collect(Object input) throws HiveException {
// noop
@@ -99,9 +109,12 @@ public final class TestUtils {
udtf.close();
}
+ // -------------------------------
+ // Hive version of Kryo
+
@Nonnull
public static byte[] serializeObjectByKryo(@Nonnull Object obj) {
- Kryo kryo = getKryo();
+ Kryo kryo = getHiveKryo();
ByteArrayOutputStream bos = new ByteArrayOutputStream();
Output output = new Output(bos);
kryo.writeObject(output, obj);
@@ -111,7 +124,7 @@ public final class TestUtils {
@Nonnull
public static <T> T deserializeObjectByKryo(@Nonnull byte[] in, @Nonnull
Class<T> clazz) {
- Kryo kryo = getKryo();
+ Kryo kryo = getHiveKryo();
Input inp = new Input(in);
T t = kryo.readObject(inp, clazz);
inp.close();
@@ -119,8 +132,45 @@ public final class TestUtils {
}
@Nonnull
- private static Kryo getKryo() {
+ private static Kryo getHiveKryo() {
return Utilities.runtimeSerializationKryo.get();
}
+ // -------------------------------
+ // esotericsoftware's original version of Kryo
+
+ @Nonnull
+ public static byte[] serializeObjectByOriginalKryo(@Nonnull Object obj) {
+ com.esotericsoftware.kryo.Kryo kryo = getOriginalKryo();
+ ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ com.esotericsoftware.kryo.io.Output output = new
com.esotericsoftware.kryo.io.Output(bos);
+ kryo.writeObject(output, obj);
+ output.close();
+ return bos.toByteArray();
+ }
+
+ @Nonnull
+ public static <T> T deserializeObjectByOriginalKryo(@Nonnull byte[] in,
+ @Nonnull Class<T> clazz) {
+ com.esotericsoftware.kryo.Kryo kryo = getOriginalKryo();
+ com.esotericsoftware.kryo.io.Input inp = new
com.esotericsoftware.kryo.io.Input(in);
+ T t = kryo.readObject(inp, clazz);
+ inp.close();
+ return t;
+ }
+
+ @Nonnull
+ private static com.esotericsoftware.kryo.Kryo getOriginalKryo() {
+ com.esotericsoftware.kryo.Kryo kryo = new
com.esotericsoftware.kryo.Kryo();
+
+ // kryo.setReferences(true);
+ // kryo.setRegistrationRequired(false);
+
+ // see https://stackoverflow.com/a/23962797/5332768
+ ((com.esotericsoftware.kryo.Kryo.DefaultInstantiatorStrategy)
kryo.getInstantiatorStrategy()).setFallbackInstantiatorStrategy(
+ new StdInstantiatorStrategy());
+
+ return kryo;
+ }
+
}
diff --git a/pom.xml b/pom.xml
index 1b635f7..b75b3aa 100644
--- a/pom.xml
+++ b/pom.xml
@@ -272,6 +272,7 @@
<maven-enforcer.requireMavenVersion>[3.3.1,)</maven-enforcer.requireMavenVersion>
<surefire.version>2.19.1</surefire.version>
<netty.version>4.1.42.Final</netty.version>
+ <kryo.version>2.24.0</kryo.version>
</properties>
<distributionManagement>
diff --git a/xgboost/pom.xml b/xgboost/pom.xml
index ca8725a..63f9f64 100644
--- a/xgboost/pom.xml
+++ b/xgboost/pom.xml
@@ -97,7 +97,7 @@
<!-- NOTE: This dependency needed for xgboost4j -->
<groupId>com.esotericsoftware.kryo</groupId>
<artifactId>kryo</artifactId>
- <version>2.21</version>
+ <version>${kryo.version}</version>
<scope>compile</scope>
</dependency>
<dependency>
