This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 918c2e0009c [HUDI-7976] Fix BUG introduced in HUDI-7955 due to usage
of wrong class (#11612)
918c2e0009c is described below
commit 918c2e0009c054f9fcd4ca19ba3258c491483708
Author: voonhous <[email protected]>
AuthorDate: Sun Jul 14 11:29:44 2024 +0800
[HUDI-7976] Fix BUG introduced in HUDI-7955 due to usage of wrong class
(#11612)
---
.../hudi/hadoop/utils/HiveAvroSerializer.java | 3 +-
.../apache/hudi/hadoop/utils/HoodieHiveUtils.java | 8 +---
.../apache/hudi/hadoop/utils/shims/Hive2Shim.java | 9 +----
.../apache/hudi/hadoop/utils/shims/Hive3Shim.java | 45 ++++++++++------------
.../apache/hudi/hadoop/utils/shims/HiveShim.java | 4 +-
5 files changed, 26 insertions(+), 43 deletions(-)
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
index 47d984c89c3..0c3362ba981 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HiveAvroSerializer.java
@@ -304,8 +304,7 @@ public class HiveAvroSerializer {
case DATE:
return HoodieHiveUtils.getDays(structFieldData);
case TIMESTAMP:
- Object timestamp = HoodieHiveUtils.getTimestamp(structFieldData);
- return HoodieHiveUtils.getMills(timestamp);
+ return HoodieHiveUtils.getMills(structFieldData);
case INT:
if (schema.getLogicalType() != null &&
schema.getLogicalType().getName().equals("date")) {
return new
WritableDateObjectInspector().getPrimitiveWritableObject(structFieldData).getDays();
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java
index ced39ccf379..b4894c35d41 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/HoodieHiveUtils.java
@@ -182,15 +182,11 @@ public class HoodieHiveUtils {
return HIVE_SHIM.getDateWriteable(value);
}
- public static Object getTimestamp(Object fieldData) {
- return HIVE_SHIM.unwrapTimestampAsPrimitive(fieldData);
- }
-
public static int getDays(Object dateWritable) {
return HIVE_SHIM.getDays(dateWritable);
}
- public static long getMills(Object timestamp) {
- return HIVE_SHIM.getMills(timestamp);
+ public static long getMills(Object timestampWritable) {
+ return HIVE_SHIM.getMills(timestampWritable);
}
}
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive2Shim.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive2Shim.java
index e2a4f36cb7f..7f4b683d246 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive2Shim.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive2Shim.java
@@ -42,11 +42,6 @@ public class Hive2Shim implements HiveShim {
return new TimestampWritable(timestamp);
}
- @Override
- public Object unwrapTimestampAsPrimitive(Object o) {
- return o == null ? null : ((TimestampWritable) o).getTimestamp();
- }
-
public Writable getDateWriteable(int value) {
return new DateWritable(value);
}
@@ -55,7 +50,7 @@ public class Hive2Shim implements HiveShim {
return ((DateWritable) dateWritable).getDays();
}
- public long getMills(Object timestamp) {
- return ((Timestamp) timestamp).getTime();
+ public long getMills(Object timestampWritable) {
+ return ((TimestampWritable) timestampWritable).getTimestamp().getTime();
}
}
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java
index 9d6dca4f2b3..bc5b7b3e124 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/Hive3Shim.java
@@ -36,11 +36,12 @@ public class Hive3Shim implements HiveShim {
public static final Logger LOG = LoggerFactory.getLogger(Hive3Shim.class);
- public static final String HIVE_TIMESTAMP_TYPE_CLASS =
"org.apache.hadoop.hive.common.type.Timestamp";
- public static final String TIMESTAMP_WRITEABLE_V2_CLASS =
"org.apache.hadoop.hive.serde2.io.TimestampWritableV2";
- public static final String DATE_WRITEABLE_V2_CLASS =
"org.apache.hadoop.hive.serde2.io.DateWritableV2";
+ public static final String TIMESTAMP_CLASS_NAME =
"org.apache.hadoop.hive.common.type.Timestamp";
+ public static final String TIMESTAMP_WRITEABLE_V2_CLASS_NAME =
"org.apache.hadoop.hive.serde2.io.TimestampWritableV2";
+ public static final String DATE_WRITEABLE_V2_CLASS_NAME =
"org.apache.hadoop.hive.serde2.io.DateWritableV2";
private static Class<?> TIMESTAMP_CLASS = null;
+ private static Class<?> TIMESTAMP_WRITABLE_V2_CLASS = null;
private static Method SET_TIME_IN_MILLIS = null;
private static Method TO_SQL_TIMESTAMP = null;
private static Method GET_TIMESTAMP = null;
@@ -53,22 +54,28 @@ public class Hive3Shim implements HiveShim {
static {
// timestamp
try {
- TIMESTAMP_CLASS = Class.forName(HIVE_TIMESTAMP_TYPE_CLASS);
+ // Hive.Timestamp methods
+ TIMESTAMP_CLASS = Class.forName(TIMESTAMP_CLASS_NAME);
SET_TIME_IN_MILLIS =
TIMESTAMP_CLASS.getDeclaredMethod("setTimeInMillis", long.class);
- GET_TIMESTAMP = TIMESTAMP_CLASS.getDeclaredMethod("getTimestamp");
TO_SQL_TIMESTAMP = TIMESTAMP_CLASS.getDeclaredMethod("toSqlTimestamp");
- TIMESTAMP_WRITEABLE_V2_CONSTRUCTOR =
Class.forName(TIMESTAMP_WRITEABLE_V2_CLASS).getConstructor(TIMESTAMP_CLASS);
+
+ // Hive.TimestampWritable methods
+ TIMESTAMP_WRITABLE_V2_CLASS =
Class.forName(TIMESTAMP_WRITEABLE_V2_CLASS_NAME);
+ GET_TIMESTAMP =
TIMESTAMP_WRITABLE_V2_CLASS.getDeclaredMethod("getTimestamp");
+ TIMESTAMP_WRITEABLE_V2_CONSTRUCTOR =
TIMESTAMP_WRITABLE_V2_CLASS.getConstructor(TIMESTAMP_CLASS);
} catch (ClassNotFoundException | NoSuchMethodException e) {
- LOG.trace("can not find hive3 timestampv2 class or method, use hive2
class!", e);
+ // This will be printed out when Hive3Shim initialization as a singleton
fails
+ LOG.warn("cannot find hive3 timestampv2 class or method, use hive2
class!", e);
}
// date
try {
- DATE_WRITEABLE_CLASS = Class.forName(DATE_WRITEABLE_V2_CLASS);
+ DATE_WRITEABLE_CLASS = Class.forName(DATE_WRITEABLE_V2_CLASS_NAME);
GET_DAYS = DATE_WRITEABLE_CLASS.getDeclaredMethod("getDays");
DATE_WRITEABLE_V2_CONSTRUCTOR =
DATE_WRITEABLE_CLASS.getConstructor(int.class);
} catch (ClassNotFoundException | NoSuchMethodException e) {
- LOG.trace("can not find hive3 datev2 class or method, use hive2 class!",
e);
+ // This will be printed out when Hive3Shim initialization as a singleton
fails
+ LOG.warn("cannot find hive3 datev2 class or method, use hive2 class!",
e);
}
}
@@ -96,19 +103,6 @@ public class Hive3Shim implements HiveShim {
}
}
- @Override
- public Object unwrapTimestampAsPrimitive(Object o) {
- if (o == null) {
- return null;
- }
-
- try {
- return GET_TIMESTAMP.invoke(o);
- } catch (IllegalAccessException | InvocationTargetException e) {
- throw new HoodieException("unable to get timestamp from writable using
v2 class!", e);
- }
- }
-
/**
* Get date writeable object from int value.
* Hive3 use DateWritableV2 to build date objects and Hive2 use DateWritable.
@@ -124,15 +118,16 @@ public class Hive3Shim implements HiveShim {
public int getDays(Object dateWritable) {
try {
- return (int)GET_DAYS.invoke(dateWritable);
+ return (int) GET_DAYS.invoke(dateWritable);
} catch (IllegalAccessException | InvocationTargetException e) {
throw new HoodieException("can not create writable v2 class!", e);
}
}
- public long getMills(Object timestamp) {
+ public long getMills(Object timestampWritable) {
try {
- return ((Timestamp) TO_SQL_TIMESTAMP.invoke(timestamp)).getTime();
+ Object hiveTimestamp = GET_TIMESTAMP.invoke(timestampWritable);
+ return ((Timestamp) TO_SQL_TIMESTAMP.invoke(hiveTimestamp)).getTime();
} catch (IllegalAccessException | InvocationTargetException e) {
throw new HoodieException("can not create writable v2 class!", e);
}
diff --git
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/HiveShim.java
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/HiveShim.java
index 45a8206ce57..3cc5edac390 100644
---
a/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/HiveShim.java
+++
b/hudi-hadoop-mr/src/main/java/org/apache/hudi/hadoop/utils/shims/HiveShim.java
@@ -27,11 +27,9 @@ public interface HiveShim {
Writable getTimestampWriteable(long value, boolean timestampMillis);
- Object unwrapTimestampAsPrimitive(Object o);
-
Writable getDateWriteable(int value);
int getDays(Object dateWritable);
- long getMills(Object timestamp);
+ long getMills(Object timestampWritable);
}
