Author: rohini
Date: Mon Jan 16 01:50:52 2023
New Revision: 1906696
URL: http://svn.apache.org/viewvc?rev=1906696&view=rev
Log:
PIG-5431: Date datatype is different between Hive 1.x and Hive 3.x (rohini)
Added:
pig/trunk/test/org/apache/pig/builtin/orc/datetype.orc (with props)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/shims/src/hive1/org/apache/pig/hive/HiveShims.java
pig/trunk/shims/src/hive3/org/apache/pig/hive/HiveShims.java
pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java
pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1906696&r1=1906695&r2=1906696&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Jan 16 01:50:52 2023
@@ -116,6 +116,8 @@ OPTIMIZATIONS
BUG FIXES
+PIG-5431: Date datatype is different between Hive 1.x and Hive 3.x (rohini)
+
PIG-5433: Fix test failures with TestHBaseStorage and htrace dependency
(rohini)
PIG-5430: TestTezGraceParallelism failing due to tez log change (knoguchi)
Modified: pig/trunk/shims/src/hive1/org/apache/pig/hive/HiveShims.java
URL:
http://svn.apache.org/viewvc/pig/trunk/shims/src/hive1/org/apache/pig/hive/HiveShims.java?rev=1906696&r1=1906695&r2=1906696&view=diff
==============================================================================
--- pig/trunk/shims/src/hive1/org/apache/pig/hive/HiveShims.java (original)
+++ pig/trunk/shims/src/hive1/org/apache/pig/hive/HiveShims.java Mon Jan 16
01:50:52 2023
@@ -19,6 +19,7 @@ package org.apache.pig.hive;
import java.math.BigDecimal;
import java.math.BigInteger;
+import java.sql.Date;
import java.sql.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -143,6 +144,17 @@ public class HiveShims {
return new SimpleGenericUDAFParameterInfo(arguments, distinct,
allColumns);
}
+ public static class DateShim {
+
+ public static Date cast(Object ts) {
+ return (Date) ts;
+ }
+
+ public static long millisFromDate(Object ts) {
+ return cast(ts).getTime();
+ }
+ }
+
public static class TimestampShim {
public static Timestamp cast(Object ts) {
Modified: pig/trunk/shims/src/hive3/org/apache/pig/hive/HiveShims.java
URL:
http://svn.apache.org/viewvc/pig/trunk/shims/src/hive3/org/apache/pig/hive/HiveShims.java?rev=1906696&r1=1906695&r2=1906696&view=diff
==============================================================================
--- pig/trunk/shims/src/hive3/org/apache/pig/hive/HiveShims.java (original)
+++ pig/trunk/shims/src/hive3/org/apache/pig/hive/HiveShims.java Mon Jan 16
01:50:52 2023
@@ -20,6 +20,7 @@ package org.apache.pig.hive;
import java.math.BigDecimal;
import java.math.BigInteger;
+import org.apache.hadoop.hive.common.type.Date;
import org.apache.hadoop.hive.common.type.HiveDecimal;
import org.apache.hadoop.hive.common.type.Timestamp;
import org.apache.hadoop.hive.conf.HiveConf;
@@ -165,6 +166,17 @@ public class HiveShims {
return new SimpleGenericUDAFParameterInfo(arguments, false, distinct,
allColumns);
}
+ public static class DateShim {
+
+ public static Date cast(Object ts) {
+ return (Date) ts;
+ }
+
+ public static long millisFromDate(Object ts) {
+ return cast(ts).toEpochMilli();
+ }
+ }
+
public static class TimestampShim {
public static Timestamp cast(Object ts) {
@@ -191,4 +203,4 @@ public class HiveShims {
}
}
-}
\ No newline at end of file
+}
Modified: pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java?rev=1906696&r1=1906695&r2=1906696&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/hive/HiveUtils.java Mon Jan 16
01:50:52 2023
@@ -184,8 +184,7 @@ public class HiveUtils {
result = new
DateTime(HiveShims.TimestampShim.millisFromTimestamp(poi.getPrimitiveJavaObject(obj)));
break;
case DATE:
- java.sql.Date origDate =
(java.sql.Date)poi.getPrimitiveJavaObject(obj);
- result = new DateTime(origDate.getTime());
+ result = new
DateTime(HiveShims.DateShim.millisFromDate(poi.getPrimitiveJavaObject(obj)));
break;
case DECIMAL:
org.apache.hadoop.hive.common.type.HiveDecimal origDecimal =
Modified: pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java?rev=1906696&r1=1906695&r2=1906696&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestOrcStorage.java Mon Jan 16
01:50:52 2023
@@ -183,6 +183,20 @@ public class TestOrcStorage {
}
assertEquals(count, 10000);
}
+
+ @Test
+ public void testDate() throws Exception {
+ pigServer.registerQuery("A = load '" + basedir + "datetype.orc'" + "
using OrcStorage();" );
+ Schema schema = pigServer.dumpSchema("A");
+ assertEquals(schema.size(), 1);
+ assertEquals(schema.getField(0).type, DataType.DATETIME);
+ Iterator<Tuple> iter = pigServer.openIterator("A");
+ Tuple t = iter.next();
+ assertTrue(t.get(0) instanceof DateTime);
+ assertEquals(t.toString(), "(2022-07-07T00:00:00.000Z)");
+ assertEquals(iter.next().toString(), "(2022-09-09T00:00:00.000Z)");
+ assertFalse(iter.hasNext());
+ }
@Test
// See PIG-4218
Added: pig/trunk/test/org/apache/pig/builtin/orc/datetype.orc
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/orc/datetype.orc?rev=1906696&view=auto
==============================================================================
Binary file - no diff available.
Propchange: pig/trunk/test/org/apache/pig/builtin/orc/datetype.orc
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
