Repository: nifi Updated Branches: refs/heads/master e62aa0252 -> b4894c557
NIFI-5475: Upgraded Hive 3 bundle to Apache Hive 3.1.0 Project: http://git-wip-us.apache.org/repos/asf/nifi/repo Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/b4894c55 Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/b4894c55 Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/b4894c55 Branch: refs/heads/master Commit: b4894c5572fd81713de57d866e13bd72130c5157 Parents: e62aa02 Author: Matthew Burgess <[email protected]> Authored: Tue Jul 31 12:51:46 2018 -0400 Committer: Matt Gilman <[email protected]> Committed: Wed Aug 1 13:42:31 2018 -0400 ---------------------------------------------------------------------- .../apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java | 14 ++++++++++---- .../org/apache/nifi/processors/orc/PutORCTest.java | 8 ++++---- nifi-nar-bundles/nifi-hive-bundle/pom.xml | 5 ++--- 3 files changed, 16 insertions(+), 11 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/nifi/blob/b4894c55/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java index 7231421..2e6d2ca 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/main/java/org/apache/hadoop/hive/ql/io/orc/NiFiOrcUtils.java @@ -24,8 +24,8 @@ import org.apache.avro.util.Utf8; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hive.serde2.io.DateWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector; import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory; import org.apache.hadoop.hive.serde2.objectinspector.SettableStructObjectInspector; @@ -113,10 +113,16 @@ public class NiFiOrcUtils { return new BytesWritable(((ByteBuffer) o).array()); } if (o instanceof Timestamp) { - return new TimestampWritable((Timestamp) o); + Timestamp t = (Timestamp) o; + org.apache.hadoop.hive.common.type.Timestamp timestamp = new org.apache.hadoop.hive.common.type.Timestamp(); + timestamp.setTimeInMillis(t.getTime(), t.getNanos()); + return new TimestampWritableV2(timestamp); } if (o instanceof Date) { - return new DateWritable((Date) o); + Date d = (Date) o; + org.apache.hadoop.hive.common.type.Date date = new org.apache.hadoop.hive.common.type.Date(); + date.setTimeInMillis(d.getTime()); + return new DateWritableV2(date); } if (o instanceof Object[]) { Object[] objArray = (Object[]) o; http://git-wip-us.apache.org/repos/asf/nifi/blob/b4894c55/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java index e1af5a1..552544f 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java +++ b/nifi-nar-bundles/nifi-hive-bundle/nifi-hive3-processors/src/test/java/org/apache/nifi/processors/orc/PutORCTest.java @@ -25,9 +25,9 @@ import org.apache.hadoop.hive.ql.io.orc.OrcFile; import org.apache.hadoop.hive.ql.io.orc.OrcStruct; import org.apache.hadoop.hive.ql.io.orc.Reader; import org.apache.hadoop.hive.ql.io.orc.RecordReader; -import org.apache.hadoop.hive.serde2.io.DateWritable; +import org.apache.hadoop.hive.serde2.io.DateWritableV2; import org.apache.hadoop.hive.serde2.io.DoubleWritable; -import org.apache.hadoop.hive.serde2.io.TimestampWritable; +import org.apache.hadoop.hive.serde2.io.TimestampWritableV2; import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo; import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils; @@ -246,8 +246,8 @@ public class PutORCTest { verifyORCUsers(orcFile, 10, (x, currUser) -> { assertEquals((int) currUser, ((IntWritable) x.get(0)).get()); assertEquals(timeMillis, ((IntWritable) x.get(1)).get()); - assertEquals(timestampMillis, ((TimestampWritable) x.get(2)).getTimestamp()); - assertEquals(dt, ((DateWritable) x.get(3)).get()); + assertEquals(timestampMillis, ((TimestampWritableV2) x.get(2)).getTimestamp().toSqlTimestamp()); + assertEquals(dt.toLocalDate().toEpochDay(), ((DateWritableV2) x.get(3)).get().toEpochDay()); assertEquals(dec, ((DoubleWritable) x.get(4)).get(), Double.MIN_VALUE); return null; } http://git-wip-us.apache.org/repos/asf/nifi/blob/b4894c55/nifi-nar-bundles/nifi-hive-bundle/pom.xml ---------------------------------------------------------------------- diff --git a/nifi-nar-bundles/nifi-hive-bundle/pom.xml b/nifi-nar-bundles/nifi-hive-bundle/pom.xml index 2bb421b..c67a267 100644 --- a/nifi-nar-bundles/nifi-hive-bundle/pom.xml +++ b/nifi-nar-bundles/nifi-hive-bundle/pom.xml @@ -38,9 +38,8 @@ <properties> <hive.version>1.2.1</hive.version> <hive.hadoop.version>2.6.2</hive.hadoop.version> - <hive3.version>3.0.0</hive3.version> - <hive3.hadoop.version>3.0.0</hive3.hadoop.version> - <orc.version>1.4.3</orc.version> + <hive3.version>3.1.0</hive3.version> + <hive3.hadoop.version>3.1.0</hive3.hadoop.version> </properties> <build>
