Repository: orc Updated Branches: refs/heads/master 377c63c9d -> 32787e852
ORC-55. Resynchronize storage-api and orc with the hive trunk. (omalley reviewed by Sergey) Signed-off-by: Owen O'Malley <[email protected]> Fixes #25 Project: http://git-wip-us.apache.org/repos/asf/orc/repo Commit: http://git-wip-us.apache.org/repos/asf/orc/commit/32787e85 Tree: http://git-wip-us.apache.org/repos/asf/orc/tree/32787e85 Diff: http://git-wip-us.apache.org/repos/asf/orc/diff/32787e85 Branch: refs/heads/master Commit: 32787e8520e3399e2aafe4de115867a832ae654e Parents: 377c63c Author: Owen O'Malley <[email protected]> Authored: Fri May 20 16:50:01 2016 -0700 Committer: Owen O'Malley <[email protected]> Committed: Tue May 24 10:20:52 2016 -0700 ---------------------------------------------------------------------- java/core/pom.xml | 2 +- .../apache/orc/impl/TestRecordReaderImpl.java | 4 +- java/mapreduce/pom.xml | 5 -- java/storage-api/pom.xml | 4 +- .../hadoop/hive/common/type/RandomTypeUtil.java | 95 ++++++++++++++++++++ 5 files changed, 100 insertions(+), 10 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/core/pom.xml ---------------------------------------------------------------------- diff --git a/java/core/pom.xml b/java/core/pom.xml index 85d9ae6..c3efa0b 100644 --- a/java/core/pom.xml +++ b/java/core/pom.xml @@ -31,7 +31,7 @@ <dependency> <groupId>org.apache.hive</groupId> <artifactId>hive-storage-api</artifactId> - <version>2.0.0.1-SNAPSHOT</version> + <version>2.0.2-pre-orc</version> </dependency> <!-- inter-project --> http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java ---------------------------------------------------------------------- diff --git a/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java b/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java index a55b378..cdd62ac 100644 --- a/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java +++ b/java/core/src/test/org/apache/orc/impl/TestRecordReaderImpl.java @@ -1676,10 +1676,8 @@ public class TestRecordReaderImpl { private void closeMockedRecordReader(DataReader mockedDataReader) throws IOException { Configuration conf = new Configuration(); - FileSystem fs = FileSystem.getLocal(conf).getRaw(); - fs.delete(workDir, true); - fs.mkdirs(workDir); Path path = new Path(workDir, "empty.orc"); + FileSystem.get(conf).delete(path, true); Writer writer = OrcFile.createWriter(path, OrcFile.writerOptions(conf) .setSchema(TypeDescription.createLong())); writer.close(); http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/mapreduce/pom.xml ---------------------------------------------------------------------- diff --git a/java/mapreduce/pom.xml b/java/mapreduce/pom.xml index 8792e1c..cbb625f 100644 --- a/java/mapreduce/pom.xml +++ b/java/mapreduce/pom.xml @@ -29,11 +29,6 @@ <dependencies> <dependency> - <groupId>org.apache.hive</groupId> - <artifactId>hive-storage-api</artifactId> - <version>2.0.0.1-SNAPSHOT</version> - </dependency> - <dependency> <groupId>org.apache.orc</groupId> <artifactId>orc-core</artifactId> <version>1.1.0-SNAPSHOT</version> http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/storage-api/pom.xml ---------------------------------------------------------------------- diff --git a/java/storage-api/pom.xml b/java/storage-api/pom.xml index 85bd77c..7636446 100644 --- a/java/storage-api/pom.xml +++ b/java/storage-api/pom.xml @@ -19,7 +19,9 @@ <groupId>org.apache.hive</groupId> <artifactId>hive-storage-api</artifactId> - <version>2.0.0.1-SNAPSHOT</version> + <!-- remove our custom version of storage-api once we get the changes + released as hive 2.0.2 --> + <version>2.0.2-pre-orc</version> <packaging>jar</packaging> <name>Hive Storage API</name> http://git-wip-us.apache.org/repos/asf/orc/blob/32787e85/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java ---------------------------------------------------------------------- diff --git a/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java b/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java index 3fb0cfd..53a7823 100644 --- a/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java +++ b/java/storage-api/src/java/org/apache/hadoop/hive/common/type/RandomTypeUtil.java @@ -17,6 +17,7 @@ */ package org.apache.hadoop.hive.common.type; +import java.sql.Date; import java.sql.Timestamp; import java.text.DateFormat; import java.text.ParseException; @@ -26,6 +27,100 @@ import java.util.concurrent.TimeUnit; public class RandomTypeUtil { + public static String getRandString(Random r) { + return getRandString(r, null, r.nextInt(10)); + } + + public static String getRandString(Random r, String characters, int length) { + if (characters == null) { + characters = "ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + + } + StringBuilder sb = new StringBuilder(); + for (int i = 0; i < length; i++) { + if (characters == null) { + sb.append((char) (r.nextInt(128))); + } else { + sb.append(characters.charAt(r.nextInt(characters.length()))); + } + } + return sb.toString(); + } + + public static byte[] getRandBinary(Random r, int len){ + byte[] bytes = new byte[len]; + for (int j = 0; j < len; j++){ + bytes[j] = Byte.valueOf((byte) r.nextInt()); + } + return bytes; + } + + private static final String DECIMAL_CHARS = "0123456789"; + + public static class HiveDecimalAndPrecisionScale { + public HiveDecimal hiveDecimal; + public int precision; + public int scale; + + HiveDecimalAndPrecisionScale(HiveDecimal hiveDecimal, int precision, int scale) { + this.hiveDecimal = hiveDecimal; + this.precision = precision; + this.scale = scale; + } + } + + public static HiveDecimalAndPrecisionScale getRandHiveDecimal(Random r) { + int precision; + int scale; + while (true) { + StringBuilder sb = new StringBuilder(); + precision = 1 + r.nextInt(18); + scale = 0 + r.nextInt(precision + 1); + + int integerDigits = precision - scale; + + if (r.nextBoolean()) { + sb.append("-"); + } + + if (integerDigits == 0) { + sb.append("0"); + } else { + sb.append(getRandString(r, DECIMAL_CHARS, integerDigits)); + } + if (scale != 0) { + sb.append("."); + sb.append(getRandString(r, DECIMAL_CHARS, scale)); + } + + HiveDecimal bd = HiveDecimal.create(sb.toString()); + precision = bd.precision(); + scale = bd.scale(); + if (scale > precision) { + // Sometimes weird decimals are produced? + continue; + } + + // For now, punt. + precision = HiveDecimal.SYSTEM_DEFAULT_PRECISION; + scale = HiveDecimal.SYSTEM_DEFAULT_SCALE; + return new HiveDecimalAndPrecisionScale(bd, precision, scale); + } + } + + public static Date getRandDate(Random r) { + String dateStr = String.format("%d-%02d-%02d", + Integer.valueOf(1800 + r.nextInt(500)), // year + Integer.valueOf(1 + r.nextInt(12)), // month + Integer.valueOf(1 + r.nextInt(28))); // day + Date dateVal = Date.valueOf(dateStr); + return dateVal; + } + + /** + * TIMESTAMP. + */ + public static final long NANOSECONDS_PER_SECOND = TimeUnit.SECONDS.toNanos(1); public static final long MILLISECONDS_PER_SECOND = TimeUnit.SECONDS.toMillis(1); public static final long NANOSECONDS_PER_MILLISSECOND = TimeUnit.MILLISECONDS.toNanos(1);
