This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-1.6
in repository https://gitbox.apache.org/repos/asf/orc.git
The following commit(s) were added to refs/heads/branch-1.6 by this push:
new d78cc39 ORC-671. Add OrcTail.getStripeStatistics back for backward
compatiblility
d78cc39 is described below
commit d78cc39a9299b62bc8a5d8f5c3fac9201e03cb8b
Author: Dongjoon Hyun <[email protected]>
AuthorDate: Mon Oct 5 00:26:21 2020 -0700
ORC-671. Add OrcTail.getStripeStatistics back for backward compatiblility
Fixes #549
Signed-off-by: Owen O'Malley <[email protected]>
(cherry picked from commit a95926fbcc3986dade353b480f8bda86dc2791c3)
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../core/src/java/org/apache/orc/impl/OrcTail.java | 31 +++++++++++++++++++++-
.../src/java/org/apache/orc/impl/ReaderImpl.java | 4 +--
.../test/org/apache/orc/impl/TestReaderImpl.java | 23 ++++++++++++++++
3 files changed, 55 insertions(+), 3 deletions(-)
diff --git a/java/core/src/java/org/apache/orc/impl/OrcTail.java
b/java/core/src/java/org/apache/orc/impl/OrcTail.java
index 409d8f3..93eef33 100644
--- a/java/core/src/java/org/apache/orc/impl/OrcTail.java
+++ b/java/core/src/java/org/apache/orc/impl/OrcTail.java
@@ -18,17 +18,24 @@ package org.apache.orc.impl;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.ArrayList;
import java.util.List;
import org.apache.orc.CompressionKind;
import org.apache.orc.OrcFile;
import org.apache.orc.OrcProto;
import org.apache.orc.OrcUtils;
+import org.apache.orc.Reader;
import org.apache.orc.StripeInformation;
+import org.apache.orc.StripeStatistics;
import org.apache.orc.TypeDescription;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
// TODO: Make OrcTail implement FileMetadata or Reader interface
public final class OrcTail {
+ private static final Logger LOG = LoggerFactory.getLogger(OrcTail.class);
+
// postscript + footer - Serialized in OrcSplit
private final OrcProto.FileTail fileTail;
// serialized representation of metadata, footer and postscript
@@ -36,6 +43,7 @@ public final class OrcTail {
private final TypeDescription schema;
// used to invalidate cache entries
private final long fileModificationTime;
+ private final Reader reader;
public OrcTail(OrcProto.FileTail fileTail,
ByteBuffer serializedTail) throws IOException {
@@ -50,13 +58,19 @@ public final class OrcTail {
}
public OrcTail(OrcProto.FileTail fileTail, BufferChunk serializedTail,
- long fileModificationTime) throws IOException {
+ long fileModificationTime) throws IOException {
+ this(fileTail, serializedTail, fileModificationTime, null);
+ }
+
+ public OrcTail(OrcProto.FileTail fileTail, BufferChunk serializedTail,
+ long fileModificationTime, Reader reader) throws IOException {
this.fileTail = fileTail;
this.serializedTail = serializedTail;
this.fileModificationTime = fileModificationTime;
List<OrcProto.Type> types = getTypes();
OrcUtils.isValidTypeTree(types, 0);
this.schema = OrcUtils.convertTypeFromProtobuf(types, 0);
+ this.reader = reader;
}
public ByteBuffer getSerializedTail() {
@@ -185,4 +199,19 @@ public final class OrcTail {
fileTailBuilder.setFooter(footerBuilder.build());
return fileTailBuilder.build();
}
+
+ /**
+ * Get the stripe statistics from the file tail.
+ * This code is for compatibility with ORC 1.5.
+ * @return the stripe statistics
+ * @deprecated the user should use Reader.getStripeStatistics instead.
+ */
+ public List<StripeStatistics> getStripeStatistics() throws IOException {
+ if (reader == null) {
+ LOG.warn("Please use Reader.getStripeStatistics or give `Reader` to
OrcTail constructor.");
+ return new ArrayList<>();
+ } else {
+ return reader.getStripeStatistics();
+ }
+ }
}
diff --git a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
index 1b8d47e..e062239 100644
--- a/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
+++ b/java/core/src/java/org/apache/orc/impl/ReaderImpl.java
@@ -626,7 +626,7 @@ public class ReaderImpl implements Reader {
result.setPostscript(postscript);
result.setFileLength(0);
result.setPostscriptLength(0);
- return new OrcTail(result.build(), new BufferChunk(0, 0), -1);
+ return new OrcTail(result.build(), new BufferChunk(0, 0), -1, this);
}
private static void read(FSDataInputStream file,
@@ -727,7 +727,7 @@ public class ReaderImpl implements Reader {
new IOException("Problem reading file footer " + path, thr);
}
- return new OrcTail(fileTailBuilder.build(), buffer, modificationTime);
+ return new OrcTail(fileTailBuilder.build(), buffer, modificationTime,
this);
}
@Override
diff --git a/java/core/src/test/org/apache/orc/impl/TestReaderImpl.java
b/java/core/src/test/org/apache/orc/impl/TestReaderImpl.java
index 465d52e..5e8c9cf 100644
--- a/java/core/src/test/org/apache/orc/impl/TestReaderImpl.java
+++ b/java/core/src/test/org/apache/orc/impl/TestReaderImpl.java
@@ -20,6 +20,7 @@ import static org.junit.Assert.assertTrue;
import java.io.ByteArrayInputStream;
import java.io.EOFException;
+import java.io.File;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
@@ -42,15 +43,20 @@ import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.Progressable;
import org.apache.orc.FileFormatException;
import org.apache.orc.OrcFile;
+import org.apache.orc.OrcProto;
import org.apache.orc.Reader;
import org.apache.orc.RecordReader;
+import org.apache.orc.StripeStatistics;
import org.apache.orc.TestVectorOrcFile;
+import org.apache.orc.TimestampColumnStatistics;
import org.junit.Before;
import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
public class TestReaderImpl {
+ private Path workDir = new Path(System.getProperty("example.dir",
+ "../../examples/"));
@Rule
public ExpectedException thrown = ExpectedException.none();
@@ -345,4 +351,21 @@ public class TestReaderImpl {
reader2.close();
assertEquals(0, fs.streamCount());
}
+
+ @Test
+ public void testOrcTailStripeStats() throws Exception {
+ Configuration conf = new Configuration();
+ Path path = new Path(workDir, "orc_split_elim_new.orc");
+ FileSystem fs = path.getFileSystem(conf);
+ try (ReaderImpl reader = (ReaderImpl) OrcFile.createReader(path,
+ OrcFile.readerOptions(conf).filesystem(fs))) {
+ OrcTail tail = reader.extractFileTail(fs, path, Long.MAX_VALUE);
+ List<StripeStatistics> stats = tail.getStripeStatistics();
+ assertEquals(1, stats.size());
+ OrcProto.TimestampStatistics tsStats =
+ stats.get(0).getColumn(5).getTimestampStatistics();
+ assertEquals(-28800000, tsStats.getMinimumUtc());
+ assertEquals(-28550000, tsStats.getMaximumUtc());
+ }
+ }
}
