HIVE-16744 : LLAP index update may be broken after ORC switch (Sergey Shelukhin, reviewed by Prasanth Jayachandran)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/019ad8c8 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/019ad8c8 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/019ad8c8 Branch: refs/heads/hive-14535 Commit: 019ad8c822786b1bf195bcf0d4be94eec0449dda Parents: 788d486 Author: sergey <[email protected]> Authored: Fri Jun 2 13:09:08 2017 -0700 Committer: sergey <[email protected]> Committed: Fri Jun 2 13:17:01 2017 -0700 ---------------------------------------------------------------------- .../apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java | 2 +- .../apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/019ad8c8/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java index 9693826..7081140 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/encoded/OrcEncodedDataReader.java @@ -394,7 +394,7 @@ public class OrcEncodedDataReader extends CallableWithNdc<Void> LlapIoImpl.ORC_LOGGER.trace("Updating indexes in stripe {} metadata for includes: {}", stripeKey.stripeIx, DebugUtils.toString(globalIncludes)); } - assert isFoundInCache; + assert isFoundInCache; // If it's not fresh from the cache, indexes should be present. counters.incrCounter(LlapIOCounters.METADATA_CACHE_MISS); ensureMetadataReader(); updateLoadedIndexes(stripeMetadata, stripe, globalIncludes, sargColumns); http://git-wip-us.apache.org/repos/asf/hive/blob/019ad8c8/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java ---------------------------------------------------------------------- diff --git a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java index 1f3f7ea..96c4cce 100644 --- a/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java +++ b/llap-server/src/java/org/apache/hadoop/hive/llap/io/metadata/OrcStripeMetadata.java @@ -113,7 +113,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer implements ConsumerSt public void loadMissingIndexes(DataReader mr, StripeInformation stripe, boolean[] includes, boolean[] sargColumns) throws IOException { - // Do not loose the old indexes. Create a super set includes + // Do not lose the old indexes. Create a super set includes OrcProto.RowIndex[] existing = getRowIndexes(); boolean superset[] = new boolean[Math.max(existing.length, includes.length)]; for (int i = 0; i < includes.length; i++) { @@ -123,7 +123,7 @@ public class OrcStripeMetadata extends LlapCacheableBuffer implements ConsumerSt superset[i] = superset[i] || (existing[i] != null); } // TODO: should we save footer to avoid a read here? - rowIndex = mr.readRowIndex(stripe, schema, null, true, includes, + rowIndex = mr.readRowIndex(stripe, schema, null, true, superset, rowIndex.getRowGroupIndex(), sargColumns, writerVersion, rowIndex.getBloomFilterKinds(), rowIndex.getBloomFilterIndex());
