Repository: hive Updated Branches: refs/heads/master 8141a4e63 -> 9e4803cd3
HIVE-10651: ORC file footer cache should be bounded (Prasanth Jayachandran reviewed by Sergey Shelukhin) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/9e4803cd Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/9e4803cd Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/9e4803cd Branch: refs/heads/master Commit: 9e4803cd373ccf4ba0df484bc4eefad8c0685514 Parents: 8141a4e Author: Prasanth Jayachandran <[email protected]> Authored: Mon May 11 13:07:16 2015 -0700 Committer: Prasanth Jayachandran <[email protected]> Committed: Mon May 11 13:07:16 2015 -0700 ---------------------------------------------------------------------- common/src/java/org/apache/hadoop/hive/conf/HiveConf.java | 2 +- .../org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java | 8 ++++++-- 2 files changed, 7 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/9e4803cd/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java ---------------------------------------------------------------------- diff --git a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java index 115a5d0..eff4d30 100644 --- a/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java +++ b/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java @@ -1022,7 +1022,7 @@ public class HiveConf extends Configuration { "If turned on splits generated by orc will include metadata about the stripes in the file. This\n" + "data is read remotely (from the client or HS2 machine) and sent to all the tasks."), HIVE_ORC_CACHE_STRIPE_DETAILS_SIZE("hive.orc.cache.stripe.details.size", 10000, - "Cache size for keeping meta info about orc splits cached in the client."), + "Max cache size for keeping meta info about orc splits cached in the client."), HIVE_ORC_COMPUTE_SPLITS_NUM_THREADS("hive.orc.compute.splits.num.threads", 10, "How many threads orc should use to create splits in parallel."), HIVE_ORC_SKIP_CORRUPT_DATA("hive.exec.orc.skip.corrupt.data", false, http://git-wip-us.apache.org/repos/asf/hive/blob/9e4803cd/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java index 3ae4688..5d6c9da 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcInputFormat.java @@ -412,8 +412,12 @@ public class OrcInputFormat implements InputFormat<NullWritable, OrcStruct>, } if (footerCache == null && cacheStripeDetails) { - footerCache = CacheBuilder.newBuilder().concurrencyLevel(numThreads) - .initialCapacity(cacheStripeDetailsSize).softValues().build(); + footerCache = CacheBuilder.newBuilder() + .concurrencyLevel(numThreads) + .initialCapacity(cacheStripeDetailsSize) + .maximumSize(cacheStripeDetailsSize) + .softValues() + .build(); } } String value = conf.get(ValidTxnList.VALID_TXNS_KEY,
