Repository: hive Updated Branches: refs/heads/master bef6c9fd2 -> dfc926284
HIVE-20604: Minor compaction disables ORC column stats (Eugene Koifman, reviewed by Prasanth Jayachandran) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/dfc92628 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/dfc92628 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/dfc92628 Branch: refs/heads/master Commit: dfc9262842c7405543a7f92efb18a6eead92414e Parents: bef6c9f Author: Eugene Koifman <[email protected]> Authored: Wed Sep 26 11:31:14 2018 -0700 Committer: Eugene Koifman <[email protected]> Committed: Wed Sep 26 11:31:14 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java | 9 ++++++--- .../apache/hadoop/hive/metastore/conf/MetastoreConf.java | 4 ++++ 2 files changed, 10 insertions(+), 3 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/dfc92628/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java index 835f0e8..c4c56f8 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcOutputFormat.java @@ -24,6 +24,7 @@ import java.util.Arrays; import java.util.List; import java.util.Properties; +import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.orc.CompressionKind; import org.apache.orc.TypeDescription; import org.slf4j.Logger; @@ -288,9 +289,11 @@ public class OrcOutputFormat extends FileOutputFormat<NullWritable, OrcSerdeRow> if (!options.isWritingBase()) { opts.bufferSize(OrcRecordUpdater.DELTA_BUFFER_SIZE) .stripeSize(OrcRecordUpdater.DELTA_STRIPE_SIZE) - .blockPadding(false) - .compress(CompressionKind.NONE) - .rowIndexStride(0); + .blockPadding(false); + if(!MetastoreConf.getBoolVar(options.getConfiguration(), + MetastoreConf.ConfVars.COMPACTOR_MINOR_STATS_COMPRESSION)) { + opts.compress(CompressionKind.NONE).rowIndexStride(0); + } } final OrcRecordUpdater.KeyIndexBuilder watcher = new OrcRecordUpdater.KeyIndexBuilder("compactor"); http://git-wip-us.apache.org/repos/asf/hive/blob/dfc92628/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java index 30ea7f8..946f644 100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/MetastoreConf.java @@ -392,6 +392,10 @@ public class MetastoreConf { "tables or partitions to be compacted once they are determined to need compaction.\n" + "It will also increase the background load on the Hadoop cluster as more MapReduce jobs\n" + "will be running in the background."), + COMPACTOR_MINOR_STATS_COMPRESSION( + "metastore.compactor.enable.stats.compression", + "metastore.compactor.enable.stats.compression", true, + "Can be used to disable compression and ORC indexes for files produced by minor compaction."), CONNECTION_DRIVER("javax.jdo.option.ConnectionDriverName", "javax.jdo.option.ConnectionDriverName", "org.apache.derby.jdbc.EmbeddedDriver", "Driver class name for a JDBC metastore"),
