HIVE-17833: Publish split generation counters (Prasanth Jayachandran reviewed by Sergey Shelukhin)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/fff86f3a Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/fff86f3a Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/fff86f3a Branch: refs/heads/master Commit: fff86f3a6d03fdabfba3c1dc8d3a04c48be93aa8 Parents: 525f8ab Author: Prasanth Jayachandran <prasan...@apache.org> Authored: Tue Jan 23 10:52:41 2018 -0800 Committer: Prasanth Jayachandran <prasan...@apache.org> Committed: Tue Jan 23 10:52:41 2018 -0800 ---------------------------------------------------------------------- .../hive/jdbc/TestTriggersNoTezSessionPool.java | 24 +- .../jdbc/TestTriggersTezSessionPoolManager.java | 60 +- .../test/resources/testconfiguration.properties | 1 + .../hadoop/hive/ql/exec/ReduceSinkOperator.java | 12 +- .../apache/hadoop/hive/ql/exec/Utilities.java | 14 + .../hadoop/hive/ql/exec/spark/SparkTask.java | 3 +- .../hive/ql/exec/tez/HiveInputCounters.java | 29 + .../hive/ql/exec/tez/HiveSplitGenerator.java | 61 +- .../ql/exec/tez/monitoring/TezJobMonitor.java | 108 +- .../ql/hooks/PostExecTezSummaryPrinter.java | 7 + .../hadoop/hive/ql/wm/VertexCounterLimit.java | 11 +- .../apache/hadoop/hive/ql/wm/TestTrigger.java | 24 +- .../queries/clientpositive/tez_input_counters.q | 25 + .../clientpositive/llap/dp_counter_mm.q.out | 48 + .../clientpositive/llap/dp_counter_non_mm.q.out | 48 + .../clientpositive/llap/orc_llap_counters.q.out | 210 ++ .../llap/orc_llap_counters1.q.out | 10 + .../clientpositive/llap/orc_ppd_basic.q.out | 230 ++ .../llap/orc_ppd_schema_evol_3a.q.out | 250 ++ .../llap/tez_input_counters.q.out | 2303 ++++++++++++++++++ 20 files changed, 3439 insertions(+), 39 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersNoTezSessionPool.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersNoTezSessionPool.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersNoTezSessionPool.java index e9f6718..2117b68 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersNoTezSessionPool.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersNoTezSessionPool.java @@ -44,8 +44,28 @@ public class TestTriggersNoTezSessionPool extends AbstractJdbcTriggersTest { } @Test(timeout = 60000) - public void testTriggerTotalTasks() throws Exception { - Expression expression = ExpressionFactory.fromString("TOTAL_TASKS > 50"); + public void testTriggerVertexTotalTasks() throws Exception { + Expression expression = ExpressionFactory.fromString("VERTEX_TOTAL_TASKS > 50"); + Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); + setupTriggers(Lists.newArrayList(trigger)); + String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + + " t2 on t1.under_col>=t2.under_col"; + runQueryWithTrigger(query, getConfigs(), trigger + " violated"); + } + + @Test(timeout = 60000) + public void testTriggerDAGTotalTasks() throws Exception { + Expression expression = ExpressionFactory.fromString("DAG_TOTAL_TASKS > 50"); + Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); + setupTriggers(Lists.newArrayList(trigger)); + String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + + " t2 on t1.under_col>=t2.under_col"; + runQueryWithTrigger(query, getConfigs(), trigger + " violated"); + } + + @Test(timeout = 60000) + public void testTriggerTotalLaunchedTasks() throws Exception { + Expression expression = ExpressionFactory.fromString("TOTAL_LAUNCHED_TASKS > 50"); Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); setupTriggers(Lists.newArrayList(trigger)); String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java ---------------------------------------------------------------------- diff --git a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java index 3b6eb71..b2bb443 100644 --- a/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java +++ b/itests/hive-unit/src/test/java/org/apache/hive/jdbc/TestTriggersTezSessionPoolManager.java @@ -91,7 +91,17 @@ public class TestTriggersTezSessionPoolManager extends AbstractJdbcTriggersTest @Test(timeout = 60000) public void testTriggerTotalTasks() throws Exception { - Expression expression = ExpressionFactory.fromString("TOTAL_TASKS > 50"); + Expression expression = ExpressionFactory.fromString("VERTEX_TOTAL_TASKS > 50"); + Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); + setupTriggers(Lists.newArrayList(trigger)); + String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + + " t2 on t1.under_col>=t2.under_col"; + runQueryWithTrigger(query, getConfigs(), trigger + " violated"); + } + + @Test(timeout = 60000) + public void testTriggerDagTotalTasks() throws Exception { + Expression expression = ExpressionFactory.fromString("DAG_TOTAL_TASKS > 50"); Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); setupTriggers(Lists.newArrayList(trigger)); String query = "select sleep(t1.under_col, 5), t1.value from " + tableName + " t1 join " + tableName + @@ -219,6 +229,54 @@ public class TestTriggersTezSessionPoolManager extends AbstractJdbcTriggersTest } @Test(timeout = 60000) + public void testTriggerDagRawInputSplitsKill() throws Exception { + // Map 1 - 55 splits + // Map 3 - 55 splits + Expression expression = ExpressionFactory.fromString("DAG_RAW_INPUT_SPLITS > 100"); + Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); + setupTriggers(Lists.newArrayList(trigger)); + String query = "select t1.under_col, t1.value from " + tableName + " t1 join " + tableName + + " t2 on t1.under_col>=t2.under_col"; + runQueryWithTrigger(query, getConfigs(), "Query was cancelled"); + } + + @Test(timeout = 60000) + public void testTriggerVertexRawInputSplitsNoKill() throws Exception { + // Map 1 - 55 splits + // Map 3 - 55 splits + Expression expression = ExpressionFactory.fromString("VERTEX_RAW_INPUT_SPLITS > 100"); + Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); + setupTriggers(Lists.newArrayList(trigger)); + String query = "select t1.under_col, t1.value from " + tableName + " t1 join " + tableName + + " t2 on t1.under_col>=t2.under_col"; + runQueryWithTrigger(query, getConfigs(), null); + } + + @Test(timeout = 60000) + public void testTriggerVertexRawInputSplitsKill() throws Exception { + // Map 1 - 55 splits + // Map 3 - 55 splits + Expression expression = ExpressionFactory.fromString("VERTEX_RAW_INPUT_SPLITS > 50"); + Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); + setupTriggers(Lists.newArrayList(trigger)); + String query = "select t1.under_col, t1.value from " + tableName + " t1 join " + tableName + + " t2 on t1.under_col>=t2.under_col"; + runQueryWithTrigger(query, getConfigs(), "Query was cancelled"); + } + + @Test(timeout = 60000) + public void testTriggerDefaultRawInputSplits() throws Exception { + // Map 1 - 55 splits + // Map 3 - 55 splits + Expression expression = ExpressionFactory.fromString("RAW_INPUT_SPLITS > 50"); + Trigger trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); + setupTriggers(Lists.newArrayList(trigger)); + String query = "select t1.under_col, t1.value from " + tableName + " t1 join " + tableName + + " t2 on t1.under_col>=t2.under_col"; + runQueryWithTrigger(query, getConfigs(), "Query was cancelled"); + } + + @Test(timeout = 60000) public void testMultipleTriggers1() throws Exception { Expression shuffleExpression = ExpressionFactory.fromString("HDFS_BYTES_READ > 1000000"); Trigger shuffleTrigger = new ExecutionTrigger("big_shuffle", shuffleExpression, new Action(Action.Type.KILL_QUERY)); http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/itests/src/test/resources/testconfiguration.properties ---------------------------------------------------------------------- diff --git a/itests/src/test/resources/testconfiguration.properties b/itests/src/test/resources/testconfiguration.properties index 53da72b..1017249 100644 --- a/itests/src/test/resources/testconfiguration.properties +++ b/itests/src/test/resources/testconfiguration.properties @@ -586,6 +586,7 @@ minillaplocal.query.files=\ orc_analyze.q,\ orc_llap_nonvector.q,\ orc_ppd_date.q,\ + tez_input_counters.q,\ orc_ppd_decimal.q,\ orc_ppd_timestamp.q,\ order_null.q,\ http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java index 0e63031..395a5f4 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/ReduceSinkOperator.java @@ -163,7 +163,8 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc> cntr = 1; logEveryNRows = HiveConf.getLongVar(hconf, HiveConf.ConfVars.HIVE_LOG_N_RECORDS); - statsMap.put(getCounterName(Counter.RECORDS_OUT_INTERMEDIATE, hconf), recordCounter); + final String vertexName = hconf.get(Operator.CONTEXT_NAME_KEY, ""); + statsMap.put(Utilities.getVertexCounterName(Counter.RECORDS_OUT_INTERMEDIATE.name(), vertexName), recordCounter); List<ExprNodeDesc> keys = conf.getKeyCols(); @@ -248,15 +249,6 @@ public class ReduceSinkOperator extends TerminalOperator<ReduceSinkDesc> } } - public String getCounterName(Counter counter, Configuration hconf) { - String context = hconf.get(Operator.CONTEXT_NAME_KEY, ""); - if (context != null && !context.isEmpty()) { - context = "_" + context.replace(" ", "_"); - } - return counter + context; - } - - /** * Initializes array of ExprNodeEvaluator. Adds Union field for distinct * column indices for group by. http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java index 4da9d57..2e1fd37 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java @@ -3246,6 +3246,20 @@ public final class Utilities { } /** + * Appends vertex name to specified counter name. + * + * @param counter counter to be appended with + * @param vertexName vertex name + * @return counter name with vertex name appended + */ + public static String getVertexCounterName(String counter, String vertexName) { + if (vertexName != null && !vertexName.isEmpty()) { + vertexName = "_" + vertexName.replace(" ", "_"); + } + return counter + vertexName; + } + + /** * Computes a list of all input paths needed to compute the given MapWork. All aliases * are considered and a merged list of input paths is returned. If any input path points * to an empty table or partition a dummy file in the scratch dir is instead created and http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java index 324ff17..c6e17b5 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/SparkTask.java @@ -374,8 +374,9 @@ public class SparkTask extends Task<SparkWork> { hiveCounters.add(((FileSinkOperator) operator).getCounterName(counter)); } } else if (operator instanceof ReduceSinkOperator) { + final String contextName = conf.get(Operator.CONTEXT_NAME_KEY, ""); for (ReduceSinkOperator.Counter counter : ReduceSinkOperator.Counter.values()) { - hiveCounters.add(((ReduceSinkOperator) operator).getCounterName(counter, conf)); + hiveCounters.add(Utilities.getVertexCounterName(counter.name(), contextName)); } } else if (operator instanceof ScriptOperator) { for (ScriptOperator.Counter counter : ScriptOperator.Counter.values()) { http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveInputCounters.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveInputCounters.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveInputCounters.java new file mode 100644 index 0000000..085d6a7 --- /dev/null +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveInputCounters.java @@ -0,0 +1,29 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.ql.exec.tez; + +import org.apache.hadoop.classification.InterfaceAudience; + +@InterfaceAudience.Private +public enum HiveInputCounters { + RAW_INPUT_SPLITS, + GROUPED_INPUT_SPLITS, + INPUT_FILES, + INPUT_DIRECTORIES +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java index 7479641..98f4bc0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java @@ -23,10 +23,16 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.BitSet; import java.util.Comparator; +import java.util.HashSet; import java.util.List; +import java.util.Set; import com.google.common.base.Preconditions; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hive.ql.exec.Operator; +import org.apache.tez.common.counters.TezCounters; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; @@ -158,7 +164,7 @@ public class HiveSplitGenerator extends InputInitializer { // Need to instantiate the realInputFormat InputFormat<?, ?> inputFormat = (InputFormat<?, ?>) ReflectionUtils.newInstance(JavaUtils.loadClass(realInputFormatName), - jobConf); + jobConf); int totalResource = 0; int taskResource = 0; @@ -178,10 +184,10 @@ public class HiveSplitGenerator extends InputInitializer { if (HiveConf.getLongVar(conf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, 1) <= 1) { // broken configuration from mapred-default.xml final long blockSize = conf.getLong(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, - DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); + DFSConfigKeys.DFS_BLOCK_SIZE_DEFAULT); final long minGrouping = conf.getLong( - TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE, - TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT); + TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE, + TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_MIN_SIZE_DEFAULT); final long preferredSplitSize = Math.min(blockSize / 2, minGrouping); HiveConf.setLongVar(jobConf, HiveConf.ConfVars.MAPREDMINSPLITSIZE, preferredSplitSize); LOG.info("The preferred split size is " + preferredSplitSize); @@ -189,15 +195,47 @@ public class HiveSplitGenerator extends InputInitializer { // Create the un-grouped splits float waves = - conf.getFloat(TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES, - TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES_DEFAULT); + conf.getFloat(TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES, + TezMapReduceSplitsGrouper.TEZ_GROUPING_SPLIT_WAVES_DEFAULT); // Raw splits InputSplit[] splits = inputFormat.getSplits(jobConf, (int) (availableSlots * waves)); // Sort the splits, so that subsequent grouping is consistent. Arrays.sort(splits, new InputSplitComparator()); LOG.info("Number of input splits: " + splits.length + ". " + availableSlots - + " available slots, " + waves + " waves. Input format is: " + realInputFormatName); + + " available slots, " + waves + " waves. Input format is: " + realInputFormatName); + + // increment/set input counters + InputInitializerContext inputInitializerContext = getContext(); + TezCounters tezCounters = null; + String counterName; + String groupName = null; + String vertexName = null; + if (inputInitializerContext != null) { + tezCounters = new TezCounters(); + groupName = HiveInputCounters.class.getName(); + vertexName = jobConf.get(Operator.CONTEXT_NAME_KEY, ""); + counterName = Utilities.getVertexCounterName(HiveInputCounters.RAW_INPUT_SPLITS.name(), vertexName); + tezCounters.findCounter(groupName, counterName).increment(splits.length); + final List<Path> paths = Utilities.getInputPathsTez(jobConf, work); + counterName = Utilities.getVertexCounterName(HiveInputCounters.INPUT_DIRECTORIES.name(), vertexName); + tezCounters.findCounter(groupName, counterName).increment(paths.size()); + final Set<String> files = new HashSet<>(); + for (InputSplit inputSplit : splits) { + if (inputSplit instanceof FileSplit) { + final FileSplit fileSplit = (FileSplit) inputSplit; + final Path path = fileSplit.getPath(); + // The assumption here is the path is a file. Only case this is different is ACID deltas. + // The isFile check is avoided here for performance reasons. + final String fileStr = path.toString(); + if (!files.contains(fileStr)) { + files.add(fileStr); + } + } + } + counterName = Utilities.getVertexCounterName(HiveInputCounters.INPUT_FILES.name(), vertexName); + tezCounters.findCounter(groupName, counterName).increment(files.size()); + } if (work.getIncludedBuckets() != null) { splits = pruneBuckets(work, splits); @@ -208,6 +246,15 @@ public class HiveSplitGenerator extends InputInitializer { // And finally return them in a flat array InputSplit[] flatSplits = groupedSplits.values().toArray(new InputSplit[0]); LOG.info("Number of split groups: " + flatSplits.length); + if (inputInitializerContext != null) { + counterName = Utilities.getVertexCounterName(HiveInputCounters.GROUPED_INPUT_SPLITS.name(), vertexName); + tezCounters.findCounter(groupName, counterName).setValue(flatSplits.length); + + if (LOG.isDebugEnabled()) { + LOG.debug("Published tez counters: " + tezCounters); + } + inputInitializerContext.addCounters(tezCounters); + } List<TaskLocationHint> locationHints = splitGrouper.createTaskLocationHints(flatSplits, generateConsistentSplits); http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java index 55e7d7d..3558475 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/monitoring/TezJobMonitor.java @@ -30,6 +30,7 @@ import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.TimeUnit; +import java.util.stream.Collectors; import org.apache.commons.lang3.exception.ExceptionUtils; import org.apache.hadoop.hive.common.log.InPlaceUpdate; @@ -44,8 +45,8 @@ import org.apache.hadoop.hive.ql.plan.BaseWork; import org.apache.hadoop.hive.ql.session.SessionState; import org.apache.hadoop.hive.ql.session.SessionState.LogHelper; import org.apache.hadoop.hive.ql.wm.TimeCounterLimit; -import org.apache.hadoop.hive.ql.wm.WmContext; import org.apache.hadoop.hive.ql.wm.VertexCounterLimit; +import org.apache.hadoop.hive.ql.wm.WmContext; import org.apache.hive.common.util.ShutdownHookManager; import org.apache.tez.common.counters.CounterGroup; import org.apache.tez.common.counters.TezCounter; @@ -169,10 +170,15 @@ public class TezJobMonitor { TezCounters dagCounters = status.getDAGCounters(); vertexProgressMap = status.getVertexProgress(); wmContext = context.getWmContext(); + List<String> vertexNames = vertexProgressMap.keySet() + .stream() + .map(k -> k.replaceAll(" ", "_")) + .collect(Collectors.toList()); if (dagCounters != null && wmContext != null) { Set<String> desiredCounters = wmContext.getSubscribedCounters(); if (desiredCounters != null && !desiredCounters.isEmpty()) { - Map<String, Long> currentCounters = getCounterValues(dagCounters, vertexProgressMap, desiredCounters, done); + Map<String, Long> currentCounters = getCounterValues(dagCounters, vertexNames, vertexProgressMap, + desiredCounters, done); wmContext.setCurrentCounters(currentCounters); } } @@ -292,32 +298,62 @@ public class TezJobMonitor { } private Map<String, Long> getCounterValues(final TezCounters dagCounters, - final Map<String, Progress> vertexProgressMap, + final List<String> vertexNames, final Map<String, Progress> vertexProgressMap, final Set<String> desiredCounters, final boolean done) { // DAG specific counters Map<String, Long> updatedCounters = new HashMap<>(); for (CounterGroup counterGroup : dagCounters) { for (TezCounter tezCounter : counterGroup) { String counterName = tezCounter.getName(); - if (desiredCounters.contains(counterName)) { - updatedCounters.put(counterName, tezCounter.getValue()); + for (String desiredCounter : desiredCounters) { + if (counterName.equals(desiredCounter)) { + updatedCounters.put(counterName, tezCounter.getValue()); + } else if (isDagLevelCounter(desiredCounter)) { + // by default, we aggregate counters across the entire DAG. Example: SHUFFLE_BYTES would mean SHUFFLE_BYTES + // of each vertex aggregated together to create DAG level SHUFFLE_BYTES. + // Use case: If SHUFFLE_BYTES across the entire DAG is > limit perform action + String prefixRemovedCounterName = getCounterFromDagCounter(desiredCounter); + aggregateCountersSum(updatedCounters, vertexNames, prefixRemovedCounterName, desiredCounter, tezCounter); + } else if (isVertexLevelCounter(desiredCounter)) { + // if counter name starts with VERTEX_ then we just return max value across all vertex since trigger + // validation is only interested in violation that are greater than limit (*any* vertex violation). + // Use case: If SHUFFLE_BYTES for any single vertex is > limit perform action + String prefixRemovedCounterName = getCounterFromVertexCounter(desiredCounter); + aggregateCountersMax(updatedCounters, vertexNames, prefixRemovedCounterName, desiredCounter, tezCounter); + } else if (counterName.startsWith(desiredCounter)) { + // Counters with vertex name as suffix + // desiredCounter = INPUT_FILES + // counters: {INPUT_FILES_Map_1 : 5, INPUT_FILES_Map_4 : 10} + // outcome: INPUT_FILE : 15 + String prefixRemovedCounterName = desiredCounter; + aggregateCountersSum(updatedCounters, vertexNames, prefixRemovedCounterName, desiredCounter, tezCounter); + } } } } - // Process per vertex counters. - String counterName = VertexCounterLimit.VertexCounter.TOTAL_TASKS.name(); + // Process per vertex counters that are available only via vertex Progress + String counterName = VertexCounterLimit.VertexCounter.VERTEX_TOTAL_TASKS.name(); if (desiredCounters.contains(counterName) && vertexProgressMap != null) { for (Map.Entry<String, Progress> entry : vertexProgressMap.entrySet()) { - // TOTAL_TASKS counter is per vertex counter, but triggers are validated at query level - // looking for query level violations. So we always choose max TOTAL_TASKS among all vertices. - // Publishing TOTAL_TASKS for all vertices is not really useful from the context of triggers. long currentMax = 0; if (updatedCounters.containsKey(counterName)) { currentMax = updatedCounters.get(counterName); } - long totalTasks = Math.max(currentMax, entry.getValue().getTotalTaskCount()); - updatedCounters.put(counterName, totalTasks); + long newMax = Math.max(currentMax, entry.getValue().getTotalTaskCount()); + updatedCounters.put(counterName, newMax); + } + } + + counterName = VertexCounterLimit.VertexCounter.DAG_TOTAL_TASKS.name(); + if (desiredCounters.contains(counterName) && vertexProgressMap != null) { + for (Map.Entry<String, Progress> entry : vertexProgressMap.entrySet()) { + long currentTotal = 0; + if (updatedCounters.containsKey(counterName)) { + currentTotal = updatedCounters.get(counterName); + } + long newTotal = currentTotal + entry.getValue().getTotalTaskCount(); + updatedCounters.put(counterName, newTotal); } } @@ -337,6 +373,54 @@ public class TezJobMonitor { return updatedCounters; } + private void aggregateCountersSum(final Map<String, Long> updatedCounters, final List<String> vertexNames, + final String prefixRemovedCounterName, final String desiredCounter, final TezCounter tezCounter) { + long counterValue = checkVertexSuffixAndGetValue(vertexNames, prefixRemovedCounterName, tezCounter); + long currentTotal = 0; + if (updatedCounters.containsKey(desiredCounter)) { + currentTotal = updatedCounters.get(desiredCounter); + } + long newTotal = currentTotal + counterValue; + updatedCounters.put(desiredCounter, newTotal); + } + + private void aggregateCountersMax(final Map<String, Long> updatedCounters, final List<String> vertexNames, + final String prefixRemovedCounterName, final String desiredCounter, final TezCounter tezCounter) { + long counterValue = checkVertexSuffixAndGetValue(vertexNames, prefixRemovedCounterName, tezCounter); + long currentMax = 0; + if (updatedCounters.containsKey(desiredCounter)) { + currentMax = updatedCounters.get(desiredCounter); + } + long newMax = Math.max(currentMax, counterValue); + updatedCounters.put(desiredCounter, newMax); + } + + private long checkVertexSuffixAndGetValue(final List<String> vertexNames, final String counterName, + final TezCounter tezCounter) { + for (String vertexName : vertexNames) { + if (tezCounter.getName().equalsIgnoreCase(counterName + "_" + vertexName)) { + return tezCounter.getValue(); + } + } + return 0; + } + + private String getCounterFromDagCounter(final String desiredCounter) { + return desiredCounter.substring("DAG_".length()); + } + + private String getCounterFromVertexCounter(final String desiredCounter) { + return desiredCounter.substring("VERTEX_".length()); + } + + private boolean isVertexLevelCounter(final String desiredCounter) { + return desiredCounter.startsWith("VERTEX_"); + } + + private boolean isDagLevelCounter(final String desiredCounter) { + return desiredCounter.startsWith("DAG_"); + } + private void printSummary(boolean success, Map<String, Progress> progressMap) { if (isProfilingEnabled() && success && progressMap != null) { http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java index 012adaa..14ebfa0 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/PostExecTezSummaryPrinter.java @@ -20,7 +20,9 @@ package org.apache.hadoop.hive.ql.hooks; import java.util.List; import org.apache.hadoop.hive.llap.counters.LlapIOCounters; +import org.apache.hadoop.hive.ql.exec.tez.HiveInputCounters; import org.apache.tez.common.counters.FileSystemCounter; +import org.apache.tez.dag.api.client.DAGClient; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.apache.hadoop.hive.conf.HiveConf; @@ -66,6 +68,11 @@ public class PostExecTezSummaryPrinter implements ExecuteWithHookContext { for (TezCounter counter : group) { console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); } + } else if (group.getName().equals(HiveInputCounters.class.getName())) { + console.printInfo(tezTask.getId() + " INPUT COUNTERS:", false); + for (TezCounter counter : group) { + console.printInfo(" " + counter.getDisplayName() + ": " + counter.getValue(), false); + } } else if (group.getName().equals(FileSystemCounter.class.getName())) { console.printInfo(tezTask.getId() + " FILE SYSTEM COUNTERS:", false); for (TezCounter counter : group) { http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/java/org/apache/hadoop/hive/ql/wm/VertexCounterLimit.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/wm/VertexCounterLimit.java b/ql/src/java/org/apache/hadoop/hive/ql/wm/VertexCounterLimit.java index 7d6482a..5c71aa3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/wm/VertexCounterLimit.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/wm/VertexCounterLimit.java @@ -20,7 +20,16 @@ package org.apache.hadoop.hive.ql.wm; */ public class VertexCounterLimit implements CounterLimit { public enum VertexCounter { - TOTAL_TASKS + DAG_TOTAL_TASKS, + DAG_GROUPED_INPUT_SPLITS, + DAG_INPUT_DIRECTORIES, + DAG_INPUT_FILES, + DAG_RAW_INPUT_SPLITS, + VERTEX_TOTAL_TASKS, + VERTEX_GROUPED_INPUT_SPLITS, + VERTEX_INPUT_DIRECTORIES, + VERTEX_INPUT_FILES, + VERTEX_RAW_INPUT_SPLITS } private VertexCounter vertexCounter; http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/test/org/apache/hadoop/hive/ql/wm/TestTrigger.java ---------------------------------------------------------------------- diff --git a/ql/src/test/org/apache/hadoop/hive/ql/wm/TestTrigger.java b/ql/src/test/org/apache/hadoop/hive/ql/wm/TestTrigger.java index a3e8336..c5748f5 100644 --- a/ql/src/test/org/apache/hadoop/hive/ql/wm/TestTrigger.java +++ b/ql/src/test/org/apache/hadoop/hive/ql/wm/TestTrigger.java @@ -72,9 +72,16 @@ public class TestTrigger { assertTrue(trigger.apply(100000)); expression = ExpressionFactory.createExpression(new VertexCounterLimit(VertexCounterLimit.VertexCounter - .TOTAL_TASKS,10000)); + .VERTEX_TOTAL_TASKS, 10000)); trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); - assertEquals("counter: TOTAL_TASKS limit: 10000", expression.getCounterLimit().toString()); + assertEquals("counter: VERTEX_TOTAL_TASKS limit: 10000", expression.getCounterLimit().toString()); + assertFalse(trigger.apply(1000)); + assertTrue(trigger.apply(100000)); + + expression = ExpressionFactory.createExpression(new VertexCounterLimit(VertexCounterLimit.VertexCounter + .DAG_TOTAL_TASKS, 10000)); + trigger = new ExecutionTrigger("highly_parallel", expression, new Action(Action.Type.KILL_QUERY)); + assertEquals("counter: DAG_TOTAL_TASKS limit: 10000", expression.getCounterLimit().toString()); assertFalse(trigger.apply(1000)); assertTrue(trigger.apply(100000)); @@ -163,10 +170,17 @@ public class TestTrigger { assertEquals(expected, expression); assertEquals(expected.hashCode(), expression.hashCode()); - expression = ExpressionFactory.fromString(" TOTAL_TASKS > 10000"); + expression = ExpressionFactory.fromString(" VERTEX_TOTAL_TASKS > 10000"); + expected = ExpressionFactory.createExpression(new VertexCounterLimit(VertexCounterLimit.VertexCounter + .VERTEX_TOTAL_TASKS, 10000)); + assertEquals("counter: VERTEX_TOTAL_TASKS limit: 10000", expression.getCounterLimit().toString()); + assertEquals(expected, expression); + assertEquals(expected.hashCode(), expression.hashCode()); + + expression = ExpressionFactory.fromString(" DAG_TOTAL_TASKS > 10000"); expected = ExpressionFactory.createExpression(new VertexCounterLimit(VertexCounterLimit.VertexCounter - .TOTAL_TASKS,10000)); - assertEquals("counter: TOTAL_TASKS limit: 10000", expression.getCounterLimit().toString()); + .DAG_TOTAL_TASKS, 10000)); + assertEquals("counter: DAG_TOTAL_TASKS limit: 10000", expression.getCounterLimit().toString()); assertEquals(expected, expression); assertEquals(expected.hashCode(), expression.hashCode()); http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/test/queries/clientpositive/tez_input_counters.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/tez_input_counters.q b/ql/src/test/queries/clientpositive/tez_input_counters.q new file mode 100644 index 0000000..d471c3d --- /dev/null +++ b/ql/src/test/queries/clientpositive/tez_input_counters.q @@ -0,0 +1,25 @@ +set hive.compute.query.using.stats=false; +set hive.exec.dynamic.partition.mode=nonstrict; +set hive.exec.max.dynamic.partitions=400; +set hive.exec.max.dynamic.partitions.pernode=400; +set hive.mapred.mode=nonstrict; +set hive.fetch.task.conversion=none; +set hive.map.aggr=false; +-- disabling map side aggregation as that can lead to different intermediate record counts +set hive.tez.exec.print.summary=true; + +create table testpart (k int) partitioned by (v string); +insert overwrite table testpart partition(v) select * from src; +insert into table testpart partition(v) select * from src; + +set hive.exec.post.hooks=org.apache.hadoop.hive.ql.hooks.PostExecTezSummaryPrinter; +select sum(hash(*)) from testpart; +select sum(hash(*)) from testpart where v < 'val_100'; +select sum(hash(*)) from testpart where v < 'val_200'; + +set hive.tez.dynamic.partition.pruning=true; +create table testpart1 like testpart; +insert overwrite table testpart1 partition(v) select * from testpart where v < 'val_200'; + +explain select sum(hash(*)) from testpart t1 join testpart1 t2 on t1.v = t2.v; +select sum(hash(*)) from testpart t1 join testpart1 t2 on t1.v = t2.v; http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out b/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out index 662c02a..981f260 100644 --- a/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out +++ b/ql/src/test/results/clientpositive/llap/dp_counter_mm.q.out @@ -21,6 +21,11 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 RECORDS_OUT_1_default.src2: 84 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: insert into table src2 partition (value) select * from src where key < 200 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -32,6 +37,11 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 RECORDS_OUT_1_default.src2: 189 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: drop table src2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@src2 @@ -51,6 +61,11 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 RECORDS_OUT_1_default.src2: 189 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: insert into table src2 partition (value) select * from src where key < 300 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -62,6 +77,11 @@ Stage-1 HIVE COUNTERS: DESERIALIZE_ERRORS: 0 RECORDS_IN_Map_1: 500 RECORDS_OUT_1_default.src2: 292 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: drop table src2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@src2 @@ -91,6 +111,11 @@ Stage-2 HIVE COUNTERS: RECORDS_IN_Map_1: 500 RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_2_default.src3: 105 +Stage-2 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: from src insert into table src2 partition (value) select * where key < 100 insert into table src3 partition (value) select * where key >= 100 and key < 300 @@ -106,6 +131,11 @@ Stage-2 HIVE COUNTERS: RECORDS_IN_Map_1: 500 RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_2_default.src3: 208 +Stage-2 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: drop table src2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@src2 @@ -130,6 +160,15 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 500 RECORDS_IN_Map_3: 500 RECORDS_OUT_1_default.src2: 189 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + GROUPED_INPUT_SPLITS_Map_3: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_DIRECTORIES_Map_3: 1 + INPUT_FILES_Map_1: 1 + INPUT_FILES_Map_3: 1 + RAW_INPUT_SPLITS_Map_1: 1 + RAW_INPUT_SPLITS_Map_3: 1 PREHOOK: query: insert into table src2 partition (value) select temps.* from ( select * from src where key < 100 @@ -146,3 +185,12 @@ Stage-1 HIVE COUNTERS: RECORDS_IN_Map_1: 500 RECORDS_IN_Map_3: 500 RECORDS_OUT_1_default.src2: 292 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + GROUPED_INPUT_SPLITS_Map_3: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_DIRECTORIES_Map_3: 1 + INPUT_FILES_Map_1: 1 + INPUT_FILES_Map_3: 1 + RAW_INPUT_SPLITS_Map_1: 1 + RAW_INPUT_SPLITS_Map_3: 1 http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out b/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out index e2ff9a6..9f6426c 100644 --- a/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out +++ b/ql/src/test/results/clientpositive/llap/dp_counter_non_mm.q.out @@ -23,6 +23,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 57 RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_INTERMEDIATE_Map_1: 57 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: insert into table src2 partition (value) select * from src where key < 200 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -36,6 +41,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 121 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: drop table src2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@src2 @@ -57,6 +67,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 121 RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 121 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: insert into table src2 partition (value) select * from src where key < 300 PREHOOK: type: QUERY PREHOOK: Input: default@src @@ -70,6 +85,11 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_0: 184 RECORDS_OUT_1_default.src2: 292 RECORDS_OUT_INTERMEDIATE_Map_1: 184 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: drop table src2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@src2 @@ -101,6 +121,11 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_2_default.src3: 105 RECORDS_OUT_INTERMEDIATE_Map_1: 121 +Stage-2 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: from src insert into table src2 partition (value) select * where key < 100 insert into table src3 partition (value) select * where key >= 100 and key < 300 @@ -118,6 +143,11 @@ Stage-2 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 84 RECORDS_OUT_2_default.src3: 208 RECORDS_OUT_INTERMEDIATE_Map_1: 184 +Stage-2 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 PREHOOK: query: drop table src2 PREHOOK: type: DROPTABLE PREHOOK: Input: default@src2 @@ -145,6 +175,15 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 189 RECORDS_OUT_INTERMEDIATE_Map_1: 57 RECORDS_OUT_INTERMEDIATE_Map_4: 64 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + GROUPED_INPUT_SPLITS_Map_4: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_DIRECTORIES_Map_4: 1 + INPUT_FILES_Map_1: 1 + INPUT_FILES_Map_4: 1 + RAW_INPUT_SPLITS_Map_1: 1 + RAW_INPUT_SPLITS_Map_4: 1 PREHOOK: query: insert into table src2 partition (value) select temps.* from ( select * from src where key < 100 @@ -164,3 +203,12 @@ Stage-1 HIVE COUNTERS: RECORDS_OUT_1_default.src2: 292 RECORDS_OUT_INTERMEDIATE_Map_1: 57 RECORDS_OUT_INTERMEDIATE_Map_4: 127 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + GROUPED_INPUT_SPLITS_Map_4: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_DIRECTORIES_Map_4: 1 + INPUT_FILES_Map_1: 1 + INPUT_FILES_Map_4: 1 + RAW_INPUT_SPLITS_Map_1: 1 + RAW_INPUT_SPLITS_Map_4: 1 http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out index 94e1ce2..aa2dcc7 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters.q.out @@ -270,6 +270,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 SELECTED_ROWGROUPS: 3 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2100 PREHOOK: query: select count(*) from orc_ppd where t > 127 PREHOOK: type: QUERY @@ -284,6 +289,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where t = 55 PREHOOK: type: QUERY @@ -311,6 +321,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 8 PREHOOK: query: select count(*) from orc_ppd where t <=> 50 PREHOOK: type: QUERY @@ -336,6 +351,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 22 PREHOOK: query: select count(*) from orc_ppd where t <=> 100 PREHOOK: type: QUERY @@ -361,6 +381,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 16 PREHOOK: query: select count(*) from orc_ppd where t = "54" PREHOOK: type: QUERY @@ -386,6 +411,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 18 PREHOOK: query: select count(*) from orc_ppd where t = -10.0 PREHOOK: type: QUERY @@ -411,6 +441,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 1 PREHOOK: query: select count(*) from orc_ppd where t = cast(53 as float) PREHOOK: type: QUERY @@ -436,6 +471,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 32 PREHOOK: query: select count(*) from orc_ppd where t = cast(53 as double) PREHOOK: type: QUERY @@ -461,6 +501,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 32 PREHOOK: query: select count(*) from orc_ppd where t < 100 PREHOOK: type: QUERY @@ -486,6 +531,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 1697 PREHOOK: query: select count(*) from orc_ppd where t < 100 and t > 98 PREHOOK: type: QUERY @@ -511,6 +561,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 12 PREHOOK: query: select count(*) from orc_ppd where t <= 100 PREHOOK: type: QUERY @@ -536,6 +591,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 1713 PREHOOK: query: select count(*) from orc_ppd where t is null PREHOOK: type: QUERY @@ -561,6 +621,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 6 PREHOOK: query: select count(*) from orc_ppd where t in (5, 120) PREHOOK: type: QUERY @@ -586,6 +651,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 1100 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 50 PREHOOK: query: select count(*) from orc_ppd where t between 60 and 80 PREHOOK: type: QUERY @@ -611,6 +681,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 318 PREHOOK: query: select count(*) from orc_ppd where t = -100 PREHOOK: type: QUERY @@ -625,6 +700,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where t <=> -100 PREHOOK: type: QUERY @@ -639,6 +719,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where t = 125 PREHOOK: type: QUERY @@ -661,6 +746,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where t IN (-100, 125, 200) PREHOOK: type: QUERY @@ -683,6 +773,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s > "zzz" PREHOOK: type: QUERY @@ -697,6 +792,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where s = "zach young" PREHOOK: type: QUERY @@ -724,6 +824,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s <=> "zach zipper" PREHOOK: type: QUERY @@ -749,6 +854,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 6 PREHOOK: query: select count(*) from orc_ppd where s <=> "" PREHOOK: type: QUERY @@ -774,6 +884,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 6 PREHOOK: query: select count(*) from orc_ppd where s is null PREHOOK: type: QUERY @@ -788,6 +903,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where s is not null PREHOOK: type: QUERY @@ -813,6 +933,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 SELECTED_ROWGROUPS: 3 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2100 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as char(50)) PREHOOK: type: QUERY @@ -835,6 +960,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as char(10)) PREHOOK: type: QUERY @@ -860,6 +990,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as varchar(10)) PREHOOK: type: QUERY @@ -885,6 +1020,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as varchar(50)) PREHOOK: type: QUERY @@ -910,6 +1050,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s < "b" PREHOOK: type: QUERY @@ -935,6 +1080,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 81 PREHOOK: query: select count(*) from orc_ppd where s > "alice" and s < "bob" PREHOOK: type: QUERY @@ -960,6 +1110,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 74 PREHOOK: query: select count(*) from orc_ppd where s in ("alice allen", "") PREHOOK: type: QUERY @@ -985,6 +1140,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 12 PREHOOK: query: select count(*) from orc_ppd where s between "" and "alice allen" PREHOOK: type: QUERY @@ -1010,6 +1170,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 13 PREHOOK: query: select count(*) from orc_ppd where s between "zz" and "zzz" PREHOOK: type: QUERY @@ -1035,6 +1200,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 1 PREHOOK: query: select count(*) from orc_ppd where s between "zach zipper" and "zzz" PREHOOK: type: QUERY @@ -1060,6 +1230,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 1100 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 7 PREHOOK: query: select count(*) from orc_ppd where s = "hello world" PREHOOK: type: QUERY @@ -1082,6 +1257,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s <=> "apache hive" PREHOOK: type: QUERY @@ -1104,6 +1284,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s IN ("a", "z") PREHOOK: type: QUERY @@ -1126,6 +1311,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s = "sarah ovid" PREHOOK: type: QUERY @@ -1151,6 +1341,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" PREHOOK: type: QUERY @@ -1176,6 +1371,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 1100 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 6 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" and t < 0 PREHOOK: type: QUERY @@ -1201,6 +1401,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" and t > 100 PREHOOK: type: QUERY @@ -1226,4 +1431,9 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out index e61917f..eb54a81 100644 --- a/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_llap_counters1.q.out @@ -271,6 +271,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 SELECTED_ROWGROUPS: 3 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2094 PREHOOK: query: select count(*) from orc_ppd where t > -100 PREHOOK: type: QUERY @@ -296,6 +301,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 SELECTED_ROWGROUPS: 3 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2094 PREHOOK: query: DROP TABLE staging PREHOOK: type: DROPTABLE http://git-wip-us.apache.org/repos/asf/hive/blob/fff86f3a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out index 7904661..c5302c3 100644 --- a/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out +++ b/ql/src/test/results/clientpositive/llap/orc_ppd_basic.q.out @@ -223,6 +223,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 SELECTED_ROWGROUPS: 3 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2100 PREHOOK: query: select count(*) from orc_ppd where t > 127 PREHOOK: type: QUERY @@ -237,6 +242,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where t = 55 PREHOOK: type: QUERY @@ -264,6 +274,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 8 PREHOOK: query: select count(*) from orc_ppd where t <=> 50 PREHOOK: type: QUERY @@ -289,6 +304,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 22 PREHOOK: query: select count(*) from orc_ppd where t <=> 100 PREHOOK: type: QUERY @@ -314,6 +334,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 16 PREHOOK: query: select count(*) from orc_ppd where t = "54" PREHOOK: type: QUERY @@ -339,6 +364,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 18 PREHOOK: query: select count(*) from orc_ppd where t = -10.0 PREHOOK: type: QUERY @@ -364,6 +394,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 1 PREHOOK: query: select count(*) from orc_ppd where t = cast(53 as float) PREHOOK: type: QUERY @@ -389,6 +424,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 32 PREHOOK: query: select count(*) from orc_ppd where t = cast(53 as double) PREHOOK: type: QUERY @@ -414,6 +454,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 32 PREHOOK: query: select count(*) from orc_ppd where t < 100 PREHOOK: type: QUERY @@ -439,6 +484,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 1697 PREHOOK: query: select count(*) from orc_ppd where t < 100 and t > 98 PREHOOK: type: QUERY @@ -464,6 +514,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 12 PREHOOK: query: select count(*) from orc_ppd where t <= 100 PREHOOK: type: QUERY @@ -489,6 +544,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 1713 PREHOOK: query: select count(*) from orc_ppd where t is null PREHOOK: type: QUERY @@ -514,6 +574,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 6 PREHOOK: query: select count(*) from orc_ppd where t in (5, 120) PREHOOK: type: QUERY @@ -539,6 +604,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 1100 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 50 PREHOOK: query: select count(*) from orc_ppd where t between 60 and 80 PREHOOK: type: QUERY @@ -564,6 +634,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 318 PREHOOK: query: select count(*) from orc_ppd where t = -100 PREHOOK: type: QUERY @@ -578,6 +653,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where t <=> -100 PREHOOK: type: QUERY @@ -592,6 +672,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where t = 125 PREHOOK: type: QUERY @@ -614,6 +699,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where t IN (-100, 125, 200) PREHOOK: type: QUERY @@ -636,6 +726,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s > "zzz" PREHOOK: type: QUERY @@ -650,6 +745,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where s = "zach young" PREHOOK: type: QUERY @@ -677,6 +777,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s <=> "zach zipper" PREHOOK: type: QUERY @@ -702,6 +807,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 6 PREHOOK: query: select count(*) from orc_ppd where s <=> "" PREHOOK: type: QUERY @@ -727,6 +837,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 6 PREHOOK: query: select count(*) from orc_ppd where s is null PREHOOK: type: QUERY @@ -741,6 +856,11 @@ Stage-1 FILE SYSTEM COUNTERS: Stage-1 HIVE COUNTERS: CREATED_FILES: 1 RECORDS_OUT_0: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 0 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 0 + RAW_INPUT_SPLITS_Map_1: 0 0 PREHOOK: query: select count(*) from orc_ppd where s is not null PREHOOK: type: QUERY @@ -766,6 +886,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 SELECTED_ROWGROUPS: 3 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2100 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as char(50)) PREHOOK: type: QUERY @@ -788,6 +913,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as char(10)) PREHOOK: type: QUERY @@ -813,6 +943,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as varchar(10)) PREHOOK: type: QUERY @@ -838,6 +973,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = cast("zach young" as varchar(50)) PREHOOK: type: QUERY @@ -863,6 +1003,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s < "b" PREHOOK: type: QUERY @@ -888,6 +1033,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 81 PREHOOK: query: select count(*) from orc_ppd where s > "alice" and s < "bob" PREHOOK: type: QUERY @@ -913,6 +1063,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 74 PREHOOK: query: select count(*) from orc_ppd where s in ("alice allen", "") PREHOOK: type: QUERY @@ -938,6 +1093,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 12 PREHOOK: query: select count(*) from orc_ppd where s between "" and "alice allen" PREHOOK: type: QUERY @@ -963,6 +1123,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 2000 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 13 PREHOOK: query: select count(*) from orc_ppd where s between "zz" and "zzz" PREHOOK: type: QUERY @@ -988,6 +1153,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 1 PREHOOK: query: select count(*) from orc_ppd where s between "zach zipper" and "zzz" PREHOOK: type: QUERY @@ -1013,6 +1183,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 1100 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 7 PREHOOK: query: select count(*) from orc_ppd where s = "hello world" PREHOOK: type: QUERY @@ -1035,6 +1210,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s <=> "apache hive" PREHOOK: type: QUERY @@ -1057,6 +1237,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s IN ("a", "z") PREHOOK: type: QUERY @@ -1079,6 +1264,11 @@ Stage-1 LLAP IO COUNTERS: CACHE_MISS_BYTES: 0 METADATA_CACHE_HIT: 2 SELECTED_ROWGROUPS: 0 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 0 PREHOOK: query: select count(*) from orc_ppd where s = "sarah ovid" PREHOOK: type: QUERY @@ -1104,6 +1294,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" PREHOOK: type: QUERY @@ -1129,6 +1324,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 2 ROWS_EMITTED: 1100 SELECTED_ROWGROUPS: 2 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 6 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" and t < 0 PREHOOK: type: QUERY @@ -1154,6 +1354,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 1000 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where s = "wendy king" and t > 100 PREHOOK: type: QUERY @@ -1179,6 +1384,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY @@ -1206,6 +1416,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 SELECTED_ROWGROUPS: 3 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY @@ -1233,6 +1448,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY @@ -1258,6 +1478,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 3 ROWS_EMITTED: 2100 SELECTED_ROWGROUPS: 3 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: select count(*) from orc_ppd where f=74.72 PREHOOK: type: QUERY @@ -1283,6 +1508,11 @@ Stage-1 LLAP IO COUNTERS: NUM_VECTOR_BATCHES: 1 ROWS_EMITTED: 100 SELECTED_ROWGROUPS: 1 +Stage-1 INPUT COUNTERS: + GROUPED_INPUT_SPLITS_Map_1: 1 + INPUT_DIRECTORIES_Map_1: 1 + INPUT_FILES_Map_1: 1 + RAW_INPUT_SPLITS_Map_1: 1 2 PREHOOK: query: create temporary table tmp_orcppd stored as orc