Author: hashutosh
Date: Sun Nov 10 03:19:52 2013
New Revision: 1540436
URL: http://svn.apache.org/r1540436
Log:
HIVE-3990 : Provide input threshold for direct-fetcher (Navis via Ashutosh
Chauhan)
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/InputEstimator.java
hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q
hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml.template
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1540436&r1=1540435&r2=1540436&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Sun
Nov 10 03:19:52 2013
@@ -663,6 +663,7 @@ public class HiveConf extends Configurat
// 'minimal', 'more' (and 'all' later)
HIVEFETCHTASKCONVERSION("hive.fetch.task.conversion", "minimal",
new StringsValidator("minimal", "more")),
+ HIVEFETCHTASKCONVERSIONTHRESHOLD("hive.fetch.task.conversion.threshold",
-1l),
HIVEFETCHTASKAGGR("hive.fetch.task.aggr", false),
Modified: hive/trunk/conf/hive-default.xml.template
URL:
http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml.template?rev=1540436&r1=1540435&r2=1540436&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml.template (original)
+++ hive/trunk/conf/hive-default.xml.template Sun Nov 10 03:19:52 2013
@@ -1799,6 +1799,16 @@
<property>
+ <name>hive.fetch.task.conversion.threshold</name>
+ <value>-1</value>
+ <description>
+ Input threshold for applying hive.fetch.task.conversion. If target table
is native, input length
+ is calculated by summation of file lengths. If it's not native, storage
handler for the table
+ can optionally implement org.apache.hadoop.hive.ql.metadata.InputEstimator
interface.
+ </description>
+</property>
+
+<property>
<name>hive.hmshandler.retry.attempts</name>
<value>1</value>
<description>The number of times to retry a HMSHandler call if there were a
connection error</description>
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1540436&r1=1540435&r2=1540436&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java Sun
Nov 10 03:19:52 2013
@@ -100,6 +100,7 @@ import org.apache.hadoop.hive.conf.HiveC
import org.apache.hadoop.hive.metastore.Warehouse;
import org.apache.hadoop.hive.metastore.api.FieldSchema;
import org.apache.hadoop.hive.metastore.api.Order;
+import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants;
import org.apache.hadoop.hive.ql.Context;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.QueryPlan;
@@ -125,6 +126,9 @@ import org.apache.hadoop.hive.ql.io.rcfi
import org.apache.hadoop.hive.ql.io.rcfile.truncate.ColumnTruncateWork;
import org.apache.hadoop.hive.ql.log.PerfLogger;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
+import org.apache.hadoop.hive.ql.metadata.HiveUtils;
+import org.apache.hadoop.hive.ql.metadata.InputEstimator;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.parse.SemanticException;
@@ -2001,7 +2005,7 @@ public final class Utilities {
}
}
- public static Object INPUT_SUMMARY_LOCK = new Object();
+ private static final Object INPUT_SUMMARY_LOCK = new Object();
/**
* Calculate the total size of input files.
@@ -2084,26 +2088,47 @@ public final class Utilities {
// is not correct.
final Configuration myConf = conf;
final JobConf myJobConf = jobConf;
+ final Map<String, Operator<?>> aliasToWork = work.getAliasToWork();
+ final Map<String, ArrayList<String>> pathToAlias =
work.getPathToAliases();
final PartitionDesc partDesc = work.getPathToPartitionInfo().get(
p.toString());
Runnable r = new Runnable() {
public void run() {
try {
- ContentSummary resultCs;
-
Class<? extends InputFormat> inputFormatCls = partDesc
.getInputFileFormatClass();
InputFormat inputFormatObj =
HiveInputFormat.getInputFormatFromCache(
inputFormatCls, myJobConf);
if (inputFormatObj instanceof ContentSummaryInputFormat) {
- resultCs = ((ContentSummaryInputFormat)
inputFormatObj).getContentSummary(p,
- myJobConf);
- } else {
+ ContentSummaryInputFormat cs = (ContentSummaryInputFormat)
inputFormatObj;
+ resultMap.put(pathStr, cs.getContentSummary(p, myJobConf));
+ return;
+ }
+ HiveStorageHandler handler =
HiveUtils.getStorageHandler(myConf,
+ partDesc.getOverlayedProperties().getProperty(
+ hive_metastoreConstants.META_TABLE_STORAGE));
+ if (handler == null) {
+ // native table
FileSystem fs = p.getFileSystem(myConf);
- resultCs = fs.getContentSummary(p);
+ resultMap.put(pathStr, fs.getContentSummary(p));
+ return;
+ }
+ if (handler instanceof InputEstimator) {
+ long total = 0;
+ TableDesc tableDesc = partDesc.getTableDesc();
+ InputEstimator estimator = (InputEstimator) handler;
+ for (String alias :
HiveFileFormatUtils.doGetAliasesFromPath(pathToAlias, p)) {
+ JobConf jobConf = new JobConf(myJobConf);
+ TableScanOperator scanOp = (TableScanOperator)
aliasToWork.get(alias);
+ Utilities.setColumnNameList(jobConf, scanOp, true);
+ Utilities.setColumnTypeList(jobConf, scanOp, true);
+
PlanUtils.configureInputJobPropertiesForStorageHandler(tableDesc);
+ Utilities.copyTableJobPropertiesToConf(tableDesc, jobConf);
+ total += estimator.estimate(myJobConf, scanOp,
-1).getTotalLength();
+ }
+ resultMap.put(pathStr, new ContentSummary(total, -1, -1));
}
- resultMap.put(pathStr, resultCs);
- } catch (IOException e) {
+ } catch (Exception e) {
// We safely ignore this exception for summary data.
// We don't update the cache to protect it from polluting other
// usages. The worst case is that IOException will always be
@@ -2290,12 +2315,19 @@ public final class Utilities {
}
public static void setColumnNameList(JobConf jobConf, Operator op) {
+ setColumnNameList(jobConf, op, false);
+ }
+
+ public static void setColumnNameList(JobConf jobConf, Operator op, boolean
excludeVCs) {
RowSchema rowSchema = op.getSchema();
if (rowSchema == null) {
return;
}
StringBuilder columnNames = new StringBuilder();
for (ColumnInfo colInfo : rowSchema.getSignature()) {
+ if (excludeVCs && colInfo.getIsVirtualCol()) {
+ continue;
+ }
if (columnNames.length() > 0) {
columnNames.append(",");
}
@@ -2306,12 +2338,19 @@ public final class Utilities {
}
public static void setColumnTypeList(JobConf jobConf, Operator op) {
+ setColumnTypeList(jobConf, op, false);
+ }
+
+ public static void setColumnTypeList(JobConf jobConf, Operator op, boolean
excludeVCs) {
RowSchema rowSchema = op.getSchema();
if (rowSchema == null) {
return;
}
StringBuilder columnTypes = new StringBuilder();
for (ColumnInfo colInfo : rowSchema.getSignature()) {
+ if (excludeVCs && colInfo.getIsVirtualCol()) {
+ continue;
+ }
if (columnTypes.length() > 0) {
columnTypes.append(",");
}
Added:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/InputEstimator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/InputEstimator.java?rev=1540436&view=auto
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/InputEstimator.java
(added)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/InputEstimator.java
Sun Nov 10 03:19:52 2013
@@ -0,0 +1,54 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hive.ql.metadata;
+
+import org.apache.hadoop.hive.ql.exec.TableScanOperator;
+import org.apache.hadoop.mapred.JobConf;
+
+// Plugin interface for storage handler which supports input estimation
+public interface InputEstimator {
+
+ /**
+ * Estimate input size based on filter and projection on table scan operator
+ *
+ * @param remaining Early exit condition. If it has positive value, further
estimation
+ * can be canceled on the point of exceeding it. In this
case,
+ * return any bigger length value then this
(Long.MAX_VALUE, for eaxmple).
+ */
+ Estimation estimate(JobConf job, TableScanOperator ts, long remaining)
throws HiveException;
+
+ public static class Estimation {
+
+ private int rowCount;
+ private long totalLength;
+
+ public Estimation(int rowCount, long totalLength) {
+ this.rowCount = rowCount;
+ this.totalLength = totalLength;
+ }
+
+ public int getRowCount() {
+ return rowCount;
+ }
+
+ public long getTotalLength() {
+ return totalLength;
+ }
+ }
+}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java?rev=1540436&r1=1540435&r2=1540436&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/optimizer/SimpleFetchOptimizer.java
Sun Nov 10 03:19:52 2013
@@ -18,6 +18,7 @@
package org.apache.hadoop.hive.ql.optimizer;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
@@ -25,6 +26,8 @@ import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.ContentSummary;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.exec.FetchTask;
import org.apache.hadoop.hive.ql.exec.FileSinkOperator;
@@ -37,7 +40,11 @@ import org.apache.hadoop.hive.ql.exec.Ta
import org.apache.hadoop.hive.ql.exec.TaskFactory;
import org.apache.hadoop.hive.ql.exec.Utilities;
import org.apache.hadoop.hive.ql.hooks.ReadEntity;
+import org.apache.hadoop.hive.ql.io.ContentSummaryInputFormat;
+import org.apache.hadoop.hive.ql.io.HiveInputFormat;
+import org.apache.hadoop.hive.ql.metadata.InputEstimator;
import org.apache.hadoop.hive.ql.metadata.HiveException;
+import org.apache.hadoop.hive.ql.metadata.HiveStorageHandler;
import org.apache.hadoop.hive.ql.metadata.Partition;
import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.optimizer.ppr.PartitionPruner;
@@ -53,6 +60,8 @@ import org.apache.hadoop.hive.ql.plan.Op
import org.apache.hadoop.hive.ql.plan.PartitionDesc;
import org.apache.hadoop.hive.ql.plan.PlanUtils;
import org.apache.hadoop.hive.ql.plan.TableDesc;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.JobConf;
/**
* Tries to convert simple fetch query to single fetch task, which fetches
rows directly
@@ -75,7 +84,7 @@ public class SimpleFetchOptimizer implem
if (fetchTask != null) {
pctx.setFetchTask(fetchTask);
}
- } catch (HiveException e) {
+ } catch (Exception e) {
// Has to use full name to make sure it does not conflict with
// org.apache.commons.lang.StringUtils
LOG.error(org.apache.hadoop.util.StringUtils.stringifyException(e));
@@ -92,13 +101,13 @@ public class SimpleFetchOptimizer implem
// returns non-null FetchTask instance when succeeded
@SuppressWarnings("unchecked")
private FetchTask optimize(ParseContext pctx, String alias,
TableScanOperator source)
- throws HiveException {
+ throws Exception {
String mode = HiveConf.getVar(
pctx.getConf(), HiveConf.ConfVars.HIVEFETCHTASKCONVERSION);
boolean aggressive = "more".equals(mode);
FetchData fetch = checkTree(aggressive, pctx, alias, source);
- if (fetch != null) {
+ if (fetch != null && checkThreshold(fetch, pctx)) {
int limit = pctx.getQB().getParseInfo().getOuterQueryLimit();
FetchWork fetchWork = fetch.convertToWork();
FetchTask fetchTask = (FetchTask) TaskFactory.get(fetchWork,
pctx.getConf());
@@ -110,6 +119,21 @@ public class SimpleFetchOptimizer implem
return null;
}
+ private boolean checkThreshold(FetchData data, ParseContext pctx) throws
Exception {
+ long threshold = HiveConf.getLongVar(pctx.getConf(),
+ HiveConf.ConfVars.HIVEFETCHTASKCONVERSIONTHRESHOLD);
+ if (threshold < 0) {
+ return true;
+ }
+ long remaining = threshold;
+ remaining -= data.getInputLength(pctx, remaining);
+ if (remaining < 0) {
+ LOG.info("Threshold " + remaining + " exceeded for pseudoMR mode");
+ return false;
+ }
+ return true;
+ }
+
// all we can handle is LimitOperator, FilterOperator SelectOperator and
final FS
//
// for non-aggressive mode (minimal)
@@ -144,7 +168,8 @@ public class SimpleFetchOptimizer implem
PrunedPartitionList pruned = pctx.getPrunedPartitions(alias, ts);
if (aggressive || !pruned.hasUnknownPartitions()) {
bypassFilter &= !pruned.hasUnknownPartitions();
- return checkOperators(new FetchData(pruned, splitSample), ts,
aggressive, bypassFilter);
+ return checkOperators(new FetchData(table, pruned, splitSample), ts,
+ aggressive, bypassFilter);
}
}
return null;
@@ -171,6 +196,7 @@ public class SimpleFetchOptimizer implem
}
}
if (op instanceof FileSinkOperator) {
+ fetch.scanOp = ts;
fetch.fileSink = op;
return fetch;
}
@@ -184,6 +210,9 @@ public class SimpleFetchOptimizer implem
private final PrunedPartitionList partsList;
private final HashSet<ReadEntity> inputs = new HashSet<ReadEntity>();
+ // source table scan
+ private TableScanOperator scanOp;
+
// this is always non-null when conversion is completed
private Operator<?> fileSink;
@@ -193,15 +222,15 @@ public class SimpleFetchOptimizer implem
this.splitSample = splitSample;
}
- private FetchData(PrunedPartitionList partsList, SplitSample splitSample) {
- this.table = null;
+ private FetchData(Table table, PrunedPartitionList partsList, SplitSample
splitSample) {
+ this.table = table;
this.partsList = partsList;
this.splitSample = splitSample;
}
private FetchWork convertToWork() throws HiveException {
inputs.clear();
- if (table != null) {
+ if (!table.isPartitioned()) {
inputs.add(new ReadEntity(table));
String path = table.getPath().toString();
FetchWork work = new FetchWork(path, Utilities.getTableDesc(table));
@@ -236,6 +265,56 @@ public class SimpleFetchOptimizer implem
pctx.getSemanticInputs().addAll(inputs);
return replaceFSwithLS(fileSink, work.getSerializationNullFormat());
}
+
+ private long getInputLength(ParseContext pctx, long remaining) throws
Exception {
+ if (splitSample != null && splitSample.getTotalLength() != null) {
+ return splitSample.getTotalLength();
+ }
+ long length = calculateLength(pctx, remaining);
+ if (splitSample != null) {
+ return splitSample.getTargetSize(length);
+ }
+ return length;
+ }
+
+ private long calculateLength(ParseContext pctx, long remaining) throws
Exception {
+ JobConf jobConf = new JobConf(pctx.getConf());
+ Utilities.setColumnNameList(jobConf, scanOp, true);
+ Utilities.setColumnTypeList(jobConf, scanOp, true);
+ HiveStorageHandler handler = table.getStorageHandler();
+ if (handler instanceof InputEstimator) {
+ InputEstimator estimator = (InputEstimator) handler;
+ TableDesc tableDesc = Utilities.getTableDesc(table);
+ PlanUtils.configureInputJobPropertiesForStorageHandler(tableDesc);
+ Utilities.copyTableJobPropertiesToConf(tableDesc, jobConf);
+ return estimator.estimate(jobConf, scanOp, remaining).getTotalLength();
+ }
+ if (table.isNonNative()) {
+ return 0; // nothing can be done
+ }
+ if (!table.isPartitioned()) {
+ return getFileLength(jobConf, table.getPath(),
table.getInputFormatClass());
+ }
+ long total = 0;
+ for (Partition partition : partsList.getNotDeniedPartns()) {
+ Path path = partition.getPartitionPath();
+ total += getFileLength(jobConf, path, partition.getInputFormatClass());
+ }
+ return total;
+ }
+
+ // from Utilities.getInputSummary()
+ private long getFileLength(JobConf conf, Path path, Class<? extends
InputFormat> clazz)
+ throws IOException {
+ ContentSummary summary;
+ if (ContentSummaryInputFormat.class.isAssignableFrom(clazz)) {
+ InputFormat input = HiveInputFormat.getInputFormatFromCache(clazz,
conf);
+ summary = ((ContentSummaryInputFormat)input).getContentSummary(path,
conf);
+ } else {
+ summary = path.getFileSystem(conf).getContentSummary(path);
+ }
+ return summary.getLength();
+ }
}
public static ListSinkOperator replaceFSwithLS(Operator<?> fileSink, String
nullFormat) {
Added: hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q?rev=1540436&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q
(added)
+++ hive/trunk/ql/src/test/queries/clientpositive/nonmr_fetch_threshold.q Sun
Nov 10 03:19:52 2013
@@ -0,0 +1,9 @@
+set hive.fetch.task.conversion=more;
+
+explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10;
+explain select cast(key as int) * 10, upper(value) from src limit 10;
+
+set hive.fetch.task.conversion.threshold=100;
+
+explain select * from srcpart where ds='2008-04-08' AND hr='11' limit 10;
+explain select cast(key as int) * 10, upper(value) from src limit 10;
Added: hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out?rev=1540436&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out
(added)
+++ hive/trunk/ql/src/test/results/clientpositive/nonmr_fetch_threshold.q.out
Sun Nov 10 03:19:52 2013
@@ -0,0 +1,142 @@
+PREHOOK: query: explain select * from srcpart where ds='2008-04-08' AND
hr='11' limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from srcpart where ds='2008-04-08' AND
hr='11' limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR
TOK_ALLCOLREF)) (TOK_WHERE (AND (= (TOK_TABLE_OR_COL ds) '2008-04-08') (=
(TOK_TABLE_OR_COL hr) '11'))) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+ Processor Tree:
+ TableScan
+ alias: srcpart
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ expr: ds
+ type: string
+ expr: hr
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ ListSink
+
+
+PREHOOK: query: explain select cast(key as int) * 10, upper(value) from src
limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select cast(key as int) * 10, upper(value) from src
limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (*
(TOK_FUNCTION TOK_INT (TOK_TABLE_OR_COL key)) 10)) (TOK_SELEXPR (TOK_FUNCTION
upper (TOK_TABLE_OR_COL value)))) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+ Processor Tree:
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: (UDFToInteger(key) * 10)
+ type: int
+ expr: upper(value)
+ type: string
+ outputColumnNames: _col0, _col1
+ Limit
+ ListSink
+
+
+PREHOOK: query: explain select * from srcpart where ds='2008-04-08' AND
hr='11' limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select * from srcpart where ds='2008-04-08' AND
hr='11' limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME srcpart))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR
TOK_ALLCOLREF)) (TOK_WHERE (AND (= (TOK_TABLE_OR_COL ds) '2008-04-08') (=
(TOK_TABLE_OR_COL hr) '11'))) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ srcpart
+ TableScan
+ alias: srcpart
+ Select Operator
+ expressions:
+ expr: key
+ type: string
+ expr: value
+ type: string
+ expr: ds
+ type: string
+ expr: hr
+ type: string
+ outputColumnNames: _col0, _col1, _col2, _col3
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
+PREHOOK: query: explain select cast(key as int) * 10, upper(value) from src
limit 10
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select cast(key as int) * 10, upper(value) from src
limit 10
+POSTHOOK: type: QUERY
+ABSTRACT SYNTAX TREE:
+ (TOK_QUERY (TOK_FROM (TOK_TABREF (TOK_TABNAME src))) (TOK_INSERT
(TOK_DESTINATION (TOK_DIR TOK_TMP_FILE)) (TOK_SELECT (TOK_SELEXPR (*
(TOK_FUNCTION TOK_INT (TOK_TABLE_OR_COL key)) 10)) (TOK_SELEXPR (TOK_FUNCTION
upper (TOK_TABLE_OR_COL value)))) (TOK_LIMIT 10)))
+
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 is a root stage
+
+STAGE PLANS:
+ Stage: Stage-1
+ Map Reduce
+ Alias -> Map Operator Tree:
+ src
+ TableScan
+ alias: src
+ Select Operator
+ expressions:
+ expr: (UDFToInteger(key) * 10)
+ type: int
+ expr: upper(value)
+ type: string
+ outputColumnNames: _col0, _col1
+ Limit
+ File Output Operator
+ compressed: false
+ GlobalTableId: 0
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: 10
+
+
