Author: gunther
Date: Wed Dec 18 18:56:10 2013
New Revision: 1552060
URL: http://svn.apache.org/r1552060
Log:
HIVE-6055: Cleanup aisle tez (Gunther Hagleitner)
Modified:
hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
Modified:
hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/branches/tez/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
Wed Dec 18 18:56:10 2013
@@ -872,7 +872,7 @@ public class HiveConf extends Configurat
HIVE_RPC_QUERY_PLAN("hive.rpc.query.plan", false),
// Whether to generate the splits locally or in the AM (tez only)
- HIVE_AM_SPLIT_GENERATION("hive.compute.splits.in.am", true),
+ HIVE_AM_SPLIT_GENERATION("hive.compute.splits.in.am", true),
// none, idonly, traverse, execution
HIVESTAGEIDREARRANGE("hive.stageid.rearrange", "none"),
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/CommonJoinOperator.java
Wed Dec 18 18:56:10 2013
@@ -34,8 +34,6 @@ import org.apache.hadoop.hive.ql.exec.pe
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.JoinCondDesc;
import org.apache.hadoop.hive.ql.plan.JoinDesc;
-import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/DemuxOperator.java
Wed Dec 18 18:56:10 2013
@@ -27,11 +27,9 @@ import java.util.Map.Entry;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.DemuxDesc;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
-import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.api.OperatorType;
import org.apache.hadoop.hive.serde2.Deserializer;
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/TableScanOperator.java
Wed Dec 18 18:56:10 2013
@@ -27,12 +27,10 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hive.common.FileUtils;
-import org.apache.hadoop.hive.conf.HiveConf;
import org.apache.hadoop.hive.common.StatsSetupConst;
import org.apache.hadoop.hive.ql.ErrorMsg;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
-import org.apache.hadoop.hive.ql.plan.Statistics;
import org.apache.hadoop.hive.ql.plan.OperatorDesc;
import org.apache.hadoop.hive.ql.plan.TableDesc;
import org.apache.hadoop.hive.ql.plan.TableScanDesc;
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java
Wed Dec 18 18:56:10 2013
@@ -3163,59 +3163,6 @@ public final class Utilities {
}
}
- public static long getSize(String alias, Table table, HiveConf conf,
- TableScanOperator topOp, ExprNodeDesc expr) throws HiveException {
- long result = 0;
- int numPartitions = 0;
- Map<String, PrunedPartitionList> prunedPartitionsMap
- = new HashMap<String, PrunedPartitionList>();
-
- if (!table.isPartitioned()) {
- result = getSize(conf, table);
- }
- else {
- // For partitioned tables, get the size of all the partitions
- PrunedPartitionList partsList = PartitionPruner.prune(table, expr, conf,
- alias, prunedPartitionsMap);
- numPartitions = partsList.getNotDeniedPartns().size();
- for (Partition part : partsList.getNotDeniedPartns()) {
- result += getSize(conf, part);
- }
- }
- return result;
- }
-
- private static long getSize(HiveConf conf, String size, Path path) {
- // If the size is present in the metastore, use it
- if (size != null) {
- try {
- return Long.valueOf(size);
- } catch (NumberFormatException e) {
- return -1;
- }
- }
-
- try {
- FileSystem fs = path.getFileSystem(conf);
- return fs.getContentSummary(path).getLength();
- } catch (Exception e) {
- return -1;
- }
- }
-
- private static long getSize(HiveConf conf, Table table) {
- Path path = table.getPath();
- String size = table.getProperty("totalSize");
- return getSize(conf, size, path);
- }
-
- private static long getSize(HiveConf conf, Partition partition) {
- Path path = partition.getPartitionPath();
- String size = partition.getParameters().get("totalSize");
-
- return getSize(conf, size, path);
- }
-
public static void clearWorkMap() {
gWorkMap.clear();
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/PrunerOperatorFactory.java
Wed Dec 18 18:56:10 2013
@@ -125,9 +125,6 @@ public abstract class PrunerOperatorFact
// Put the mapping from table scan operator to pruner_pred
opToPrunner.put(top, pruner_pred);
- // Set the predicate in the table directly
- top.getConf().setPruningPredicate(pruner_pred);
-
return;
}
@@ -168,9 +165,6 @@ public abstract class PrunerOperatorFact
// Put the mapping from table scan operator to part-pruner map
opToPrunner.put(top, partToPruner);
- // Set the predicate in the table directly
- top.getConf().setPruningPredicate(pruner_pred);
-
return;
}
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/optimizer/ppr/PartitionPruner.java
Wed Dec 18 18:56:10 2013
@@ -155,7 +155,7 @@ public class PartitionPruner implements
* pruner condition.
* @throws HiveException
*/
- public static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr,
+ private static PrunedPartitionList prune(Table tab, ExprNodeDesc prunerExpr,
HiveConf conf, String alias, Map<String, PrunedPartitionList>
prunedPartitionsMap)
throws HiveException {
LOG.trace("Started pruning partiton");
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java
Wed Dec 18 18:56:10 2013
@@ -8352,9 +8352,6 @@ public class SemanticAnalyzer extends Ba
// Add a mapping from the table scan operator to Table
topToTable.put((TableScanOperator) top, tab);
- // set the table in the tablescan descriptor directly
- ((TableScanOperator) top).getConf().setTable(tab);
-
Map<String, String> props = qb.getTabPropsForAlias(alias);
if (props != null) {
topToTableProps.put((TableScanOperator) top, props);
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java?rev=1552060&r1=1552059&r2=1552060&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TableScanDesc.java
Wed Dec 18 18:56:10 2013
@@ -22,8 +22,6 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Map;
-import org.apache.hadoop.hive.ql.exec.PTFUtils;
-import org.apache.hadoop.hive.ql.metadata.Table;
import org.apache.hadoop.hive.ql.metadata.VirtualColumn;
/**
@@ -54,10 +52,6 @@ public class TableScanDesc extends Abstr
*/
private int rowLimit = -1;
- private transient Table table;
-
- private transient ExprNodeDesc pruningPredicate;
-
/**
* A boolean variable set to true by the semantic analyzer only in case of
the analyze command.
*
@@ -77,32 +71,10 @@ public class TableScanDesc extends Abstr
// input file name (big) to bucket number
private Map<String, Integer> bucketFileNameMapping;
- static{
- PTFUtils.makeTransient(TableScanDesc.class, "pruningPredicate");
- PTFUtils.makeTransient(TableScanDesc.class, "table");
- }
-
-
@SuppressWarnings("nls")
public TableScanDesc() {
}
- public Table getTable() {
- return table;
- }
-
- public void setTable(Table t) {
- table = t;
- }
-
- public ExprNodeDesc getPruningPredicate() {
- return pruningPredicate;
- }
-
- public void setPruningPredicate(ExprNodeDesc expr) {
- pruningPredicate = expr;
- }
-
public TableScanDesc(final String alias) {
this.alias = alias;
}
