Author: gunther
Date: Mon Sep 30 21:36:42 2013
New Revision: 1527791
URL: http://svn.apache.org/r1527791
Log:
HIVE-5387: Need to create edge properties for hive on tez (Vikram Dixit K via
Gunther Hagleitner)
Modified:
hive/branches/tez/eclipse-templates/.classpath
hive/branches/tez/ivy/libraries.properties
hive/branches/tez/ql/ivy.xml
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
Modified: hive/branches/tez/eclipse-templates/.classpath
URL:
http://svn.apache.org/viewvc/hive/branches/tez/eclipse-templates/.classpath?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/eclipse-templates/.classpath (original)
+++ hive/branches/tez/eclipse-templates/.classpath Mon Sep 30 21:36:42 2013
@@ -75,6 +75,7 @@
<classpathentry kind="lib"
path="build/ivy/lib/default/[email protected]@.jar"/>
<classpathentry kind="lib"
path="build/ivy/lib/default/[email protected]@.jar"/>
<classpathentry kind="lib"
path="build/ivy/lib/default/[email protected]@.jar"/>
+ <classpathentry kind="lib"
path="build/ivy/lib/default/[email protected]@.jar"/>
<classpathentry kind="lib"
path="build/ivy/lib/default/[email protected]@.jar"/>
<classpathentry kind="lib"
path="build/ivy/lib/default/[email protected]@.jar"/>
<classpathentry kind="lib"
path="build/ivy/lib/default/[email protected]@.jar"/>
Modified: hive/branches/tez/ivy/libraries.properties
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ivy/libraries.properties?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ivy/libraries.properties (original)
+++ hive/branches/tez/ivy/libraries.properties Mon Sep 30 21:36:42 2013
@@ -40,6 +40,7 @@ commons-lang.version=2.4
commons-logging.version=1.0.4
commons-logging-api.version=1.0.4
commons-pool.version=1.5.4
+commons-lang3.version=3.1
derby.version=10.4.2.0
guava.version=11.0.2
hbase.version=0.94.6.1
Modified: hive/branches/tez/ql/ivy.xml
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/ivy.xml?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/ivy.xml (original)
+++ hive/branches/tez/ql/ivy.xml Mon Sep 30 21:36:42 2013
@@ -66,7 +66,8 @@
<dependency org="org.json" name="json" rev="${json.version}"/>
<dependency org="commons-collections" name="commons-collections"
rev="${commons-collections.version}"/>
<dependency org="commons-configuration" name="commons-configuration"
rev="${commons-configuration.version}"
- transitive="false"/>
+ transitive="false"/>
+ <dependency org="org.apache.commons" name="commons-lang3"
rev="${commons-lang3.version}"/>
<dependency org="com.googlecode.javaewah" name="JavaEWAH"
rev="${javaewah.version}"/>
<dependency org="javolution" name="javolution"
rev="${javolution.version}"/>
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java
Mon Sep 30 21:36:42 2013
@@ -174,10 +174,8 @@ public class TezTask extends Task<TezWor
for (BaseWork v: work.getChildren(w)) {
assert workToVertex.containsKey(v);
Edge e = null;
- EdgeType edgeType = EdgeType.SIMPLE_EDGE;
- if (work.isBroadCastEdge(w, v)) {
- edgeType = EdgeType.BROADCAST_EDGE;
- }
+
+ EdgeType edgeType = work.getEdgeProperty(w, v);
e = DagUtils.createEdge(wxConf, wx, workToConf.get(v),
workToVertex.get(v), edgeType);
dag.addEdge(e);
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
--- hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
(original)
+++ hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/plan/TezWork.java
Mon Sep 30 21:36:42 2013
@@ -25,6 +25,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -49,7 +51,8 @@ public class TezWork extends AbstractOpe
private final Set<BaseWork> leaves = new HashSet<BaseWork>();
private final Map<BaseWork, List<BaseWork>> workGraph = new
HashMap<BaseWork, List<BaseWork>>();
private final Map<BaseWork, List<BaseWork>> invertedWorkGraph = new
HashMap<BaseWork, List<BaseWork>>();
- private final Map<BaseWork, List<BaseWork>> broadcastEdge = new
HashMap<BaseWork, List<BaseWork>>();
+ private final Map<Pair<BaseWork, BaseWork>, EdgeType> edgeProperties =
+ new HashMap<Pair<BaseWork, BaseWork>, EdgeType>();
/**
* getAllWork returns a topologically sorted list of BaseWork
@@ -95,7 +98,6 @@ public class TezWork extends AbstractOpe
}
workGraph.put(w, new LinkedList<BaseWork>());
invertedWorkGraph.put(w, new LinkedList<BaseWork>());
- broadcastEdge.put(w, new LinkedList<BaseWork>());
roots.add(w);
leaves.add(w);
}
@@ -109,13 +111,8 @@ public class TezWork extends AbstractOpe
invertedWorkGraph.get(b).add(a);
roots.remove(b);
leaves.remove(a);
- switch (edgeType) {
- case BROADCAST_EDGE:
- broadcastEdge.get(a).add(b);
- break;
- default:
- break;
- }
+ ImmutablePair workPair = new ImmutablePair(a, b);
+ edgeProperties.put(workPair, edgeType);
}
/**
@@ -201,11 +198,7 @@ public class TezWork extends AbstractOpe
invertedWorkGraph.remove(work);
}
- // checks if a and b need a broadcast edge between them
- public boolean isBroadCastEdge(BaseWork a, BaseWork b) {
- if ((broadcastEdge.get(a).contains(b)) ||
(broadcastEdge.get(b).contains(a))) {
- return true;
- }
- return false;
+ public EdgeType getEdgeProperty(BaseWork a, BaseWork b) {
+ return edgeProperties.get(new ImmutablePair(a,b));
}
}
Modified:
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java?rev=1527791&r1=1527790&r2=1527791&view=diff
==============================================================================
---
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
(original)
+++
hive/branches/tez/ql/src/test/org/apache/hadoop/hive/ql/plan/TestTezWork.java
Mon Sep 30 21:36:42 2013
@@ -77,6 +77,8 @@ public class TestTezWork {
Assert.assertEquals(work.getParents(w).size(), 0);
Assert.assertEquals(work.getChildren(w).size(), 0);
}
+
+ Assert.assertEquals(work.getEdgeProperty(parent, child),
EdgeType.SIMPLE_EDGE);
}
@Test
@@ -99,6 +101,8 @@ public class TestTezWork {
Assert.assertEquals(work.getParents(w).size(), 0);
Assert.assertEquals(work.getChildren(w).size(), 0);
}
+
+ Assert.assertEquals(work.getEdgeProperty(parent, child),
EdgeType.BROADCAST_EDGE);
}
@Test
