Author: xuefu
Date: Tue Dec 16 05:17:09 2014
New Revision: 1645842
URL: http://svn.apache.org/r1645842
Log:
HIVE-9101: bucket_map_join_spark4.q failed due to NPE.[Spark Branch] (Jimmy via
Xuefu)
Modified:
hive/branches/spark/data/conf/spark/hive-site.xml
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out
Modified: hive/branches/spark/data/conf/spark/hive-site.xml
URL:
http://svn.apache.org/viewvc/hive/branches/spark/data/conf/spark/hive-site.xml?rev=1645842&r1=1645841&r2=1645842&view=diff
==============================================================================
--- hive/branches/spark/data/conf/spark/hive-site.xml (original)
+++ hive/branches/spark/data/conf/spark/hive-site.xml Tue Dec 16 05:17:09 2014
@@ -220,4 +220,9 @@
<value>${maven.local.repository}/org/apache/hive/hive-it-util/${hive.version}/hive-it-util-${hive.version}.jar</value>
</property>
+<property>
+ <name>hive.users.in.admin.role</name>
+ <value>hive_admin_user</value>
+</property>
+
</configuration>
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java?rev=1645842&r1=1645841&r2=1645842&view=diff
==============================================================================
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/exec/spark/HashTableLoader.java
Tue Dec 16 05:17:09 2014
@@ -97,13 +97,17 @@ public class HashTableLoader implements
}
String bigInputPath = currentInputPath;
if (currentInputPath != null && mapJoinCtx != null) {
- Set<String> aliases =
-
((SparkBucketMapJoinContext)mapJoinCtx).getPosToAliasMap().get(pos);
- String alias = aliases.iterator().next();
- // Any one small table input path
- String smallInputPath =
-
mapJoinCtx.getAliasBucketFileNameMapping().get(alias).get(bigInputPath).get(0);
- bigInputPath = mapJoinCtx.getMappingBigFile(alias, smallInputPath);
+ if (!desc.isBucketMapJoin()) {
+ bigInputPath = null;
+ } else {
+ Set<String> aliases =
+
((SparkBucketMapJoinContext)mapJoinCtx).getPosToAliasMap().get(pos);
+ String alias = aliases.iterator().next();
+ // Any one small table input path
+ String smallInputPath =
+
mapJoinCtx.getAliasBucketFileNameMapping().get(alias).get(bigInputPath).get(0);
+ bigInputPath = mapJoinCtx.getMappingBigFile(alias, smallInputPath);
+ }
}
String fileName = localWork.getBucketFileName(bigInputPath);
Path path = Utilities.generatePath(baseDir, desc.getDumpFilePrefix(),
(byte)pos, fileName);
Modified:
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java?rev=1645842&r1=1645841&r2=1645842&view=diff
==============================================================================
---
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
(original)
+++
hive/branches/spark/ql/src/java/org/apache/hadoop/hive/ql/optimizer/AbstractBucketJoinProc.java
Tue Dec 16 05:17:09 2014
@@ -240,12 +240,14 @@ abstract public class AbstractBucketJoin
for (Map.Entry<String, Operator<? extends OperatorDesc>> topOpEntry :
topOps.entrySet()) {
if (topOpEntry.getValue() == tso) {
String newAlias = topOpEntry.getKey();
- joinAliases.set(index, newAlias);
- if (baseBigAlias.equals(alias)) {
- baseBigAlias = newAlias;
+ if (!newAlias.equals(alias)) {
+ joinAliases.set(index, newAlias);
+ if (baseBigAlias.equals(alias)) {
+ baseBigAlias = newAlias;
+ }
+ aliasToNewAliasMap.put(alias, newAlias);
+ alias = newAlias;
}
- aliasToNewAliasMap.put(alias, newAlias);
- alias = newAlias;
break;
}
}
Modified:
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out?rev=1645842&r1=1645841&r2=1645842&view=diff
==============================================================================
---
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out
(original)
+++
hive/branches/spark/ql/src/test/results/clientpositive/spark/bucket_map_join_spark4.q.out
Tue Dec 16 05:17:09 2014
@@ -448,6 +448,64 @@ POSTHOOK: Input: default@tbl1
POSTHOOK: Input: default@tbl2
POSTHOOK: Input: default@tbl3
#### A masked pattern was here ####
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+0 val_0 val_0 val_0
+2 val_2 val_2 val_2
+4 val_4 val_4 val_4
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+5 val_5 val_5 val_5
+8 val_8 val_8 val_8
+9 val_9 val_9 val_9
PREHOOK: query: explain extended
select a.key as key, a.value as val1, b.value as val2, c.value as val3
from tbl1 a join tbl2 b on a.key = b.key join tbl3 c on a.value = c.value
