Author: namit
Date: Mon Oct 24 23:52:51 2011
New Revision: 1188449
URL: http://svn.apache.org/viewvc?rev=1188449&view=rev
Log:
HIVE-1567. Remove hive.mapjoin.maxsize - it was not being used
(Asuhtosh Chauhan via namit)
Modified:
hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
hive/trunk/conf/hive-default.xml
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
Modified: hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
URL:
http://svn.apache.org/viewvc/hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1188449&r1=1188448&r2=1188449&view=diff
==============================================================================
--- hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java
(original)
+++ hive/trunk/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Mon
Oct 24 23:52:51 2011
@@ -344,7 +344,6 @@ public class HiveConf extends Configurat
HIVEMERGEMAPONLY("hive.mergejob.maponly", true),
HIVESENDHEARTBEAT("hive.heartbeat.interval", 1000),
- HIVEMAXMAPJOINSIZE("hive.mapjoin.maxsize", 100000),
HIVELIMITMAXROWSIZE("hive.limit.row.max.size", 100000L),
HIVELIMITOPTLIMITFILE("hive.limit.optimize.limit.file", 10),
HIVELIMITOPTENABLE("hive.limit.optimize.enable", false),
Modified: hive/trunk/conf/hive-default.xml
URL:
http://svn.apache.org/viewvc/hive/trunk/conf/hive-default.xml?rev=1188449&r1=1188448&r2=1188449&view=diff
==============================================================================
--- hive/trunk/conf/hive-default.xml (original)
+++ hive/trunk/conf/hive-default.xml Mon Oct 24 23:52:51 2011
@@ -417,12 +417,6 @@
</property>
<property>
- <name>hive.mapjoin.maxsize</name>
- <value>100000</value>
- <description>Maximum # of rows of the small table that can be handled by
map-side join. If the size is reached and hive.task.progress is set, a fatal
error counter is set and the job will be killed.</description>
-</property>
-
-<property>
<name>hive.mapjoin.cache.numrows</name>
<value>25000</value>
<description>How many rows should be cached by jdbm for map join.
</description>
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java?rev=1188449&r1=1188448&r2=1188449&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/AbstractMapJoinOperator.java
Mon Oct 24 23:52:51 2011
@@ -63,8 +63,8 @@ public abstract class AbstractMapJoinOpe
private static final transient String[] FATAL_ERR_MSG = {
null, // counter value 0 means no error
- "Mapside join size exceeds hive.mapjoin.maxsize. "
- + "Please increase that or remove the mapjoin hint."
+ "Mapside join exceeds available memory. "
+ + "Please try removing the mapjoin hint."
};
transient boolean firstRow;
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java?rev=1188449&r1=1188448&r2=1188449&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/HashTableSinkOperator.java
Mon Oct 24 23:52:51 2011
@@ -46,8 +46,8 @@ import org.apache.hadoop.hive.serde2.Ser
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorFactory;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils;
-import
org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.ObjectInspectorUtils.ObjectInspectorCopyOption;
+import
org.apache.hadoop.hive.serde2.objectinspector.StandardStructObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
import org.apache.hadoop.util.ReflectionUtils;
@@ -162,11 +162,10 @@ public class HashTableSinkOperator exten
private static final transient String[] FATAL_ERR_MSG = {
null, // counter value 0 means no error
- "Mapside join size exceeds hive.mapjoin.maxsize. "
- + "Please increase that or remove the mapjoin hint."};
+ "Mapside join exceeds available memory. "
+ + "Please try removing the mapjoin hint."};
private final int metadataKeyTag = -1;
transient int[] metadataValueTag;
- transient int maxMapJoinSize;
public HashTableSinkOperator() {
@@ -181,8 +180,6 @@ public class HashTableSinkOperator exten
protected void initializeOp(Configuration hconf) throws HiveException {
boolean isSilent = HiveConf.getBoolVar(hconf,
HiveConf.ConfVars.HIVESESSIONSILENT);
console = new LogHelper(LOG, isSilent);
- maxMapJoinSize = HiveConf.getIntVar(hconf,
HiveConf.ConfVars.HIVEMAXMAPJOINSIZE);
-
numMapRowsRead = 0;
firstRow = true;
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1188449&r1=1188448&r2=1188449&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
Mon Oct 24 23:52:51 2011
@@ -57,13 +57,12 @@ public class MapJoinOperator extends Abs
private static final transient String[] FATAL_ERR_MSG = {
null, // counter value 0 means no error
- "Mapside join size exceeds hive.mapjoin.maxsize. "
- + "Please increase that or remove the mapjoin hint."};
+ "Mapside join exceeds available memory. "
+ + "Please try removing the mapjoin hint."};
protected transient Map<Byte, MapJoinRowContainer<ArrayList<Object>>>
rowContainerMap;
transient int metadataKeyTag;
transient int[] metadataValueTag;
- transient int maxMapJoinSize;
transient boolean hashTblInitedOnce;
private int bigTableAlias;
@@ -79,8 +78,6 @@ public class MapJoinOperator extends Abs
super.initializeOp(hconf);
- maxMapJoinSize = HiveConf.getIntVar(hconf,
HiveConf.ConfVars.HIVEMAXMAPJOINSIZE);
-
metadataValueTag = new int[numAliases];
for (int pos = 0; pos < numAliases; pos++) {
metadataValueTag[pos] = -1;
@@ -153,7 +150,7 @@ public class MapJoinOperator extends Abs
hashTblInitedOnce = true;
}
}
-
+
boolean localMode = HiveConf.getVar(hconf,
HiveConf.ConfVars.HADOOPJT).equals("local");
String baseDir = null;
