Author: sms
Date: Fri Jan 30 05:05:47 2009
New Revision: 739172
URL: http://svn.apache.org/viewvc?rev=739172&view=rev
Log:
PIG-647: memory sized passed on pig command line does not get propagated to
JobConf
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=739172&r1=739171&r2=739172&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Fri Jan 30 05:05:47 2009
@@ -392,3 +392,6 @@
PIG-645: Streaming is broken with the latest trunk (pradeepkth)
PIG-646: Distinct UDF should report progress (sms)
+
+ PIG-647: memory sized passed on pig command line does not get propagated
+ to JobConf (sms)
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=739172&r1=739171&r2=739172&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
Fri Jan 30 05:05:47 2009
@@ -383,14 +383,33 @@
jobConf.addResource("pig-cluster-hadoop-site.xml");
// We need to load the properties from the hadoop
configuration
- // file we just found in the hod dir. We want these to
override
- // any existing properties we have.
+ // file we just found in the hod dir. We want to
override
+ // these with any existing properties we have.
if (jobConf != null) {
+ Properties hodProperties = new Properties();
Iterator<Map.Entry<String, String>> iter = jobConf.iterator();
while (iter.hasNext()) {
Map.Entry<String, String> entry = iter.next();
- properties.put(entry.getKey(), entry.getValue());
+ hodProperties.put(entry.getKey(), entry.getValue());
}
+
+ //override hod properties with user defined properties
+ Enumeration<Object> propertiesIter = properties.keys();
+ while (propertiesIter.hasMoreElements()) {
+ String key = (String) propertiesIter.nextElement();
+ String val = properties.getProperty(key);
+ hodProperties.put(key, val);
+ }
+
+ //clear user defined properties and re-populate
+ properties.clear();
+ Enumeration<Object> hodPropertiesIter = hodProperties.keys();
+ while (hodPropertiesIter.hasMoreElements()) {
+ String key = (String) hodPropertiesIter.nextElement();
+ String val = hodProperties.getProperty(key);
+ properties.put(key, val);
+ }
+
}
hdfs = properties.getProperty(FILE_SYSTEM_LOCATION);
