Author: daijy Date: Wed Apr 14 22:38:23 2010 New Revision: 934230 URL: http://svn.apache.org/viewvc?rev=934230&view=rev Log: PIG-1338: Pig should exclude hadoop conf in local mode (PIG-1338-newhadoop-1.patch)
Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=934230&r1=934229&r2=934230&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java Wed Apr 14 22:38:23 2010 @@ -25,6 +25,7 @@ import java.util.Properties; import java.util.Map.Entry; import org.apache.hadoop.conf.Configuration; +import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil; public class ConfigurationUtil { @@ -62,4 +63,12 @@ public class ConfigurationUtil { } } + + public static Properties getLocalFSProperties() { + Configuration localConf = new Configuration(false); + localConf.addResource("core-default.xml"); + Properties props = ConfigurationUtil.toProperties(localConf); + props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///"); + return props; + } } Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java?rev=934230&r1=934229&r2=934230&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java Wed Apr 14 22:38:23 2010 @@ -320,9 +320,7 @@ public class POFRJoin extends PhysicalOp POLoad ld = new POLoad(new OperatorKey("Repl File Loader", 1L), replFile); - Properties props = new Properties(); - props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///"); - + Properties props = ConfigurationUtil.getLocalFSProperties(); PigContext pc = new PigContext(ExecType.LOCAL, props); ld.setPc(pc); // We use LocalRearrange Operator to seperate Key and Values Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java?rev=934230&r1=934229&r2=934230&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeCogroup.java Wed Apr 14 22:38:23 2010 @@ -35,6 +35,7 @@ import org.apache.pig.IndexableLoadFunc; import org.apache.pig.LoadFunc; import org.apache.pig.PigException; import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigMapReduce; import org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigSplit; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; @@ -378,8 +379,7 @@ public class POMergeCogroup extends Phys new FileSpec(indexFileName, idxFuncSpec)); // Index file is distributed through Distributed Cache to all mappers. So, read it locally. - Properties props = new Properties(); - props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///"); + Properties props = ConfigurationUtil.getLocalFSProperties(); ld.setPc(new PigContext(ExecType.LOCAL, props)); // Each index entry is read as a pair of split index and a tuple consisting of key. Modified: hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java?rev=934230&r1=934229&r2=934230&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java Wed Apr 14 22:38:23 2010 @@ -110,9 +110,7 @@ public class DefaultIndexableLoader exte Object firstLeftKey = (keys.size() == 1 ? keys.get(0): keys); POLoad ld = new POLoad(genKey(), new FileSpec(indexFile, new FuncSpec(indexFileLoadFuncSpec))); - Properties props = new Properties(); - props.setProperty(MapRedUtil.FILE_SYSTEM_NAME, "file:///"); - + Properties props = ConfigurationUtil.getLocalFSProperties(); PigContext pc = new PigContext(ExecType.LOCAL, props); ld.setPc(pc); index = new LinkedList<Tuple>();