Author: gates
Date: Thu Feb  4 00:46:18 2010
New Revision: 906309

URL: http://svn.apache.org/viewvc?rev=906309&view=rev
Log:
PIG-1154: local mode fails when hadoop config directory is specified in 
classpath.


Modified:
    hadoop/pig/trunk/CHANGES.txt
    
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=906309&r1=906308&r2=906309&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Feb  4 00:46:18 2010
@@ -80,6 +80,9 @@
 
 BUG FIXES
 
+PIG-1154: Local Mode fails when hadoop config directory is specified in 
+            classpath (ankit.modi via gates)
+
 PIG-1124: Unable to set Custom Job Name using the -Dmapred.job.name parameter 
(ashutoshc)
 
 PIG-1213: Schema serialization is broken (pradeepkth)

Modified: 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=906309&r1=906308&r2=906309&view=diff
==============================================================================
--- 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
 (original)
+++ 
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
 Thu Feb  4 00:46:18 2010
@@ -18,11 +18,13 @@
 
 package org.apache.pig.backend.hadoop.executionengine;
 
+import java.io.File;
 import java.io.IOException;
 import java.io.PrintStream;
 import java.net.Socket;
 import java.net.SocketException;
 import java.net.SocketImplFactory;
+import java.net.URL;
 import java.util.Collection;
 import java.util.List;
 import java.util.ArrayList;
@@ -51,10 +53,10 @@
 import org.apache.pig.impl.io.FileSpec;
 import org.apache.pig.impl.logicalLayer.LogicalPlan;
 import 
org.apache.pig.backend.hadoop.executionengine.physicalLayer.LogToPhyTranslationVisitor;
-import 
org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
 import org.apache.pig.impl.plan.OperatorKey;
-import 
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
 import 
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
+import 
org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator;
+import 
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
 import 
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
 import org.apache.pig.impl.plan.VisitorException;
 import org.apache.pig.tools.pigstats.PigStats;
@@ -64,6 +66,10 @@
     public static final String JOB_TRACKER_LOCATION = "mapred.job.tracker";
     private static final String FILE_SYSTEM_LOCATION = "fs.default.name";
     
+    private static final String MAPRED_SITE = "mapred-site.xml";
+    private static final String HDFS_SITE = "hdfs-site.xml";
+    private static final String MAPRED_SYS_DIR = "mapred.system.dir";
+    
     private final Log log = LogFactory.getLog(getClass());
     public static final String LOCAL = "local";
     
@@ -158,6 +164,30 @@
         } else {
             properties.setProperty(JOB_TRACKER_LOCATION, LOCAL );
             properties.setProperty(FILE_SYSTEM_LOCATION, "file:///");
+            
+            Configuration testConf = new Configuration();
+            ClassLoader cl = testConf.getClassLoader();
+            URL mapred_site = cl.getResource( MAPRED_SITE );
+            URL hdfs_site = cl.getResource( HDFS_SITE );
+            
+            if( mapred_site != null || hdfs_site != null ) {
+                log.warn( "Passing Hadoop Site Configurations in classpath " +
+                               "is not recommended for Local Mode" );
+            }
+
+            // This is one case. Here we check if mapred.system.dir  
+            // directory is present. This check causes use to print a nice 
error
+            String newMapredSystemDir = testConf.get( MAPRED_SYS_DIR, "" );
+            Configuration defaultConf = new Configuration(false);
+            defaultConf.addResource("core-default.xml");
+            defaultConf.addResource("mapred-default.xml");
+            if( defaultConf.get(MAPRED_SYS_DIR, 
"").compareTo(newMapredSystemDir) != 0 ) {
+                File systemDir = new File(newMapredSystemDir);
+                if( ! systemDir.exists() ) {
+                    throw new ExecException( MAPRED_SYS_DIR + ": " + 
newMapredSystemDir 
+                            + " mentioned in the configuration does not 
exist");
+                }
+            }
         }
         
         cluster = properties.getProperty(JOB_TRACKER_LOCATION);

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java?rev=906309&r1=906308&r2=906309&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestLocal2.java Thu Feb  4 
00:46:18 2010
@@ -202,6 +202,44 @@
         Assert.assertEquals(count, actualCount);
     }
 
+    @Test
+    public void testLocalInit() throws Exception {
+        File pigFile = new File("script.pig");
+        File siteFile = new File("mapred-site.xml");
+        try {
+            pigFile.createNewFile();
+            int status,status2;
+            status = Util.executeShellCommand("java -cp "+ 
+                    System.getProperty("java.class.path") + 
+                    "  org.apache.pig.Main -x local " + 
pigFile.getAbsolutePath() );
+
+            String contents = "<?xml version=\"1.0\"?>\n" +
+            "<?xml-stylesheet type=\"text/xsl\" 
href=\"configuration.xsl\"?>\n" +
+            "<configuration xmlns:xi=\"http://www.w3.org/2001/XInclude\";>\n" +
+            "  <property>\n" +
+            "    <name>mapred.system.dir</name>\n" +
+            "    <value>/mapredsystem/hadoop/mapredsystem</value>\n" +
+            "    <description>No description</description>\n" +
+            "    <final>true</final>\n" +
+            "  </property>\n" +
+            "</configuration>\n";
+            assertTrue( siteFile.createNewFile() );
+            PrintStream ps = new PrintStream(siteFile);
+            ps.print(contents);
+            ps.close();
+            status2 = Util.executeShellCommand("java -cp "+ 
+                    System.getProperty("java.class.path") + 
+                    "  org.apache.pig.Main -x local " + 
pigFile.getAbsolutePath() );
+            assertEquals( "Without a mapred-site.xml pig should just run", 0, 
status );
+            assertEquals( "With map.system.dir redefined in mapred-site.xml 
pig " +
+                    "should exit", 2, status2 );
+        } finally {
+            if( siteFile.exists() ) 
+                siteFile.delete();
+            if( pigFile.exists() )
+                pigFile.delete();
+        }
+    }
 
     /***
      * For generating a sample dataset
@@ -250,5 +288,4 @@
         return TestHelper.createTempFile(data) ;
     }
 
-
 }


Reply via email to