Author: rding Date: Mon Aug 30 22:57:33 2010 New Revision: 991018 URL: http://svn.apache.org/viewvc?rev=991018&view=rev Log: PIG-1569: java properties not honored in case of properties such as stop.on.failure
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/Main.java hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=991018&r1=991017&r2=991018&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Mon Aug 30 22:57:33 2010 @@ -175,6 +175,9 @@ PIG-1309: Map-side Cogroup (ashutoshc) BUG FIXES +PIG-1569: java properties not honored in case of properties such as +stop.on.failure (rding) + PIG-1570: native mapreduce operator MR job does not follow same failure handling logic as other pig MR jobs (thejas) PIG-1343: pig_log file missing even though Main tells it is creating one and Modified: hadoop/pig/trunk/src/org/apache/pig/Main.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/Main.java?rev=991018&r1=991017&r2=991018&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/Main.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/Main.java Mon Aug 30 22:57:33 2010 @@ -111,10 +111,11 @@ static int run(String args[], PigProgres int rc = 1; GenericOptionsParser parser = new GenericOptionsParser(args); - Configuration conf = parser.getConfiguration(); - Properties properties = ConfigurationUtil.toProperties(conf); + Configuration conf = parser.getConfiguration(); + Properties properties = new Properties(); PropertiesUtil.loadDefaultProperties(properties); + properties.putAll(ConfigurationUtil.toProperties(conf)); String[] pigArgs = parser.getRemainingArgs(); @@ -161,15 +162,21 @@ static int run(String args[], PigProgres if(execTypeString!=null && execTypeString.length()>0){ execType = PigServer.parseExecType(execTypeString); } - - //by default warning aggregation is on - properties.setProperty("aggregate.warning", ""+true); - - //by default multiquery optimization is on - properties.setProperty("opt.multiquery", ""+true); + + if (properties.getProperty("aggregate.warning") == null) { + //by default warning aggregation is on + properties.setProperty("aggregate.warning", ""+true); + } + + if (properties.getProperty("opt.multiquery") == null) { + //by default multiquery optimization is on + properties.setProperty("opt.multiquery", ""+true); + } - //by default we keep going on error on the backend - properties.setProperty("stop.on.failure", ""+false); + if (properties.getProperty("stop.on.failure") == null) { + //by default we keep going on error on the backend + properties.setProperty("stop.on.failure", ""+false); + } // set up client side system properties in UDF context UDFContext.getUDFContext().setClientSystemProps(); Modified: hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=991018&r1=991017&r2=991018&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Mon Aug 30 22:57:33 2010 @@ -353,7 +353,7 @@ public class ScriptState { Attributes attr = attrs.get("org/apache/pig"); pigVersion = attr.getValue("Implementation-Version"); } catch (Exception e) { - LOG.warn("unable to read pigs manifest file", e); + LOG.warn("unable to read pigs manifest file"); } } return (pigVersion == null) ? "" : pigVersion; Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=991018&r1=991017&r2=991018&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java Mon Aug 30 22:57:33 2010 @@ -30,10 +30,12 @@ import java.util.Properties; import junit.framework.Assert; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.pig.ExecType; import org.apache.pig.PigRunner; import org.apache.pig.PigRunner.ReturnCode; +import org.apache.pig.backend.hadoop.datastorage.ConfigurationUtil; import org.apache.pig.impl.PigContext; import org.apache.pig.impl.io.FileLocalizer; import org.apache.pig.newplan.Operator; @@ -72,7 +74,7 @@ public class TestPigRunner { public static void tearDownAfterClass() throws Exception { new File(INPUT_FILE).delete(); cluster.shutDown(); - } + } @Before public void setUp() { @@ -135,12 +137,12 @@ public class TestPigRunner { w.close(); try { - String[] args = { PIG_FILE }; + String[] args = { "-Dstop.on.failure=true", "-Dopt.multiquery=false", "-Daggregate.warning=false", PIG_FILE }; PigStats stats = PigRunner.run(args, new TestNotificationListener()); assertTrue(stats.isSuccessful()); - assertTrue(stats.getJobGraph().size() == 1); + assertEquals(1, stats.getNumberJobs()); String name = stats.getOutputNames().get(0); assertEquals(OUTPUT_FILE, name); assertEquals(12, stats.getBytesWritten()); @@ -148,6 +150,11 @@ public class TestPigRunner { assertEquals("A,B,C", ((JobStats)stats.getJobGraph().getSinks().get(0)).getAlias()); + + Configuration conf = ConfigurationUtil.toConfiguration(stats.getPigProperties()); + assertTrue(conf.getBoolean("stop.on.failure", false)); + assertTrue(!conf.getBoolean("aggregate.warning", true)); + assertTrue(!conf.getBoolean("opt.multiquery", true)); } finally { new File(PIG_FILE).delete(); Util.deleteFile(cluster, OUTPUT_FILE);