Author: gates Date: Tue May 13 09:35:44 2008 New Revision: 655940 URL: http://svn.apache.org/viewvc?rev=655940&view=rev Log: PIG-236: Fix properties so that values specified via the command line (-D) are not ignored.
Modified: incubator/pig/trunk/CHANGES.txt incubator/pig/trunk/src/org/apache/pig/Main.java incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java Modified: incubator/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/incubator/pig/trunk/CHANGES.txt?rev=655940&r1=655939&r2=655940&view=diff ============================================================================== --- incubator/pig/trunk/CHANGES.txt (original) +++ incubator/pig/trunk/CHANGES.txt Tue May 13 09:35:44 2008 @@ -293,3 +293,5 @@ PIG-232: let valid cache specifications through (acmurthy via olgan) PIG-237: validation of the output directory (pi_song via olgan) + + PIG-236: Fix properties so that values specified via the command line (-D) are not ignored (pkamath via gates). Modified: incubator/pig/trunk/src/org/apache/pig/Main.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/Main.java?rev=655940&r1=655939&r2=655940&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/Main.java (original) +++ incubator/pig/trunk/src/org/apache/pig/Main.java Tue May 13 09:35:44 2008 @@ -206,8 +206,6 @@ } } } - // set the cluster - properties.setProperty(HExecutionEngine.JOB_TRACKER_LOCATION, cluster); // configure logging configureLog4J(properties); // create the context with the parameter Modified: incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=655940&r1=655939&r2=655940&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original) +++ incubator/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Tue May 13 09:35:44 2008 @@ -111,8 +111,9 @@ setSSHFactory(); String hodServer = properties.getProperty(HOD_SERVER); - String cluster = properties.getProperty(JOB_TRACKER_LOCATION); - String nameNode = properties.getProperty( FILE_SYSTEM_LOCATION); + String cluster = null; + String nameNode = null; + Configuration configuration = null; if (hodServer != null && hodServer.length() > 0) { String hdfsAndMapred[] = doHod(hodServer, properties); @@ -120,6 +121,24 @@ properties.setProperty(JOB_TRACKER_LOCATION, hdfsAndMapred[1]); } else { + + // We need to build a configuration object first in the manner described below + // and then get back a properties object to inspect the JOB_TRACKER_LOCATION + // and FILE_SYSTEM_LOCATION. The reason to do this is if we looked only at + // the existing properties object, we may not get the right settings. So we want + // to read the configurations in the order specified below and only then look + // for JOB_TRACKER_LOCATION and FILE_SYSTEM_LOCATION. + + // Hadoop by default specifies two resources, loaded in-order from the classpath: + // 1. hadoop-default.xml : Read-only defaults for hadoop. + // 2. hadoop-site.xml: Site-specific configuration for a given hadoop installation. + // Now add the settings from "properties" object to override any existing properties + // All of the above is accomplished in the method call below + configuration = ConfigurationUtil.toConfiguration(properties); + properties = ConfigurationUtil.toProperties(configuration); + cluster = properties.getProperty(JOB_TRACKER_LOCATION); + nameNode = properties.getProperty(FILE_SYSTEM_LOCATION); + if (cluster != null && cluster.length() > 0) { if(!cluster.contains(":") && !cluster.equalsIgnoreCase(LOCAL)) { cluster = cluster + ":50020"; @@ -137,8 +156,13 @@ log.info("Connecting to hadoop file system at: " + (nameNode==null? LOCAL: nameNode) ) ; ds = new HDataStorage(properties); + + // The above HDataStorage constructor sets DEFAULT_REPLICATION_FACTOR_KEY in properties. + // So we need to reconstruct the configuration object for the non HOD case + // In the HOD case, this is the first time the configuration object will be created + configuration = ConfigurationUtil.toConfiguration(properties); + - Configuration configuration = ConfigurationUtil.toConfiguration(properties); if(cluster != null && !cluster.equalsIgnoreCase(LOCAL)){ log.info("Connecting to map-reduce job tracker at: " + properties.get(JOB_TRACKER_LOCATION)); if (!LOCAL.equalsIgnoreCase(cluster)) { Modified: incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java URL: http://svn.apache.org/viewvc/incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java?rev=655940&r1=655939&r2=655940&view=diff ============================================================================== --- incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java (original) +++ incubator/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java Tue May 13 09:35:44 2008 @@ -66,6 +66,10 @@ } } + // Add System properties which include command line overrides + // Any existing keys will be overridden + properties.putAll(System.getProperties()); + // For telling error fast when there are problems ConfigurationValidator.validatePigProperties(properties) ; }