Author: pisong Date: Sat Jul 19 05:40:09 2008 New Revision: 678149 URL: http://svn.apache.org/viewvc?rev=678149&view=rev Log: Fixed broken build caused by merging
Added: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java Modified: incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java Added: incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=678149&view=auto ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java (added) +++ incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java Sat Jul 19 05:40:09 2008 @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pig.backend.hadoop.datastorage; + +import java.util.Enumeration; +import java.util.Iterator; +import java.util.Map; +import java.util.Properties; + +import org.apache.hadoop.conf.Configuration; + +public class ConfigurationUtil { + + public static Configuration toConfiguration(Properties properties) { + assert properties != null; + final Configuration config = new Configuration(); + final Enumeration<Object> iter = properties.keys(); + while (iter.hasMoreElements()) { + final String key = (String) iter.nextElement(); + final String val = properties.getProperty(key); + config.set(key, val); + } + return config; + } + + public static Properties toProperties(Configuration configuration) { + Properties properties = new Properties(); + assert configuration != null; + Iterator<Map.Entry<String, String>> iter = configuration.iterator(); + while (iter.hasNext()) { + Map.Entry<String, String> entry = iter.next(); + properties.put(entry.getKey(), entry.getValue()); + } + return properties; + } +} Modified: incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java?rev=678149&r1=678148&r2=678149&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java Sat Jul 19 05:40:09 2008 @@ -97,10 +97,10 @@ this.pigContext = pigContext; JobControl jobCtrl = new JobControl(grpName); - List<MapReduceOper> leaevs = new ArrayList<MapReduceOper>(); - leaevs = plan.getLeaves(); + List<MapReduceOper> leaves ; + leaves = plan.getLeaves(); - for (MapReduceOper mro : leaevs) { + for (MapReduceOper mro : leaves) { jobCtrl.addJob(compile(mro,jobCtrl)); } return jobCtrl; @@ -116,8 +116,7 @@ * @throws JobCreationException */ private Job compile(MapReduceOper mro, JobControl jobCtrl) throws JobCreationException { - List<MapReduceOper> pred = new ArrayList<MapReduceOper>(); - pred = plan.getPredecessors(mro); + List<MapReduceOper> pred = plan.getPredecessors(mro); JobConf currJC = null; Added: incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java?rev=678149&view=auto ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java (added) +++ incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java Sat Jul 19 05:40:09 2008 @@ -0,0 +1,101 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.pig.impl.util; + +import java.io.BufferedInputStream; +import java.io.FileInputStream; +import java.io.InputStream; +import java.io.File ; +import java.util.Enumeration; +import java.util.Map; +import java.util.Properties; +import java.util.Set; + +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; + +public class PropertiesUtil { + private static final String PROPERTIES_FILE = "/pig.properties"; + private final static Log log = LogFactory.getLog(PropertiesUtil.class); + + public static void loadPropertiesFromFile(Properties properties) { + try { + Class<PropertiesUtil> clazz = PropertiesUtil.class; + InputStream inputStream = clazz + .getResourceAsStream(PROPERTIES_FILE); + if (inputStream == null) { + String msg = "no pig.properties configuration file available in the classpath"; + log.debug(msg); + } else { + properties.load(inputStream); + } + } catch (Exception e) { + log.error("unable to parse pig.properties :", e); + } + + Properties pigrcProps = new Properties() ; + try { + File pigrcFile = new File(System.getProperty("user.home") + "/.pigrc"); + if (pigrcFile.exists()) { + log.warn(pigrcFile.getAbsolutePath() + + " exists but will be deprecated soon. Use conf/pig.properties instead!"); + pigrcProps.load(new BufferedInputStream(new FileInputStream(pigrcFile))) ; + } + } catch (Exception e) { + log.error("unable to parse .pigrc :", e); + } + + // Now put all the entries from pigrcProps into properties, but + // only if they are not already set. pig.properties takes + // precedence over .pigrc + Set<Map.Entry<Object, Object>> entries = pigrcProps.entrySet(); + for (Map.Entry<Object, Object> entry : entries) { + if (!properties.containsKey(entry.getKey())) { + properties.put(entry.getKey(), entry.getValue()); + } + } + + //Now set these as system properties only if they are not already defined. + if (log.isDebugEnabled()) { + for (Object o: properties.keySet()) { + String propertyName = (String) o ; + StringBuilder sb = new StringBuilder() ; + sb.append("Found property ") ; + sb.append(propertyName) ; + sb.append("=") ; + sb.append(properties.get(propertyName).toString()) ; + log.debug(sb.toString()) ; + } + } + + // Add System properties which include command line overrides + // Any existing keys will be overridden + properties.putAll(System.getProperties()); + + // For telling error fast when there are problems + ConfigurationValidator.validatePigProperties(properties) ; + } + + public static Properties loadPropertiesFromFile() { + Properties properties = new Properties(); + loadPropertiesFromFile(properties); + return properties; + } + +}