Author: pisong
Date: Sat Jul 19 05:40:09 2008
New Revision: 678149
URL: http://svn.apache.org/viewvc?rev=678149&view=rev
Log:
Fixed broken build caused by merging
Added:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java
Added:
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java?rev=678149&view=auto
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
(added)
+++
incubator/pig/branches/types/src/org/apache/pig/backend/hadoop/datastorage/ConfigurationUtil.java
Sat Jul 19 05:40:09 2008
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.backend.hadoop.datastorage;
+
+import java.util.Enumeration;
+import java.util.Iterator;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.hadoop.conf.Configuration;
+
+public class ConfigurationUtil {
+
+ public static Configuration toConfiguration(Properties properties) {
+ assert properties != null;
+ final Configuration config = new Configuration();
+ final Enumeration<Object> iter = properties.keys();
+ while (iter.hasMoreElements()) {
+ final String key = (String) iter.nextElement();
+ final String val = properties.getProperty(key);
+ config.set(key, val);
+ }
+ return config;
+ }
+
+ public static Properties toProperties(Configuration configuration) {
+ Properties properties = new Properties();
+ assert configuration != null;
+ Iterator<Map.Entry<String, String>> iter = configuration.iterator();
+ while (iter.hasNext()) {
+ Map.Entry<String, String> entry = iter.next();
+ properties.put(entry.getKey(), entry.getValue());
+ }
+ return properties;
+ }
+}
Modified:
incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java?rev=678149&r1=678148&r2=678149&view=diff
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java
(original)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/mapReduceLayer/JobControlCompiler.java
Sat Jul 19 05:40:09 2008
@@ -97,10 +97,10 @@
this.pigContext = pigContext;
JobControl jobCtrl = new JobControl(grpName);
- List<MapReduceOper> leaevs = new ArrayList<MapReduceOper>();
- leaevs = plan.getLeaves();
+ List<MapReduceOper> leaves ;
+ leaves = plan.getLeaves();
- for (MapReduceOper mro : leaevs) {
+ for (MapReduceOper mro : leaves) {
jobCtrl.addJob(compile(mro,jobCtrl));
}
return jobCtrl;
@@ -116,8 +116,7 @@
* @throws JobCreationException
*/
private Job compile(MapReduceOper mro, JobControl jobCtrl) throws
JobCreationException {
- List<MapReduceOper> pred = new ArrayList<MapReduceOper>();
- pred = plan.getPredecessors(mro);
+ List<MapReduceOper> pred = plan.getPredecessors(mro);
JobConf currJC = null;
Added:
incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java
URL:
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java?rev=678149&view=auto
==============================================================================
---
incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java
(added)
+++
incubator/pig/branches/types/src/org/apache/pig/impl/util/PropertiesUtil.java
Sat Jul 19 05:40:09 2008
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.impl.util;
+
+import java.io.BufferedInputStream;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.File ;
+import java.util.Enumeration;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+public class PropertiesUtil {
+ private static final String PROPERTIES_FILE = "/pig.properties";
+ private final static Log log = LogFactory.getLog(PropertiesUtil.class);
+
+ public static void loadPropertiesFromFile(Properties properties) {
+ try {
+ Class<PropertiesUtil> clazz = PropertiesUtil.class;
+ InputStream inputStream = clazz
+ .getResourceAsStream(PROPERTIES_FILE);
+ if (inputStream == null) {
+ String msg = "no pig.properties configuration file available
in the classpath";
+ log.debug(msg);
+ } else {
+ properties.load(inputStream);
+ }
+ } catch (Exception e) {
+ log.error("unable to parse pig.properties :", e);
+ }
+
+ Properties pigrcProps = new Properties() ;
+ try {
+ File pigrcFile = new File(System.getProperty("user.home") +
"/.pigrc");
+ if (pigrcFile.exists()) {
+ log.warn(pigrcFile.getAbsolutePath()
+ + " exists but will be deprecated soon. Use
conf/pig.properties instead!");
+ pigrcProps.load(new BufferedInputStream(new
FileInputStream(pigrcFile))) ;
+ }
+ } catch (Exception e) {
+ log.error("unable to parse .pigrc :", e);
+ }
+
+ // Now put all the entries from pigrcProps into properties, but
+ // only if they are not already set. pig.properties takes
+ // precedence over .pigrc
+ Set<Map.Entry<Object, Object>> entries = pigrcProps.entrySet();
+ for (Map.Entry<Object, Object> entry : entries) {
+ if (!properties.containsKey(entry.getKey())) {
+ properties.put(entry.getKey(),
entry.getValue());
+ }
+ }
+
+ //Now set these as system properties only if they are not already
defined.
+ if (log.isDebugEnabled()) {
+ for (Object o: properties.keySet()) {
+ String propertyName = (String) o ;
+ StringBuilder sb = new StringBuilder() ;
+ sb.append("Found property ") ;
+ sb.append(propertyName) ;
+ sb.append("=") ;
+ sb.append(properties.get(propertyName).toString()) ;
+ log.debug(sb.toString()) ;
+ }
+ }
+
+ // Add System properties which include command line overrides
+ // Any existing keys will be overridden
+ properties.putAll(System.getProperties());
+
+ // For telling error fast when there are problems
+ ConfigurationValidator.validatePigProperties(properties) ;
+ }
+
+ public static Properties loadPropertiesFromFile() {
+ Properties properties = new Properties();
+ loadPropertiesFromFile(properties);
+ return properties;
+ }
+
+}
