Author: daijy
Date: Wed May 26 17:17:55 2010
New Revision: 948504
URL: http://svn.apache.org/viewvc?rev=948504&view=rev
Log:
PIG-1381: Need a way for Pig to take an alternative property file (option 1)
Modified:
hadoop/pig/trunk/src/org/apache/pig/Main.java
hadoop/pig/trunk/src/org/apache/pig/PigServer.java
hadoop/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java
Modified: hadoop/pig/trunk/src/org/apache/pig/Main.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/Main.java?rev=948504&r1=948503&r2=948504&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/Main.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/Main.java Wed May 26 17:17:55 2010
@@ -101,7 +101,7 @@ public static void main(String args[])
{
int rc = 1;
Properties properties = new Properties();
- PropertiesUtil.loadPropertiesFromFile(properties);
+ PropertiesUtil.loadDefaultProperties(properties);
boolean verbose = false;
boolean gruntCalled = false;
@@ -136,6 +136,7 @@ public static void main(String args[])
opts.registerOpt('x', "exectype",
CmdLineParser.ValueExpected.REQUIRED);
opts.registerOpt('F', "stop_on_failure",
CmdLineParser.ValueExpected.NOT_ACCEPTED);
opts.registerOpt('M', "no_multiquery",
CmdLineParser.ValueExpected.NOT_ACCEPTED);
+ opts.registerOpt('P', "propertyFile",
CmdLineParser.ValueExpected.REQUIRED);
ExecMode mode = ExecMode.UNKNOWN;
String file = null;
@@ -266,6 +267,10 @@ public static void main(String args[])
throw new RuntimeException("ERROR: Unrecognized
exectype.", e);
}
break;
+ case 'P':
+ PropertiesUtil.loadPropertiesFromFile(properties,
+ opts.getValStr());
+ break;
default: {
Character cc = Character.valueOf(opt);
throw new AssertionError("Unhandled option " + cc.toString());
@@ -610,7 +615,7 @@ public static void usage()
System.out.println(" options include:");
System.out.println(" -4, -log4jconf log4j configuration file,
overrides log conf");
System.out.println(" -b, -brief brief logging (no timestamps)");
- System.out.println(" -c, -cluster clustername, kryptonite is
default");
+ System.out.println(" -c, -check syntax check");
System.out.println(" -d, -debug debug level, INFO is default");
System.out.println(" -e, -execute commands to execute (within
quotes)");
System.out.println(" -f, -file path to the script to execute");
@@ -628,6 +633,7 @@ public static void usage()
System.out.println(" -F, -stop_on_failure aborts execution on the
first failed job; off by default");
System.out.println(" -M, -no_multiquery turn multiquery
optimization off; Multiquery is on by default");
+ System.out.println(" -P, -propertyFile path to property file");
}
private static String validateLogFile(String logFileName, String scriptName) {
Modified: hadoop/pig/trunk/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigServer.java?rev=948504&r1=948503&r2=948504&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/PigServer.java Wed May 26 17:17:55 2010
@@ -195,7 +195,7 @@ public class PigServer {
* @throws ExecException
*/
public PigServer(ExecType execType) throws ExecException {
- this(execType, PropertiesUtil.loadPropertiesFromFile());
+ this(execType, PropertiesUtil.loadDefaultProperties());
}
public PigServer(ExecType execType, Properties properties) throws
ExecException {
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java?rev=948504&r1=948503&r2=948504&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/util/PropertiesUtil.java Wed May
26 17:17:55 2010
@@ -19,13 +19,10 @@
package org.apache.pig.impl.util;
import java.io.BufferedInputStream;
+import java.io.File;
import java.io.FileInputStream;
import java.io.InputStream;
-import java.io.File ;
-import java.util.Enumeration;
-import java.util.Map;
import java.util.Properties;
-import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -35,66 +32,17 @@ public class PropertiesUtil {
private static final String PROPERTIES_FILE = "/pig.properties";
private final static Log log = LogFactory.getLog(PropertiesUtil.class);
- public static void loadPropertiesFromFile(Properties properties) {
- InputStream inputStream = null;
- BufferedInputStream bis = null;
- Class<PropertiesUtil> clazz = PropertiesUtil.class;
- try {
- inputStream = clazz
- .getResourceAsStream(DEFAULT_PROPERTIES_FILE);
- if (inputStream == null) {
- String msg = "no pig-default.properties configuration file
available in the classpath";
- log.debug(msg);
- } else {
- properties.load(inputStream);
- }
- } catch (Exception e) {
- log.error("unable to parse pig-default.properties :", e);
- } finally {
- if (inputStream != null) try {inputStream.close();} catch
(Exception e) {}
- }
-
- try {
- inputStream = null;
- inputStream = clazz
- .getResourceAsStream(PROPERTIES_FILE);
- if (inputStream == null) {
- String msg = "no pig.properties configuration file available
in the classpath";
- log.debug(msg);
- } else {
- properties.load(inputStream);
- }
- } catch (Exception e) {
- log.error("unable to parse pig.properties :", e);
- } finally {
- if (inputStream != null) try {inputStream.close();} catch
(Exception e) {}
- }
-
- Properties pigrcProps = new Properties() ;
- try {
- File pigrcFile = new File(System.getProperty("user.home") +
"/.pigrc");
- if (pigrcFile.exists()) {
- log.warn(pigrcFile.getAbsolutePath()
- + " exists but will be deprecated soon. Use
conf/pig.properties instead!");
-
- bis = new BufferedInputStream(new FileInputStream(pigrcFile));
- pigrcProps.load(bis) ;
- }
- } catch (Exception e) {
- log.error("unable to parse .pigrc :", e);
- } finally {
- if (bis != null) try {bis.close();} catch (Exception e) {}
- }
-
- // Now put all the entries from pigrcProps into properties, but
- // only if they are not already set. pig.properties takes
- // precedence over .pigrc
- Set<Map.Entry<Object, Object>> entries = pigrcProps.entrySet();
- for (Map.Entry<Object, Object> entry : entries) {
- if (!properties.containsKey(entry.getKey())) {
- properties.put(entry.getKey(),
entry.getValue());
- }
- }
+ /**
+ * Loads the default properties from pig-default.properties and
+ * pig.properties.
+ * @param properties Properties object that needs to be loaded with the
+ * properties' values.
+ */
+ public static void loadDefaultProperties(Properties properties) {
+ loadPropertiesFromFile(properties,
+ System.getProperty("user.home") + "/.pigrc");
+ loadPropertiesFromClasspath(properties, DEFAULT_PROPERTIES_FILE);
+ loadPropertiesFromClasspath(properties, PROPERTIES_FILE);
//Now set these as system properties only if they are not already
defined.
if (log.isDebugEnabled()) {
@@ -117,9 +65,71 @@ public class PropertiesUtil {
ConfigurationValidator.validatePigProperties(properties) ;
}
- public static Properties loadPropertiesFromFile() {
+ /**
+ * Loads the properties from a given file.
+ * @param properties Properties object that is to be loaded.
+ * @param fileName file name of the file that contains the properties.
+ */
+ public static void loadPropertiesFromFile(Properties properties,
+ String fileName) {
+ BufferedInputStream bis = null;
+ Properties pigrcProps = new Properties() ;
+ try {
+ File pigrcFile = new File(fileName);
+ if (pigrcFile.exists()) {
+ if (fileName.endsWith("/.pigrc")) {
+ log.warn(pigrcFile.getAbsolutePath()
+ + " exists but will be deprecated soon." +
+ " Use conf/pig.properties instead!");
+ }
+
+ bis = new BufferedInputStream(new FileInputStream(pigrcFile));
+ pigrcProps.load(bis) ;
+ }
+ } catch (Exception e) {
+ log.error("unable to parse .pigrc :", e);
+ } finally {
+ if (bis != null) try {bis.close();} catch (Exception e) {}
+ }
+
+ properties.putAll(pigrcProps);
+ }
+
+ /**
+ * Finds the file with the given file name in the classpath and loads the
+ * properties provided in it.
+ * @param properties the properties object that needs to be loaded with the
+ * property values provided in the file.
+ * @param fileName file name of the properties' file.
+ */
+ private static void loadPropertiesFromClasspath(Properties properties,
+ String fileName) {
+ InputStream inputStream = null;
+ Class<PropertiesUtil> clazz = PropertiesUtil.class;
+ try {
+ inputStream = clazz
+ .getResourceAsStream(fileName);
+ if (inputStream == null) {
+ String msg = "no " + fileName +
+ " configuration file available in the classpath";
+ log.debug(msg);
+ } else {
+ properties.load(inputStream);
+ }
+ } catch (Exception e) {
+ log.error("unable to parse " + fileName + " :", e);
+ } finally {
+ if (inputStream != null) try {inputStream.close();} catch
(Exception e) {}
+ }
+ }
+
+ /**
+ * Loads default properties.
+ * @return
+ */
+ public static Properties loadDefaultProperties() {
Properties properties = new Properties();
- loadPropertiesFromFile(properties);
+ loadDefaultProperties(properties);
return properties;
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=948504&r1=948503&r2=948504&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java Wed May 26
17:17:55 2010
@@ -626,25 +626,36 @@ public class TestPigServer extends TestC
public void testPigProperties() throws Throwable {
File defaultPropertyFile = new File("pig-default.properties");
File propertyFile = new File("pig.properties");
+ File cliPropertyFile = new File("commandLine_pig.properties");
- Properties properties = PropertiesUtil.loadPropertiesFromFile();
+ Properties properties = PropertiesUtil.loadDefaultProperties();
assertTrue(properties.getProperty("test123")==null);
PrintWriter out = new PrintWriter(new FileWriter(defaultPropertyFile));
out.println("test123=defaultproperties");
out.close();
- properties = PropertiesUtil.loadPropertiesFromFile();
+ properties = PropertiesUtil.loadDefaultProperties();
assertTrue(properties.getProperty("test123").equals("defaultproperties"));
out = new PrintWriter(new FileWriter(propertyFile));
out.println("test123=properties");
out.close();
- properties = PropertiesUtil.loadPropertiesFromFile();
+ properties = PropertiesUtil.loadDefaultProperties();
assertTrue(properties.getProperty("test123").equals("properties"));
+ out = new PrintWriter(new FileWriter(cliPropertyFile));
+ out.println("test123=cli_properties");
+ out.close();
+
+ properties = PropertiesUtil.loadDefaultProperties();
+ PropertiesUtil.loadPropertiesFromFile(properties,
+ "commandLine_pig.properties");
+ assertTrue(properties.getProperty("test123").equals("cli_properties"));
+
defaultPropertyFile.delete();
propertyFile.delete();
+ cliPropertyFile.delete();
}
}
