Author: daijy
Date: Wed Jun 24 05:10:33 2009
New Revision: 787908
URL: http://svn.apache.org/viewvc?rev=787908&view=rev
Log:
PIG-832: Make import list configurable
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/Main.java
hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java
hadoop/pig/trunk/test/org/apache/pig/test/Util.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Jun 24 05:10:33 2009
@@ -26,6 +26,8 @@
IMPROVEMENTS
+PIG-832: Make import list configurable (daijy)
+
PIG-697: Proposed improvements to pig's optimizer (sms)
PIG-753: Allow UDFs with no parameters (zjffdu via gates).
@@ -34,6 +36,10 @@
BUG FIXES
+ PIG-797: Limit with ORDER BY producing wrong results (daijy)
+
+ PIG-850: Dump produce wrong result while "store into" is ok (daijy)
+
PIG-852: pig -version or pig -help returns exit code of 1 (milindb via
olgan)
@@ -88,10 +94,6 @@
BUG FIXES
-PIG-797: Limit with ORDER BY producing wrong results (daijy)
-
-PIG-850: Dump produce wrong result while "store into" is ok (daijy)
-
PIG-846: MultiQuery optimization in some cases has an issue when there is a
split in the map plan (pradeepkth)
Modified: hadoop/pig/trunk/src/org/apache/pig/Main.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/Main.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/Main.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/Main.java Wed Jun 24 05:10:33 2009
@@ -272,6 +272,9 @@
if(optimizerRules.size() > 0) {
pigContext.getProperties().setProperty("pig.optimizer.rules",
ObjectSerializer.serialize(optimizerRules));
}
+
+ if (properties.get("udf.import.list")!=null)
+
PigContext.initializeImportList((String)properties.get("udf.import.list"));
LogicalPlanBuilder.classloader = pigContext.createCl(null);
Modified: hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java Wed Jun 24
05:10:33 2009
@@ -34,6 +34,7 @@
import java.util.List;
import java.util.Map;
import java.util.Properties;
+import java.util.StringTokenizer;
import java.util.Vector;
import org.apache.commons.logging.Log;
@@ -156,10 +157,23 @@
static{
packageImportList.add("");
packageImportList.add("org.apache.pig.builtin.");
- packageImportList.add("com.yahoo.pig.yst.sds.ULT.");
packageImportList.add("org.apache.pig.impl.builtin.");
}
+ public static void initializeImportList(String
importListCommandLineProperties)
+ {
+ StringTokenizer tokenizer = new
StringTokenizer(importListCommandLineProperties, ":");
+ int pos = 0;
+ while (tokenizer.hasMoreTokens())
+ {
+ String importItem = tokenizer.nextToken();
+ if (!importItem.endsWith("."))
+ importItem += ".";
+ packageImportList.add(pos, importItem);
+ pos++;
+ }
+ }
+
public void connect() throws ExecException {
switch (execType) {
@@ -614,4 +628,9 @@
return PigException.BUG;
}
}
+
+ public static ArrayList<String> getPackageImportList()
+ {
+ return packageImportList;
+ }
}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java Wed Jun 24
05:10:33 2009
@@ -20,7 +20,9 @@
import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
+import java.io.OutputStreamWriter;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
@@ -90,6 +92,58 @@
Object object =
PigContext.instantiateFuncFromSpec("org.apache.hadoop.mapred.FileAlreadyExistsException");
assertTrue(object instanceof FileAlreadyExistsException);
}
+
+ @Test
+ // See PIG-832
+ public void testImportList() throws Exception {
+ String FILE_SEPARATOR = System.getProperty("file.separator");
+ File tmpDir = File.createTempFile("test", "");
+ tmpDir.delete();
+ tmpDir.mkdir();
+
+ File tempDir = new File(tmpDir.getAbsolutePath());
+ Util.deleteDirectory(tempDir);
+ File udf1Dir = new
File(tmpDir.getAbsolutePath()+FILE_SEPARATOR+"com"+FILE_SEPARATOR+"xxx"+FILE_SEPARATOR+"udf1");
+ udf1Dir.mkdirs();
+ File udf1JavaSrc = new
File(udf1Dir.getAbsolutePath()+FILE_SEPARATOR+"TestUDF.java");
+ String udf1Src = new String("package com.xxx.udf1;\n" +
+ "public class TestUDF {}\n");
+
+ // generate java file
+ FileOutputStream outStream =
+ new FileOutputStream(udf1JavaSrc);
+
+ OutputStreamWriter outWriter = new OutputStreamWriter(outStream);
+ outWriter.write(udf1Src);
+ outWriter.close();
+
+ // compile
+ int status;
+ status = Util.executeShellCommand("javac " + udf1JavaSrc);
+
+ // generate jar file
+ String jarName = "TestUDFJar1.jar";
+ status = Util.executeShellCommand("jar -cf " +
tmpDir.getAbsolutePath() + FILE_SEPARATOR + jarName +
+ " -C " + tmpDir.getAbsolutePath() + " " + "com");
+ assertTrue(status==0);
+
+ PigServer pig = new PigServer(pigContext);
+ pig.registerJar(tmpDir.getAbsolutePath() + FILE_SEPARATOR + jarName);
+
+ PigContext.initializeImportList("com.xxx.udf1:com.xxx.udf2.");
+ ArrayList<String> importList = PigContext.getPackageImportList();
+ assertTrue(importList.size()==5);
+ assertTrue(importList.get(0).equals("com.xxx.udf1."));
+ assertTrue(importList.get(1).equals("com.xxx.udf2."));
+ assertTrue(importList.get(2).equals(""));
+ assertTrue(importList.get(3).equals("org.apache.pig.builtin."));
+ assertTrue(importList.get(4).equals("org.apache.pig.impl.builtin."));
+
+ Object udf = PigContext.instantiateFuncFromSpec("TestUDF");
+ assertTrue(udf.getClass().toString().endsWith("com.xxx.udf1.TestUDF"));
+
+ Util.deleteDirectory(tempDir);
+ }
@After
@Override
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java Wed Jun 24
05:10:33 2009
@@ -120,14 +120,6 @@
addMethod.invoke(sysLoader, new Object[]{urlToAdd});
}
- private static void executeShellCommand(String cmd) throws Exception {
- Process cmdProc = Runtime.getRuntime().exec(cmd);
-
- cmdProc.waitFor();
-
- assertTrue(cmdProc.exitValue() == 0);
- }
-
/**
* The jar file to register is not present
*/
@@ -252,16 +244,19 @@
outWriter.close();
// compile
- executeShellCommand("javac " + dir + FILE_SEPARATOR + subDir +
+ int status;
+ status = Util.executeShellCommand("javac " + dir + FILE_SEPARATOR +
subDir +
FILE_SEPARATOR + className + ".java");
+ assertTrue(status==0);
// remove src file
(new File(dir + FILE_SEPARATOR + subDir +
FILE_SEPARATOR + className + ".java")).delete();
// generate jar file
- executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " +
+ status = Util.executeShellCommand("jar -cf " + dir + FILE_SEPARATOR +
jarName + " " +
"-C " + dir + " " + subDir);
+ assertTrue(status==0);
// remove class file and sub_dir
(new File(dir + FILE_SEPARATOR + subDir +
Modified: hadoop/pig/trunk/test/org/apache/pig/test/Util.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/Util.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/Util.java Wed Jun 24 05:10:33 2009
@@ -399,4 +399,26 @@
pw.close();
pigServer.registerScript(f.getCanonicalPath());
}
+
+ public static int executeShellCommand(String cmd) throws Exception {
+ Process cmdProc = Runtime.getRuntime().exec(cmd);
+
+ cmdProc.waitFor();
+
+ return cmdProc.exitValue();
+ }
+ static public boolean deleteDirectory(File path) {
+ if(path.exists()) {
+ File[] files = path.listFiles();
+ for(int i=0; i<files.length; i++) {
+ if(files[i].isDirectory()) {
+ deleteDirectory(files[i]);
+ }
+ else {
+ files[i].delete();
+ }
+ }
+ }
+ return(path.delete());
+ }
}
