Author: daijy
Date: Wed Jun 24 05:10:33 2009
New Revision: 787908

URL: http://svn.apache.org/viewvc?rev=787908&view=rev
Log:
PIG-832: Make import list configurable

Modified:
    hadoop/pig/trunk/CHANGES.txt
    hadoop/pig/trunk/src/org/apache/pig/Main.java
    hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java
    hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java
    hadoop/pig/trunk/test/org/apache/pig/test/Util.java

Modified: hadoop/pig/trunk/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Wed Jun 24 05:10:33 2009
@@ -26,6 +26,8 @@
 
 IMPROVEMENTS
 
+PIG-832: Make import list configurable (daijy)
+
 PIG-697: Proposed improvements to pig's optimizer (sms)
 
 PIG-753:  Allow UDFs with no parameters (zjffdu via gates).
@@ -34,6 +36,10 @@
 
 BUG FIXES
 
+    PIG-797: Limit with ORDER BY producing wrong results (daijy)
+
+    PIG-850: Dump produce wrong result while "store into" is ok (daijy)
+
     PIG-852: pig -version or pig -help returns exit code of 1 (milindb via
     olgan)
 
@@ -88,10 +94,6 @@
 
 BUG FIXES
 
-PIG-797: Limit with ORDER BY producing wrong results (daijy)
-
-PIG-850: Dump produce wrong result while "store into" is ok (daijy)
-
 PIG-846: MultiQuery optimization in some cases has an issue when there is a
 split in the map plan (pradeepkth)
 

Modified: hadoop/pig/trunk/src/org/apache/pig/Main.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/Main.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/Main.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/Main.java Wed Jun 24 05:10:33 2009
@@ -272,6 +272,9 @@
         if(optimizerRules.size() > 0) {
                pigContext.getProperties().setProperty("pig.optimizer.rules", 
ObjectSerializer.serialize(optimizerRules));
         }
+        
+        if (properties.get("udf.import.list")!=null)
+            
PigContext.initializeImportList((String)properties.get("udf.import.list"));
 
         LogicalPlanBuilder.classloader = pigContext.createCl(null);
 

Modified: hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/impl/PigContext.java Wed Jun 24 
05:10:33 2009
@@ -34,6 +34,7 @@
 import java.util.List;
 import java.util.Map;
 import java.util.Properties;
+import java.util.StringTokenizer;
 import java.util.Vector;
 
 import org.apache.commons.logging.Log;
@@ -156,10 +157,23 @@
     static{
         packageImportList.add("");
         packageImportList.add("org.apache.pig.builtin.");
-        packageImportList.add("com.yahoo.pig.yst.sds.ULT.");
         packageImportList.add("org.apache.pig.impl.builtin.");        
     }
     
+    public static void initializeImportList(String 
importListCommandLineProperties)
+    {
+        StringTokenizer tokenizer = new 
StringTokenizer(importListCommandLineProperties, ":");
+        int pos = 0;
+        while (tokenizer.hasMoreTokens())
+        {
+            String importItem = tokenizer.nextToken();
+            if (!importItem.endsWith("."))
+                importItem += ".";
+            packageImportList.add(pos, importItem);
+            pos++;
+        }
+    }
+    
     public void connect() throws ExecException {
 
         switch (execType) {
@@ -614,4 +628,9 @@
             return PigException.BUG;
         }        
     }
+    
+    public static ArrayList<String> getPackageImportList()
+    {
+        return packageImportList;
+    }
 }

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigContext.java Wed Jun 24 
05:10:33 2009
@@ -20,7 +20,9 @@
 
 
 import java.io.File;
+import java.io.FileOutputStream;
 import java.io.IOException;
+import java.io.OutputStreamWriter;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.Properties;
@@ -90,6 +92,58 @@
        Object object = 
PigContext.instantiateFuncFromSpec("org.apache.hadoop.mapred.FileAlreadyExistsException");
        assertTrue(object instanceof FileAlreadyExistsException);
     }
+    
+    @Test
+    // See PIG-832
+    public void testImportList() throws Exception {
+        String FILE_SEPARATOR = System.getProperty("file.separator");
+        File tmpDir = File.createTempFile("test", "");
+        tmpDir.delete();
+        tmpDir.mkdir();
+        
+        File tempDir = new File(tmpDir.getAbsolutePath());
+        Util.deleteDirectory(tempDir);
+        File udf1Dir = new 
File(tmpDir.getAbsolutePath()+FILE_SEPARATOR+"com"+FILE_SEPARATOR+"xxx"+FILE_SEPARATOR+"udf1");
+        udf1Dir.mkdirs();
+        File udf1JavaSrc = new 
File(udf1Dir.getAbsolutePath()+FILE_SEPARATOR+"TestUDF.java");
+        String udf1Src = new String("package com.xxx.udf1;\n" +
+                "public class TestUDF {}\n");
+        
+        // generate java file
+        FileOutputStream outStream = 
+            new FileOutputStream(udf1JavaSrc);
+        
+        OutputStreamWriter outWriter = new OutputStreamWriter(outStream);
+        outWriter.write(udf1Src);
+        outWriter.close();
+        
+        // compile
+        int status;
+        status = Util.executeShellCommand("javac " + udf1JavaSrc);
+        
+        // generate jar file
+        String jarName = "TestUDFJar1.jar";
+        status = Util.executeShellCommand("jar -cf " + 
tmpDir.getAbsolutePath() + FILE_SEPARATOR + jarName + 
+                              " -C " + tmpDir.getAbsolutePath() + " " + "com");
+        assertTrue(status==0);
+
+        PigServer pig = new PigServer(pigContext);
+        pig.registerJar(tmpDir.getAbsolutePath() + FILE_SEPARATOR + jarName);
+        
+        PigContext.initializeImportList("com.xxx.udf1:com.xxx.udf2.");
+        ArrayList<String> importList = PigContext.getPackageImportList();
+        assertTrue(importList.size()==5);
+        assertTrue(importList.get(0).equals("com.xxx.udf1."));
+        assertTrue(importList.get(1).equals("com.xxx.udf2."));
+        assertTrue(importList.get(2).equals(""));
+        assertTrue(importList.get(3).equals("org.apache.pig.builtin."));
+        assertTrue(importList.get(4).equals("org.apache.pig.impl.builtin."));
+        
+        Object udf = PigContext.instantiateFuncFromSpec("TestUDF");
+        assertTrue(udf.getClass().toString().endsWith("com.xxx.udf1.TestUDF"));
+        
+        Util.deleteDirectory(tempDir);
+    }
 
     @After
     @Override

Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigServer.java Wed Jun 24 
05:10:33 2009
@@ -120,14 +120,6 @@
         addMethod.invoke(sysLoader, new Object[]{urlToAdd});
     }
     
-    private static void executeShellCommand(String cmd) throws Exception {
-        Process cmdProc = Runtime.getRuntime().exec(cmd);
-        
-        cmdProc.waitFor();
-        
-        assertTrue(cmdProc.exitValue() == 0);
-    }
-    
     /**
      * The jar file to register is not present
      */
@@ -252,16 +244,19 @@
         outWriter.close();
         
         // compile
-        executeShellCommand("javac " + dir + FILE_SEPARATOR + subDir +
+        int status;
+        status = Util.executeShellCommand("javac " + dir + FILE_SEPARATOR + 
subDir +
                                FILE_SEPARATOR + className + ".java");
+        assertTrue(status==0);
 
         // remove src file
         (new File(dir + FILE_SEPARATOR + subDir +
                   FILE_SEPARATOR + className + ".java")).delete();
 
         // generate jar file
-        executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + jarName + " " +
+        status = Util.executeShellCommand("jar -cf " + dir + FILE_SEPARATOR + 
jarName + " " +
                               "-C " + dir + " " + subDir);
+        assertTrue(status==0);
         
         // remove class file and sub_dir
         (new File(dir + FILE_SEPARATOR + subDir +

Modified: hadoop/pig/trunk/test/org/apache/pig/test/Util.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/Util.java?rev=787908&r1=787907&r2=787908&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/Util.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/Util.java Wed Jun 24 05:10:33 2009
@@ -399,4 +399,26 @@
         pw.close();
         pigServer.registerScript(f.getCanonicalPath());
     }
+    
+    public static int executeShellCommand(String cmd) throws Exception {
+        Process cmdProc = Runtime.getRuntime().exec(cmd);
+        
+        cmdProc.waitFor();
+        
+        return cmdProc.exitValue();
+    }
+    static public boolean deleteDirectory(File path) {
+        if(path.exists()) {
+            File[] files = path.listFiles();
+            for(int i=0; i<files.length; i++) {
+                if(files[i].isDirectory()) {
+                    deleteDirectory(files[i]);
+                }
+                else {
+                    files[i].delete();
+                }
+            }
+        }
+        return(path.delete());
+    }
 }


Reply via email to