Author: pradeepkth
Date: Tue Dec 22 00:36:47 2009
New Revision: 893048

URL: http://svn.apache.org/viewvc?rev=893048&view=rev
Log:
svn merge -r892943:892972 http://svn.apache.org/repos/asf/hadoop/pig/trunk

Modified:
    hadoop/pig/branches/load-store-redesign/CHANGES.txt
    
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/IndexableLoadFunc.java
    
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java
    
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
    
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java
    
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java

Modified: hadoop/pig/branches/load-store-redesign/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/CHANGES.txt?rev=893048&r1=893047&r2=893048&view=diff
==============================================================================
--- hadoop/pig/branches/load-store-redesign/CHANGES.txt (original)
+++ hadoop/pig/branches/load-store-redesign/CHANGES.txt Tue Dec 22 00:36:47 2009
@@ -92,6 +92,15 @@
 
 BUG FIXES
 
+PIG-1159: merge join right side table does not support comma seperated paths
+(rding via olgan)
+
+PIG-1158: pig command line -M option doesn't support table union correctly
+(comma seperated paths) (rding via olgan)
+
+PIG-1143: Poisson Sample Loader should compute the number of samples required
+only once (sriranjan via olgan)
+
 PIG-1157: Sucessive replicated joins do not generate Map Reduce plan and fails
 due to OOM (rding via olgan)
 

Modified: 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/IndexableLoadFunc.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/IndexableLoadFunc.java?rev=893048&r1=893047&r2=893048&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/IndexableLoadFunc.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/IndexableLoadFunc.java
 Tue Dec 22 00:36:47 2009
@@ -33,10 +33,8 @@
  * The sequence of calls made from the pig runtime are:
  * 
  * {...@link IndexableLoadFunc#initialize(Configuration)}
- * IndexableLoadFunc.bindTo(filename, bufferedPositionedInputStream, 0, 
LONG.MAX_VALUE);
- * (the bufferedPositionedInputStream is a decorator around the underlying
- * DFS input stream)
- * IndexableLoadFunc.seekNear(keys);
+ * {...@link IndexableLoadFunc#setLocation(String, 
org.apache.hadoop.mapreduce.Job)}
+ * {...@link IndexableLoadFunc#seekNear(Tuple)}
  * A series of IndexableLoadFunc.getNext(); calls to perform the join
  * IndexableLoadFunc.close(); 
  * 

Modified: 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java?rev=893048&r1=893047&r2=893048&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java
 Tue Dec 22 00:36:47 2009
@@ -26,6 +26,7 @@
 
 import org.apache.commons.logging.Log;
 import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.mapreduce.Job;
 import org.apache.pig.FuncSpec;
 import org.apache.pig.IndexableLoadFunc;
 import org.apache.pig.PigException;
@@ -395,16 +396,10 @@
         rightLoader = 
(IndexableLoadFunc)PigContext.instantiateFuncFromSpec(rightLoaderFuncSpec);
         pc = 
(PigContext)ObjectSerializer.deserialize(PigMapReduce.sJobConf.get("pig.pigContext"));
         pc.connect();
-     // XXX FIXME - make this work with new load-store redesign
-//        InputStream is = FileLocalizer.open(rightInputFileName, pc);
         // Pass signature of the loader to rightLoader
         PigMapReduce.sJobConf.set("pig.loader.signature", signature);
-//        rightLoader.initialize(PigMapReduce.sJobConf);
-        // the main purpose of this bindTo call is supply the input file name
-        // to the right loader - in the case of Pig's DefaultIndexableLoader
-        // this is really not used since the index has all information required
-        
-//        rightLoader.bindTo(rightInputFileName, new 
BufferedPositionedInputStream(is), 0, Long.MAX_VALUE);
+        rightLoader.initialize(PigMapReduce.sJobConf);
+        rightLoader.setLocation(rightInputFileName, new 
Job(PigMapReduce.sJobConf));
         rightLoader.seekNear(
                 firstLeftKey instanceof Tuple ? (Tuple)firstLeftKey : 
mTupleFactory.newTuple(firstLeftKey));
     }

Modified: 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java?rev=893048&r1=893047&r2=893048&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/src/org/apache/pig/impl/builtin/DefaultIndexableLoader.java
 Tue Dec 22 00:36:47 2009
@@ -247,7 +247,7 @@
 
     @Override
     public void setLocation(String location, Job job) throws IOException {
-        throw new UnsupportedOperationException();
+        // nothing to do
     }
     
 }

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java?rev=893048&r1=893047&r2=893048&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java 
(original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestLoad.java 
Tue Dec 22 00:36:47 2009
@@ -17,15 +17,14 @@
  */
 package org.apache.pig.test;
 
+import static org.junit.Assert.*;
+
 import java.io.BufferedReader;
 import java.io.File;
 import java.io.FileNotFoundException;
 import java.io.FileReader;
-import java.io.IOException;
 import java.util.HashMap;
 import java.util.Map;
-import java.util.Properties;
-import java.util.regex.Pattern;
 
 import junit.framework.Assert;
 
@@ -36,6 +35,9 @@
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
 import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
 import 
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus;
+import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result;
+import 
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLoad;
 import org.apache.pig.builtin.PigStorage;
 import org.apache.pig.data.DataBag;
 import org.apache.pig.data.DataByteArray;
@@ -56,7 +58,7 @@
 import org.junit.Before;
 import org.junit.Test;
 
-public class TestLoad extends junit.framework.TestCase {
+public class TestLoad {
 
     PigContext pc;
     PigServer[] servers;
@@ -101,6 +103,14 @@
             }
             inpDB.add(t);
         }
+    }
+    
+    @After
+    public void tearDown() throws Exception {
+    }
+
+    @Test
+    public void testGetNextTuple() throws ExecException {
         Tuple t=null;
         int size = 0;
         for(Result res = 
ld.getNext(t);res.returnStatus!=POStatus.STATUS_EOP;res=ld.getNext(t)){
@@ -210,40 +220,44 @@
     }    
 
     private void checkLoadPath(String orig, String expected) throws Exception {
-        pc.getProperties().setProperty("opt.multiquery", "" + true);
-                
-        DataStorage dfs = pc.getDfs();
-        dfs.setActiveContainer(dfs.asContainer("/tmp"));
-        Map<LogicalOperator, LogicalPlan> aliases = new 
HashMap<LogicalOperator, LogicalPlan>();
-        Map<OperatorKey, LogicalOperator> logicalOpTable = new 
HashMap<OperatorKey, LogicalOperator>();
-        Map<String, LogicalOperator> aliasOp = new HashMap<String, 
LogicalOperator>();
-        Map<String, String> fileNameMap = new HashMap<String, String>();
-        
-        LogicalPlanBuilder builder = new LogicalPlanBuilder(pc);
-        
-        String query = "a = load '"+orig+"';";
-        LogicalPlan lp = builder.parse("Test-Load",
-                                       query,
-                                       aliases,
-                                       logicalOpTable,
-                                       aliasOp,
-                                       fileNameMap);
-        Assert.assertTrue(lp.size()>0);
-        LogicalOperator op = lp.getRoots().get(0);
-        
-        Assert.assertTrue(op instanceof LOLoad);
-        LOLoad load = (LOLoad)op;
+        checkLoadPath(orig, expected, false);
+    }
 
-        String p = load.getInputFile().getFileName();
-        System.err.println("DEBUG: p:" + p + " expected:" + expected +", 
exectype:" + pc.getExecType());        
-        if (pc.getExecType() == ExecType.MAPREDUCE) {
-            Assert.assertTrue(p.matches("hdfs://[0-9a-zA-Z:\\.]*.*"));
-            Assert.assertEquals(p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/", "/"),
-                    expected);
-        } else {
-            Assert.assertTrue(p.matches("file://[0-9a-zA-Z:\\.]*.*"));
-            Assert.assertEquals(p.replaceAll("file://[0-9a-zA-Z:\\.]*/", "/"),
-                    expected);
+    private void checkLoadPath(String orig, String expected, boolean isTmp) 
throws Exception {
+        boolean[] multiquery = {true, false};
+        for (boolean b : multiquery) {
+            pc.getProperties().setProperty("opt.multiquery", "" + b);
+                    
+            DataStorage dfs = pc.getDfs();
+            dfs.setActiveContainer(dfs.asContainer("/tmp"));
+            Map<LogicalOperator, LogicalPlan> aliases = new 
HashMap<LogicalOperator, LogicalPlan>();
+            Map<OperatorKey, LogicalOperator> logicalOpTable = new 
HashMap<OperatorKey, LogicalOperator>();
+            Map<String, LogicalOperator> aliasOp = new HashMap<String, 
LogicalOperator>();
+            Map<String, String> fileNameMap = new HashMap<String, String>();
+            
+            LogicalPlanBuilder builder = new LogicalPlanBuilder(pc);
+            
+            String query = "a = load '"+orig+"';";
+            LogicalPlan lp = builder.parse("Test-Load",
+                                           query,
+                                           aliases,
+                                           logicalOpTable,
+                                           aliasOp,
+                                           fileNameMap);
+            Assert.assertTrue(lp.size()>0);
+            LogicalOperator op = lp.getRoots().get(0);
+            
+            Assert.assertTrue(op instanceof LOLoad);
+            LOLoad load = (LOLoad)op;
+    
+            String p = load.getInputFile().getFileName();
+            p = p.replaceAll("hdfs://[0-9a-zA-Z:\\.]*/","/");
+    
+            if (isTmp) {
+                Assert.assertTrue(p.matches("/tmp.*"));
+            } else {
+                Assert.assertEquals(p, expected);
+            }
         }
     }
 }

Modified: 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java?rev=893048&r1=893047&r2=893048&view=diff
==============================================================================
--- 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java
 (original)
+++ 
hadoop/pig/branches/load-store-redesign/test/org/apache/pig/test/TestMergeJoin.java
 Tue Dec 22 00:36:47 2009
@@ -544,6 +544,18 @@
         Assert.assertTrue(shjSch == null);
     }
     
+    @Test
+    public void testMergeJoinWithCommaSeparatedFilePaths() throws IOException{
+
+        pigServer.registerQuery("A = LOAD '" + INPUT_FILE + "';");
+        pigServer.registerQuery("B = LOAD 'temp_file,righinput_file' using " +
+                DummyIndexableLoader.class.getName() + "();");
+
+        pigServer.registerQuery("C = join A by $0, B by $0 using \"merge\";");
+            
+        Iterator<Tuple> iter = pigServer.openIterator("C");
+        Assert.assertFalse(iter.hasNext());
+    }
     
     /**
      * A dummy loader which implements {...@link IndexableLoadFunc} to test


Reply via email to