Author: pisong
Date: Thu Jul 10 07:47:18 2008
New Revision: 675599

URL: http://svn.apache.org/viewvc?rev=675599&view=rev
Log:
PIG-293 Fixed star sort parsing

Added:
    incubator/pig/branches/types/test/org/apache/pig/test/TestOrderBy2.java
Modified:
    incubator/pig/branches/types/build.xml
    
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
    
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java

Modified: incubator/pig/branches/types/build.xml
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/build.xml?rev=675599&r1=675598&r2=675599&view=diff
==============================================================================
--- incubator/pig/branches/types/build.xml (original)
+++ incubator/pig/branches/types/build.xml Thu Jul 10 07:47:18 2008
@@ -306,6 +306,7 @@
                     <exclude name="**/TestHelper.java" />
                     <exclude name="**/TestLargeFile.java" />
                     <exclude name="**/TestOrderBy.java" />
+                    <exclude name="**/TestOrderBy2.java" />
                     <exclude name="**/TestPi.java" />
                     <exclude name="**/nightly/**" />
                 </fileset>

Modified: 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt?rev=675599&r1=675598&r2=675599&view=diff
==============================================================================
--- 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 (original)
+++ 
incubator/pig/branches/types/src/org/apache/pig/impl/logicalLayer/parser/QueryParser.jjt
 Thu Jul 10 07:47:18 2008
@@ -1107,6 +1107,12 @@
        )
        |       <STAR> {star = true;} [<ASC> | <DESC> {asc = false;}] 
                {
+            LogicalPlan sortColPlan = new LogicalPlan();
+                   LOProject projectStar = new LOProject(sortColPlan, new 
OperatorKey(scope, getNextId()), op, -1);
+                       ((LOProject)projectStar).setStar(true);
+            sortColPlan.add(projectStar);
+            sortColPlans.add(sortColPlan);
+                       log.debug("Set star to true");
                        if(asc) {
                                ascOrder.add(true);
                        } else {        
@@ -1456,6 +1462,11 @@
                )
                | <STAR> {star = true;} [<ASC> | <DESC> {asc = false;}] 
                        {
+                LogicalPlan sortColPlan = new LogicalPlan();
+                       LOProject projectStar = new LOProject(sortColPlan, new 
OperatorKey(scope, getNextId()), eOp, -1);
+                           ((LOProject)projectStar).setStar(true);
+                sortColPlan.add(projectStar);
+                sortColPlans.add(sortColPlan);
                                if(asc) {
                                        ascOrder.add(true);
                                } else {        

Modified: 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java?rev=675599&r1=675598&r2=675599&view=diff
==============================================================================
--- 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 (original)
+++ 
incubator/pig/branches/types/test/org/apache/pig/test/TestLogicalPlanBuilder.java
 Thu Jul 10 07:47:18 2008
@@ -699,6 +699,9 @@
         buildPlan("d = order c by b::name, height, a::gpa;");
         buildPlan("e = order a by name, age, gpa desc;");
         buildPlan("f = order a by $0 asc, age, gpa desc;");
+        buildPlan("g = order a by * asc;");
+        buildPlan("h = cogroup a by name, b by name;");
+        buildPlan("i = foreach h {i1 = order a by *; generate i1;};");
     }
 
     @Test

Added: incubator/pig/branches/types/test/org/apache/pig/test/TestOrderBy2.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestOrderBy2.java?rev=675599&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestOrderBy2.java 
(added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestOrderBy2.java Thu 
Jul 10 07:47:18 2008
@@ -0,0 +1,309 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.PrintStream;
+import java.io.IOException;
+import java.text.DecimalFormat;
+import java.util.Iterator;
+import java.util.List;
+import java.util.ArrayList;
+
+import junit.framework.TestCase;
+import junit.framework.Assert;
+
+import org.junit.Test;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.data.DataType;
+import org.apache.pig.data.Tuple;
+
+public class TestOrderBy2 extends TestCase {
+
+    private String initString = "mapreduce";
+    MiniCluster cluster = MiniCluster.buildCluster();
+
+    private PigServer pig ;
+
+    public TestOrderBy2() throws Throwable {
+        pig = new PigServer(initString) ;
+    }
+
+    //////////////////////// Simple Order Tests ///////////////////////
+
+    @Test
+    public void testTopLevelOrderBy_Col0_ASC_NoUsing() throws Exception {
+        File tmpFile = genDataSetFile1() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $0;",
+                new int[] { 0 } ,
+                new boolean[] { false } ) ;
+    }
+
+    @Test
+    public void testTopLevelOrderBy_Col0_DESC_NoUsing() throws Exception {
+        File tmpFile = genDataSetFile1() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $0 DESC ;",
+                new int[] { 0 } ,
+                new boolean[] { true } ) ;
+    }
+
+    @Test
+    public void testTopLeveleeOrderBy_Col1_ASC_NoUsing() throws Exception {
+        File tmpFile = genDataSetFile1() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $1 ASC ;",
+                new int[] { 1 } ,
+                new boolean[] { false } ) ;
+    }
+
+    @Test
+    public void testTopLeveleeOrderBy_Col1_DESC_NoUsing() throws Exception {
+        File tmpFile = genDataSetFile1() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $1 DESC ;",
+                new int[] { 1 } ,
+                new boolean[] { true } ) ;
+    }
+
+    @Test
+    public void testTopLeveleeOrderBy_Col0Col1_NoUsing() throws Exception {
+        File tmpFile = genDataSetFile2() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $0, $1 ;",
+                new int[] { 0, 1 } ,
+                new boolean[] { false, false } ) ;
+    }
+
+
+    @Test
+    public void testTopLeveleeOrderBy_Col0Col1_DESC_NoUsing() throws Exception 
{
+        File tmpFile = genDataSetFile2() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $0 DESC, $1 
DESC ;",
+                new int[] { 0, 1 } ,
+                new boolean[] { true, true } ) ;
+    }
+
+    @Test
+    public void testTopLeveleeOrderBy_Col1Col0_NoUsing() throws Exception {
+        File tmpFile = genDataSetFile3() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $1, $0 ;",
+                new int[] { 1, 0 } ,
+                new boolean[] { false, false } ) ;
+    }
+
+
+    @Test
+    public void testTopLeveleeOrderBy_Col1Col0_DESC_NoUsing() throws Exception 
{
+        File tmpFile = genDataSetFile3() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $1 DESC, $0 
DESC ;",
+                new int[] { 1, 0 } ,
+                new boolean[] { true, true } ) ;
+    }
+
+    @Test
+    public void testTopLeveleeOrderBy_Col1Col0_ASCDESC_NoUsing() throws 
Exception {
+        File tmpFile = genDataSetFile3() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY $1 ASC, $0 DESC 
;",
+                new int[] { 1, 0 } ,
+                new boolean[] { false, true } ) ;
+    }
+
+
+    //////////////////////// Star Order Tests ///////////////////////
+    
+    @Test
+    public void testTopLevelOrderBy_Star_NoUsing() throws Exception {
+        File tmpFile = genDataSetFile3() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY * ; ",
+                new int[] { 0, 1 } ,
+                new boolean[] { false, false } ) ;
+    }
+
+    @Test
+    public void testTopLevelOrderBy_Star_DESC_NoUsing() throws Exception {
+        File tmpFile = genDataSetFile3() ;
+        runTest("myid = order (load 'file:" + tmpFile + "') BY * DESC ; ",
+                new int[] { 0, 1 } ,
+                new boolean[] { true, true } ) ;
+    }
+
+    //////////////////////// HELPERS ///////////////////////////////
+
+    /**
+     * Create temp file from a given dataset
+     * This assumes
+     *  1) The dataset has at least 1 record
+     *  2) All records are of the same size
+     */
+    private File createTempFile(String[][] data) throws IOException {
+
+        File fp1 = File.createTempFile("test", "txt");
+        PrintStream ps = new PrintStream(new FileOutputStream(fp1));
+
+        for(int i = 0; i < data.length ; i++) {
+
+            // Building up string for each line
+            StringBuilder sb = new StringBuilder() ;
+            for(int j = 0 ; j < data[0].length ; j++) {
+                if (j != 0) {
+                    sb.append("\t") ;
+                }
+                sb.append(data[i][j]) ;
+            }
+
+            // Write the line to file
+            ps.println(sb.toString());
+        }
+
+        ps.close();
+        return fp1 ;
+    }
+
+    /***
+     * Check if the given dataset is properly sorted
+     * @param dataIter the dataset to be checked
+     * @param sortCols list of sorted columns
+     * @param descFlags flags (true=desc, false=asc)
+     */
+    private void checkOrder(Iterator<Tuple> dataIter,
+                            int[] sortCols,
+                            boolean[] descFlags)
+                                        throws ExecException {
+
+        Assert.assertEquals("checkOrder params have to be of the same size",
+                                        sortCols.length, descFlags.length);
+
+        List<String> error = new ArrayList<String>() ;
+
+        Tuple lastTuple = null ;
+
+        while (dataIter.hasNext()) {
+
+            Tuple current = dataIter.next() ;
+            System.out.println(current.toString()) ;
+
+            if (lastTuple != null) {
+                // do the actual check
+                for(int i=0; i < sortCols.length ; i++) {
+
+                    int colIdx = sortCols[i] ;
+                    int lastInt = DataType.toInteger(lastTuple.get(colIdx)) ;
+                    int curInt = DataType.toInteger(current.get(colIdx)) ;
+
+                    // If it's ascending
+                    if (!descFlags[i]) {
+                        if (curInt < lastInt) {
+                            error.add("Not ASC") ;
+                        }
+                        // if this happens, no need to check further
+                        if (curInt > lastInt) {
+                            break ;
+                        }
+                    }
+                    // If it's descending
+                    else {
+                        if (curInt > lastInt) {
+                            error.add("Not DESC") ;
+                        }
+                        // if this happens, no need to check further
+                        if (curInt < lastInt) {
+                            break ;
+                        }
+                    }
+                }
+            }
+
+            lastTuple = current ;
+        }
+
+        Assert.assertTrue(error.size()==0);
+
+    }
+
+    /***
+     * Main helper for running a sort test
+     */
+    private void runTest(String query,
+                         int[] sortCols,
+                         boolean[] descFlags)
+                                        throws Exception {
+        System.out.println(query) ;
+        pig.registerQuery(query);
+        Iterator<Tuple> it = pig.openIterator("myid");
+        checkOrder(it, sortCols, descFlags) ;
+    }
+
+
+    /***
+     * For generating a sample dataset
+     */
+    private File genDataSetFile1() throws IOException {
+
+        int dataLength = 256;
+        String[][] data = new String[dataLength][] ;
+
+        DecimalFormat formatter = new DecimalFormat("0000000");
+
+        for (int i = 0; i < dataLength; i++) {
+            data[i] = new String[2] ;
+            data[i][0] = formatter.format(i);
+            data[i][1] = formatter.format(dataLength - i - 1);
+        }
+
+        return createTempFile(data) ;
+    }
+
+
+    /***
+     * For generating a sample dataset
+     */
+    private File genDataSetFile2() throws IOException {
+
+        int dataLength = 256;
+        String[][] data = new String[dataLength][] ;
+
+        DecimalFormat formatter = new DecimalFormat("0000000");
+
+        for (int i = 0; i < dataLength; i++) {
+            data[i] = new String[2] ;
+            data[i][0] = formatter.format(i % 20);
+            data[i][1] = formatter.format(dataLength - i - 1);
+        }
+
+        return createTempFile(data) ;
+    }
+
+    /***
+     * For generating a sample dataset
+     */
+    private File genDataSetFile3() throws IOException {
+
+        int dataLength = 256;
+        String[][] data = new String[dataLength][] ;
+
+        DecimalFormat formatter = new DecimalFormat("0000000");
+
+        for (int i = 0; i < dataLength; i++) {
+            data[i] = new String[2] ;
+            data[i][0] = formatter.format(i);
+            data[i][1] = formatter.format(i % 20);
+        }
+
+        return createTempFile(data) ;
+    }
+}


Reply via email to