Author: yanz
Date: Fri Apr 2 03:58:09 2010
New Revision: 930157
URL: http://svn.apache.org/viewvc?rev=930157&view=rev
Log:
PIG-1349 Hubson test failure in test case TestBasicUnion (xuefuz via yanz)
Modified:
hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveMultiTable.java
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java
Modified: hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt?rev=930157&r1=930156&r2=930157&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt Fri Apr 2
03:58:09 2010
@@ -70,6 +70,8 @@ Trunk (unreleased changes)
BUG FIXES
+ PIG-1349 Hubson test failure in test case TestBasicUnion (xuefuz via yanz)
+
PIG-1318 Invalid type for source_table field when using order-preserving
Sorted Table Union (gauravj via yanz)
PIG-1258 Number of sorted input splits is unusually high (yanz)
Modified:
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java?rev=930157&r1=930156&r2=930157&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
(original)
+++
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
Fri Apr 2 03:58:09 2010
@@ -670,83 +670,35 @@ public class TestBasicUnion extends Base
@Test
// union two tables with different column numbers and column positions
public void testReader6() throws ExecException, IOException {
-
pigServer.registerQuery(constructQuery(path1, path5, "'b,a'"));
Iterator<Tuple> it = pigServer.openIterator("records");
int i = 0;
- int k = -1;
+ int count = 0;
Tuple cur = null;
- int t = -1;
- int j = -1;
+ String[] exp1 = new String[] { "0_01", "1_01", "2_01", "3_01", "4_01",
"5_01", "6_01", "7_01", "8_01", "9_01",
+ "0_11", "1_11", "2_11", "3_11", "4_11",
"5_11", "6_11", "7_11", "8_11", "9_11"};
+ String[] exp2 = new String[] { "0_00", "1_00", "2_00", "3_00", "4_00",
"5_00", "6_00", "7_00", "8_00", "9_00",
+ "0_10", "1_10", "2_10", "3_10", "4_10",
"5_10", "6_10", "7_10", "8_10", "9_10"};
while (it.hasNext()) {
+ count++;
cur = it.next();
-
- System.out.println("cur: " + cur);
- // first table
- if (i <= 9) {
- System.out.println("first table first part: " + cur.toString());
- Assert.assertEquals(i + "_01", cur.get(0));
- Assert.assertEquals(i + "_00", cur.get(1));
- try {
- cur.get(2);
- Assert.fail("should throw index out of bound exception");
- } catch (Exception e) {
-
- }
-
- }
- if (i >= 10) {
- k++;
- }
- if (k <= 9 && k >= 0) {
- System.out.println("first table second part: : " + cur.toString());
- Assert.assertEquals(k + "_11", cur.get(0));
- Assert.assertEquals(k + "_10", cur.get(1));
- try {
- cur.get(2);
- Assert.fail("should throw index out of bound exception");
- } catch (Exception e) {
-
- }
-
- }
-
- // second table
- if (k >= 10) {
- t++;
- }
- if (t <= 9 && t >= 0) {
- System.out.println("second table first part: " + cur.toString());
- Assert.assertEquals(t + "_00", cur.get(0));
- Assert.assertEquals(t + "_01", cur.get(1));
- try {
+ System.out.println("cur #" + i + ": " + cur);
+ try {
cur.get(2);
Assert.fail("should throw index out of bound exception");
- } catch (Exception e) {
-
- }
-
+ } catch (Exception e) {
}
- if (t >= 10) {
- j++;
- }
- if (j <= 9 && j >= 0) {
- System.out.println("second table first part: " + cur.toString());
- Assert.assertEquals(j + "_10", cur.get(0));
- Assert.assertEquals(j + "_11", cur.get(1));
- try {
- cur.get(2);
- Assert.fail("should throw index out of bound exception");
- } catch (Exception e) {
-
- }
- }
- i++;
+ String b = (String)cur.get(0);
+ String a = (String)cur.get(1);
+ Assert.assertTrue( b.equals(exp1[i]) || b.equals(exp2[i]) );
+ Assert.assertTrue( a.equals(exp2[i]) || a.equals(exp1[i]) );
+ if( ++i == 20 )
+ i = 0;
}// while
- Assert.assertEquals(40, i);
+ Assert.assertEquals(40, count);
}
// both paths is hdfs:///../jars. mini cluster need to substr, real cluster
Modified:
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveMultiTable.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveMultiTable.java?rev=930157&r1=930156&r2=930157&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveMultiTable.java
(original)
+++
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestOrderPreserveMultiTable.java
Fri Apr 2 03:58:09 2010
@@ -21,6 +21,7 @@ package org.apache.hadoop.zebra.pig;
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.ArrayList;
import java.util.StringTokenizer;
@@ -182,123 +183,30 @@ public class TestOrderPreserveMultiTable
testOrderPreserveUnion(inputTables, "int1", "int1,
source_table, str1, byte1");
// Create results table for verification
- ArrayList<ArrayList<Object>> resultTable = new
ArrayList<ArrayList<Object>>();
+ HashMap<Integer, ArrayList<ArrayList<Object>>> resultTable =
+ new HashMap<Integer, ArrayList<ArrayList<Object>>>();
for (int i=0; i<NUMB_TABLE; ++i) {
+ ArrayList<ArrayList<Object>> rows = new
ArrayList<ArrayList<Object>>();
for (int j=0; j<NUMB_TABLE_ROWS; ++j) {
ArrayList<Object> resultRow = new
ArrayList<Object>();
-
resultRow.add(i); // int1
resultRow.add(i); // source_table
resultRow.add(new String("string" + j));
// str1
resultRow.add(new DataByteArray("byte" +
(NUMB_TABLE_ROWS - j))); // byte1
-
- resultTable.add(resultRow);
+ rows.add(resultRow);
}
+ resultTable.put(i, rows);
}
// Verify union table
Iterator<Tuple> it = pigServer.openIterator("records1");
- int numbRows = verifyTable(resultTable, 0, it);
+ int numbRows = verifyTable(resultTable, 0, 1, it);
Assert.assertEquals(totalTableRows, numbRows);
// Print Table
//printTable("records1");
- }
-
- /**
- * Verify union output table with expected results
- *
- */
- private int verifyTable(ArrayList<ArrayList<Object>> resultTable, int
keyColumn, Iterator<Tuple> it) throws IOException {
- int numbRows = 0;
- int index = 0;
- Object value = resultTable.get(index).get(keyColumn); // get
value of primary key
-
- while (it.hasNext()) {
- Tuple rowValues = it.next();
-
- // If last primary sort key does match then search for
next matching key
- if (! compareObj(value, rowValues.get(keyColumn))) {
- int subIndex = index + 1;
- while (subIndex < resultTable.size()) {
- if ( ! compareObj(value,
resultTable.get(subIndex).get(keyColumn)) ) { // found new key
- index = subIndex;
- value =
resultTable.get(index).get(keyColumn);
- break;
- }
- ++subIndex;
- }
- Assert.assertEquals("Table comparison error for
row : " + numbRows + " - no key found for : "
- + rowValues.get(keyColumn), value,
rowValues.get(keyColumn));
- }
- // Search for matching row with this primary key
- int subIndex = index;
-
- while (subIndex < resultTable.size()) {
- // Compare row
- ArrayList<Object> resultRow =
resultTable.get(subIndex);
- if ( compareRow(rowValues, resultRow) )
- break; // found matching row
- ++subIndex;
- Assert.assertEquals("Table comparison error for
row : " + numbRows + " - no matching row found for : "
- + rowValues.get(keyColumn), value,
resultTable.get(subIndex).get(keyColumn));
- }
- ++numbRows;
- }
- Assert.assertEquals(resultTable.size(), numbRows); // verify
expected row count
- return numbRows;
- }
-
- /**
- * Compare table rows
- *
- */
- private boolean compareRow(Tuple rowValues, ArrayList<Object>
resultRow) throws IOException {
- boolean result = true;
- Assert.assertEquals(resultRow.size(), rowValues.size());
- for (int i = 0; i < rowValues.size(); ++i) {
- if (! compareObj(rowValues.get(i), resultRow.get(i)) ) {
- result = false;
- break;
- }
- }
- return result;
- }
-
- /**
- * Compare table values
- *
- */
- private boolean compareObj(Object object1, Object object2) {
- if (object1 == null) {
- if (object2 == null)
- return true;
- else
- return false;
- } else if (object1.equals(object2))
- return true;
- else
- return false;
- }
-
- /**
- * Print Pig Table (for debugging)
- *
- */
- private int printTable(String tablename) throws IOException {
- Iterator<Tuple> it1 = pigServer.openIterator(tablename);
- int numbRows = 0;
- while (it1.hasNext()) {
- Tuple RowValue1 = it1.next();
- ++numbRows;
- System.out.println();
- for (int i = 0; i < RowValue1.size(); ++i)
- System.out.println("DEBUG: " + tablename + "
RowValue.get(" + i + ") = " + RowValue1.get(i));
- }
- System.out.println("\nRow count : " + numbRows);
- return numbRows;
- }
+ }
/**
* Return the name of the routine that called getCurrentMethodName
Modified:
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java?rev=930157&r1=930156&r2=930157&view=diff
==============================================================================
---
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java
(original)
+++
hadoop/pig/branches/branch-0.7/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestUnionMixedTypes.java
Fri Apr 2 03:58:09 2010
@@ -349,60 +349,18 @@ public class TestUnionMixedTypes extends
Iterator<Tuple> it = pigServer.openIterator("records");
Tuple cur = null;
+ String[] expected1 = new String[]{"k11", "k12", "k13", "k14" };
+ String[] expected2 = new String[]{"k13", "k14", "k11", "k12" };
int i = 0;
- int j = 0;
// total 4 lines
while (it.hasNext()) {
cur = it.next();
-
+ String key = ((Map<String, String>) cur.get(0)).get("k1");
+ Assert.assertTrue( expected1[i].equals( key ) || expected2[i].equals(
key ) );
i++;
System.out.println(" line : " + i + " : " + cur.toString());
- // first line
-
- if (i == 1) {
- System.out.println("i is : " + i);
-
- Assert.assertEquals("k11", ((Map) cur.get(0)).get("k1"));
- Assert.assertEquals(null, ((Map) cur.get(0)).get("k2"));
- }
-
- if (i == 2) {
- Assert.assertEquals("k12", ((Map) cur.get(0)).get("k1"));
- Assert.assertEquals(null, ((Map) cur.get(0)).get("k2"));
- try {
- cur.get(1);
- Assert.fail("should throw index out of bound exception");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- if (i == 3) {
- System.out.println("i is : " + i);
-
- Assert.assertEquals("k13", ((Map) cur.get(0)).get("k1"));
- Assert.assertEquals(null, ((Map) cur.get(0)).get("k2"));
- try {
- cur.get(1);
- Assert.fail("should throw index out of bound exception");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
-
- if (i == 4) {
- System.out.println("i should see this line. ");
- Assert.assertEquals("k14", ((Map) cur.get(0)).get("k1"));
- Assert.assertEquals(null, ((Map) cur.get(0)).get("k2"));
- try {
- cur.get(1);
- Assert.fail("should throw index out of bound exception");
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- }// outer while
-
+ }
Assert.assertEquals(4, i);
}
