Author: yanz
Date: Mon Feb  8 22:03:19 2010
New Revision: 907818

URL: http://svn.apache.org/viewvc?rev=907818&view=rev
Log:
PIG-1227: Throw exception if column group meta file is missing for an unsorted 
table (yanz)

Modified:
    hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java
    
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java

Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt (original)
+++ hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt Mon Feb  8 
22:03:19 2010
@@ -41,6 +41,8 @@
 
   BUG FIXES
 
+    PIG-1227: Throw exception if column group meta file is missing for an 
unsorted table (yanz)
+
     PIG-1201: unnecessary name node calls by each mapper; too big input split 
serialization size by Pig's Slice implementation (yanz)
 
     PIG-1167: Hadoop file glob support (yanz)

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/java/org/apache/hadoop/zebra/io/ColumnGroup.java
 Mon Feb  8 22:03:19 2010
@@ -293,12 +293,8 @@
       Path metaFilePath = makeMetaFilePath(path);
       /* If index file is not existing */
       if (!fs.exists(metaFilePath)) {
-        // special case for unsorted CG that did not create index properly.
-        if (cgschema.isSorted()) {
-          throw new FileNotFoundException(
-              "Missing Meta File for sorted Column Group");
-        }
-        cgindex = buildIndex(fs, path, dirty, conf);
+        throw new FileNotFoundException(
+              "Missing Meta File of " + metaFilePath);
       }
       else if (cgschema.isSorted()) {
         MetaFile.Reader metaFile = MetaFile.createReader(metaFilePath, conf);

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestBasicTable.java
 Mon Feb  8 22:03:19 2010
@@ -353,7 +353,7 @@
   @Test
   public void testNormalCases() throws IOException, ParseException {
     Path path = new Path(rootPath, "TestBasicTableNormal");
-    doReadWrite(path, 2, 250, "a, b, c", "", null, "a, d, c, f", false, false);
+    doReadWrite(path, 2, 250, "a, b, c", "", null, "a, d, c, f", true, false);
     doReadWrite(path, 2, 250, "a, b, c", "", null, "a, d, c, f", true, false);
     doReadWrite(path, 2, 250, "a, b, c", "", "a", "a, d, c, f", true, true);
   }

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroup.java
 Mon Feb  8 22:03:19 2010
@@ -411,7 +411,7 @@
   @Test
   public void testEmptyCG() throws IOException, ParseException {
     Path path = new Path(rootPath, "TestColumnGroupEmptyCG");
-    doReadWrite(path, 0, 0, "a, b, c", "a, d, c, f", false, false, null);
+    doReadWrite(path, 0, 0, "a, b, c", "a, d, c, f", true, false, null);
     doReadWrite(path, 0, 0, "a, b, c", "a, d, c, f", true, false, null);
     doReadWrite(path, 0, 0, "a, b, c", "a, d, c, f", true, true, null);
   }
@@ -419,14 +419,14 @@
   @Test
   public void testEmptyTFiles() throws IOException, ParseException {
     Path path = new Path(rootPath, "TestColumnGroupEmptyTFile");
-    doReadWrite(path, 2, 0, "a, b, c", "a, d, c, f", false, false, null);
+    doReadWrite(path, 2, 0, "a, b, c", "a, d, c, f", true, false, null);
     doReadWrite(path, 2, 0, "a, b, c", "a, d, c, f", true, false, null);
     doReadWrite(path, 2, 0, "a, b, c", "a, d, c, f", true, true, null);    
   }
 
   public void testNormalCases() throws IOException, ParseException {
     Path path = new Path(rootPath, "TestColumnGroupNormal");
-    doReadWrite(path, 2, 500, "a, b, c", "a, d, c, f", false, false, null);
+    doReadWrite(path, 2, 500, "a, b, c", "a, d, c, f", true, false, null);
     doReadWrite(path, 2, 500, "a, b, c", "a, d, c, f", true, false, null);
     doReadWrite(path, 2, 500, "a, b, c", "a, d, c, f", true, true, null);
   }
@@ -435,7 +435,7 @@
   public void testSomeEmptyTFiles() throws IOException, ParseException {
     Path path = new Path(rootPath, "TestColumnGroupSomeEmptyTFile");
                for (int[] emptyTFiles : new int[][] { { 1, 2 }}) {
-      doReadWrite(path, 2, 250, "a, b, c", "a, d, c, f", false, false,
+      doReadWrite(path, 2, 250, "a, b, c", "a, d, c, f", true, false,
           emptyTFiles);
       doReadWrite(path, 2, 250, "a, b, c", "a, d, c, f", true, false,
           emptyTFiles);

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/io/TestColumnGroupReaders.java
 Mon Feb  8 22:03:19 2010
@@ -67,7 +67,7 @@
 
   @AfterClass
   public static void tearDownOnce() throws IOException {
-    finish();
+    close();
   }
 
   @SuppressWarnings("unchecked")
@@ -175,9 +175,7 @@
     ColumnGroup.Writer writer2 = writeOnePart(null, 2);
     ColumnGroup.Writer writer3 = writeOnePart(null, 3);
 
-    writer1.finish();
-    writer2.finish();
-    writer3.finish();
+    writer3.close();
 
     // read in parts
     readOnePart(1);
@@ -327,7 +325,7 @@
 
   private static void finish() throws IOException {
     if (writer != null) {
-      writer.finish();
+      writer.close();
     }
   }
 

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/mapred/TestBasicTableIOFormatLocalFS.java
 Mon Feb  8 22:03:19 2010
@@ -305,6 +305,7 @@
     // set map-only job.
     jobConf.setNumReduceTasks(0);
     JobClient.runJob(jobConf);
+    BasicTableOutputFormat.close(jobConf);
   }
 
   /**
@@ -599,6 +600,7 @@
     jobConf.setNumReduceTasks(options.numReducer);
 
     JobClient.runJob(jobConf);
+    BasicTableOutputFormat.close(jobConf);
   }
 
   void reduce(Summary sum, Summary delta) {
@@ -950,6 +952,7 @@
     jobConf.setNumReduceTasks(1);
 
     JobClient.runJob(jobConf);
+    BasicTableOutputFormat.close(jobConf);
   }
 
   void printFreqWords() throws IOException, ParseException {

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicTableUnionLoader.java
 Mon Feb  8 22:03:19 2010
@@ -108,6 +108,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
 
     /*
      * create 2nd basic table;
@@ -141,6 +142,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
   }
 
   @AfterClass

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestBasicUnion.java
 Mon Feb  8 22:03:19 2010
@@ -126,7 +126,8 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
-
+    writer.close();
+    
     /*
      * create 2nd basic table;
      */
@@ -159,7 +160,8 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
-
+    writer.close();
+    
     /*
      * create 3rd basic table;
      */
@@ -192,6 +194,8 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
+    
     /*
      * create 4th basic table;
      */
@@ -224,6 +228,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
     /*
      * create 5th basic table;
      */
@@ -256,7 +261,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
-
+    writer.close();
   }
 
   @AfterClass

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestCollectionTableLoader.java
 Mon Feb  8 22:03:19 2010
@@ -118,6 +118,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
   }
 
   @AfterClass
@@ -137,4 +138,4 @@
       System.out.println(cur);
     }
   }
-}
\ No newline at end of file
+}

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestGlobTableLoader.java
 Mon Feb  8 22:03:19 2010
@@ -159,6 +159,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
   }
 
   @AfterClass

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestMapTableLoader.java
 Mon Feb  8 22:03:19 2010
@@ -114,6 +114,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
   }
 
   @AfterClass

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoader.java
 Mon Feb  8 22:03:19 2010
@@ -103,6 +103,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
   }
 
   @AfterClass

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/pig/TestTableLoaderPrune.java
 Mon Feb  8 22:03:19 2010
@@ -149,6 +149,7 @@
     for (int i = 0; i < numsInserters; i++) {
       inserters[i].close();
     }
+    writer.close();
   }
 
   @AfterClass

Modified: 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
URL: 
http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java?rev=907818&r1=907817&r2=907818&view=diff
==============================================================================
--- 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
 (original)
+++ 
hadoop/pig/branches/branch-0.6/contrib/zebra/src/test/org/apache/hadoop/zebra/types/TestStorageGrammar.java
 Mon Feb  8 22:03:19 2010
@@ -359,9 +359,10 @@
     } catch (Exception e) {
       e.printStackTrace();
     }
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
+
     BasicTable.dumpInfo(path1.toString(), ps, conf);
     System.out.println("start dumpinfo ===========\n" + bos.toString());
     Assert.assertEquals(true, bos.toString().contains("Serializer: pig"));
@@ -382,7 +383,7 @@
     } catch (Exception e) {
       e.printStackTrace();
     }
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
     BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -404,7 +405,7 @@
     } catch (Exception e) {
       e.printStackTrace();
     }
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
     BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -426,7 +427,7 @@
     } catch (Exception e) {
       e.printStackTrace();
     }
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
     BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -448,7 +449,7 @@
     } catch (Exception e) {
       e.printStackTrace();
     }
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
     BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -470,7 +471,7 @@
     } catch (Exception e) {
       e.printStackTrace();
     }
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
     BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -491,7 +492,7 @@
     } catch (Exception e) {
       e.printStackTrace();
     }
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
     BasicTable.dumpInfo(path1.toString(), ps, conf);
@@ -605,7 +606,7 @@
     fs = path.getFileSystem(conf);
     BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
         conf);
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
     System.out.println("start dumpinfo 17 ===========");
@@ -699,7 +700,7 @@
     fs = path.getFileSystem(conf);
     BasicTable.Writer writer = new BasicTable.Writer(path1, schema, storage,
         conf);
-    writer.finish();
+    writer.close();
     ByteArrayOutputStream bos = new ByteArrayOutputStream();
     PrintStream ps = new PrintStream(bos);
     System.out.println("start dumpinfo 22===========" + bos.toString());


Reply via email to