Author: olga
Date: Wed Jul  9 17:10:51 2008
New Revision: 675386

URL: http://svn.apache.org/viewvc?rev=675386&view=rev
Log:
PIG-258: Pig should cleanup output directory of a failed query

Added:
    incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java

Added: 
incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java
URL: 
http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java?rev=675386&view=auto
==============================================================================
--- incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java 
(added)
+++ incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java 
Wed Jul  9 17:10:51 2008
@@ -0,0 +1,253 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.InputStreamReader;
+import java.io.PrintStream;
+import java.util.Random;
+
+import junit.framework.TestCase;
+
+import org.apache.pig.PigServer;
+import org.apache.pig.impl.PigContext;
+import org.junit.Before;
+import org.junit.Test;
+
+public class TestDeleteOnFail extends TestCase {
+    
+    int LOOP_COUNT = 100;
+    File tmpFile = null;
+    PigServer server = null;
+    PigContext context = null;
+    
+    
+    @Before
+    public void setUp() throws Exception {
+       tmpFile = File.createTempFile("test", ".txt");
+       tmpFile.deleteOnExit();
+       FileOutputStream fos = new FileOutputStream(tmpFile);
+       PrintStream ps = new PrintStream(fos);
+        Random r1 = new Random(5);
+        Random r2 = new Random(3);
+        
+        for(int i = 0; i < LOOP_COUNT; i++) {
+            ps.println((char)('a'+r1.nextInt(26)) + "\t" + r2.nextInt(100));
+        }
+        ps.close();
+        fos.close();
+        
+    }
+    
+    static public boolean deleteDirectory(File path) {
+        if( path.exists() ) {
+          File[] files = path.listFiles();
+          for(int i=0; i<files.length; i++) {
+             if(files[i].isDirectory()) {
+               deleteDirectory(files[i]);
+             }
+             else {
+               files[i].delete();
+             }
+          }
+        }
+        return( path.delete() );
+    }
+    
+    @Test
+    public void testOneSuccStore() throws Throwable {
+        File scriptFile = File.createTempFile("script", ".pig");
+        scriptFile.deleteOnExit();
+        FileOutputStream fos = new FileOutputStream(scriptFile);
+        PrintStream ps = new PrintStream(fos);
+        File outputFolder = File.createTempFile("output", ".data");
+        outputFolder.delete();
+        
+        ps.println("a = load 'file:" + tmpFile + "';");
+        ps.println("b = filter a by $0>'g';");  // successful statement
+        ps.println("store b into '" + outputFolder + "';");
+        ps.close();
+        fos.close();
+        
+        Process p = Runtime.getRuntime().exec("java -cp pig.jar 
org.apache.pig.Main -f " + scriptFile);
+        p.waitFor();
+        InputStreamReader isr = new InputStreamReader(p.getInputStream());
+        BufferedReader br = new BufferedReader(isr);
+        String line;
+        String message = null;
+        while ((line= br.readLine())!=null)
+        {
+               message+=line;
+               message+="\n";
+        }
+        System.out.println(message);
+        
+        File o = new File(outputFolder.toString());
+        assertTrue(o.exists());
+        deleteDirectory(o);
+    }
+    
+    @Test
+    public void testTwoSuccStore() throws Throwable {
+        File scriptFile = File.createTempFile("script", ".pig");
+        scriptFile.deleteOnExit();
+        FileOutputStream fos = new FileOutputStream(scriptFile);
+        PrintStream ps = new PrintStream(fos);
+        File outputFolder1 = File.createTempFile("output", ".data");
+        outputFolder1.delete();
+        File outputFolder2 = File.createTempFile("output", ".data");
+        outputFolder2.delete();
+        
+        ps.println("a = load 'file:" + tmpFile + "';");
+        ps.println("b = filter a by $0>'g';");
+        ps.println("store b into '" + outputFolder1 + "';");  // successful 
statement
+        ps.println("store b into '" + outputFolder2 + "';");  // successful 
statement
+        ps.close();
+        fos.close();
+        
+        Process p = Runtime.getRuntime().exec("java -cp pig.jar 
org.apache.pig.Main -f " + scriptFile);
+        p.waitFor();
+        InputStreamReader isr = new InputStreamReader(p.getInputStream());
+        BufferedReader br = new BufferedReader(isr);
+        String line;
+        String message = null;
+        while ((line= br.readLine())!=null)
+        {
+               message+=line;
+               message+="\n";
+        }
+        System.out.println(message);
+        
+        File o1 = new File(outputFolder1.toString());
+        assertTrue(o1.exists());
+        File o2 = new File(outputFolder1.toString());
+        assertTrue(o2.exists());
+        deleteDirectory(o1);
+        deleteDirectory(o2);
+    }
+    
+    @Test
+    public void testOneFailStore() throws Throwable {
+        File scriptFile = File.createTempFile("script", ".pig");
+        scriptFile.deleteOnExit();
+        FileOutputStream fos = new FileOutputStream(scriptFile);
+        PrintStream ps = new PrintStream(fos);
+        File outputFolder = File.createTempFile("output", ".data");
+        outputFolder.delete();
+        
+        ps.println("a = load 'file:" + tmpFile + "';");
+        ps.println("b = filter a by $0>3;");  // failed statement
+        ps.println("store b into '" + outputFolder + "';");
+        ps.close();
+        fos.close();
+        
+        Process p = Runtime.getRuntime().exec("java -cp pig.jar 
org.apache.pig.Main -f " + scriptFile);
+        p.waitFor();
+        InputStreamReader isr = new InputStreamReader(p.getInputStream());
+        BufferedReader br = new BufferedReader(isr);
+        String line;
+        String message = null;
+        while ((line= br.readLine())!=null)
+        {
+               message+=line;
+               message+="\n";
+        }
+        System.out.println(message);
+        
+        File o = new File(outputFolder.toString());
+        assertFalse(o.exists());
+    }
+    
+    @Test
+    public void testTwoFailStore() throws Throwable {
+        File scriptFile = File.createTempFile("script", ".pig");
+        scriptFile.deleteOnExit();
+        FileOutputStream fos = new FileOutputStream(scriptFile);
+        PrintStream ps = new PrintStream(fos);
+        File outputFolder1 = File.createTempFile("output", ".data");
+        outputFolder1.delete();
+        File outputFolder2 = File.createTempFile("output", ".data");
+        outputFolder2.delete();
+        
+        ps.println("a = load 'file:" + tmpFile + "';");
+        ps.println("b = filter a by $0>3;");
+        ps.println("store b into '" + outputFolder1 + "';");  // failed 
statement
+        ps.println("store b into '" + outputFolder2 + "';");  // failed 
statement
+        ps.close();
+        fos.close();
+        
+        Process p = Runtime.getRuntime().exec("java -cp pig.jar 
org.apache.pig.Main -f " + scriptFile);
+        p.waitFor();
+        InputStreamReader isr = new InputStreamReader(p.getInputStream());
+        BufferedReader br = new BufferedReader(isr);
+        String line;
+        String message = null;
+        while ((line= br.readLine())!=null)
+        {
+               message+=line;
+               message+="\n";
+        }
+        System.out.println(message);
+
+        File o1 = new File(outputFolder1.toString());
+        assertFalse(o1.exists());
+        File o2 = new File(outputFolder2.toString());
+        assertFalse(o2.exists());
+    }
+    
+    @Test
+    public void testOneSuccOneFailStore() throws Throwable {
+        File scriptFile = File.createTempFile("script", ".pig");
+        scriptFile.deleteOnExit();
+        FileOutputStream fos = new FileOutputStream(scriptFile);
+        PrintStream ps = new PrintStream(fos);
+        File outputFolder1 = File.createTempFile("output", ".data");
+        outputFolder1.delete();
+        File outputFolder2 = File.createTempFile("output", ".data");
+        outputFolder2.delete();
+        
+        ps.println("a = load 'file:" + tmpFile + "';");
+        ps.println("b = filter a by $0>3;");  // failed statement
+        ps.println("c = filter a by $0>'g';");  // successful statement
+        ps.println("store b into '" + outputFolder1 + "';");
+        ps.println("store c into '" + outputFolder2 + "';");
+        ps.close();
+        fos.close();
+        
+        Process p = Runtime.getRuntime().exec("java -cp pig.jar 
org.apache.pig.Main -f " + scriptFile);
+        p.waitFor();
+        InputStreamReader isr = new InputStreamReader(p.getInputStream());
+        BufferedReader br = new BufferedReader(isr);
+        String line;
+        String message = null;
+        while ((line= br.readLine())!=null)
+        {
+               message+=line;
+               message+="\n";
+        }
+        System.out.println(message);
+
+        File o1 = new File(outputFolder1.toString());
+        assertFalse(o1.exists());
+        File o2 = new File(outputFolder2.toString());
+        assertTrue(o2.exists());
+        deleteDirectory(o2);
+    }
+}


Reply via email to