Author: olga Date: Wed Jul 9 17:10:51 2008 New Revision: 675386 URL: http://svn.apache.org/viewvc?rev=675386&view=rev Log: PIG-258: Pig should cleanup output directory of a failed query
Added: incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java Added: incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java?rev=675386&view=auto ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java (added) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestDeleteOnFail.java Wed Jul 9 17:10:51 2008 @@ -0,0 +1,253 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.pig.test; + +import java.io.BufferedReader; +import java.io.File; +import java.io.FileOutputStream; +import java.io.InputStreamReader; +import java.io.PrintStream; +import java.util.Random; + +import junit.framework.TestCase; + +import org.apache.pig.PigServer; +import org.apache.pig.impl.PigContext; +import org.junit.Before; +import org.junit.Test; + +public class TestDeleteOnFail extends TestCase { + + int LOOP_COUNT = 100; + File tmpFile = null; + PigServer server = null; + PigContext context = null; + + + @Before + public void setUp() throws Exception { + tmpFile = File.createTempFile("test", ".txt"); + tmpFile.deleteOnExit(); + FileOutputStream fos = new FileOutputStream(tmpFile); + PrintStream ps = new PrintStream(fos); + Random r1 = new Random(5); + Random r2 = new Random(3); + + for(int i = 0; i < LOOP_COUNT; i++) { + ps.println((char)('a'+r1.nextInt(26)) + "\t" + r2.nextInt(100)); + } + ps.close(); + fos.close(); + + } + + static public boolean deleteDirectory(File path) { + if( path.exists() ) { + File[] files = path.listFiles(); + for(int i=0; i<files.length; i++) { + if(files[i].isDirectory()) { + deleteDirectory(files[i]); + } + else { + files[i].delete(); + } + } + } + return( path.delete() ); + } + + @Test + public void testOneSuccStore() throws Throwable { + File scriptFile = File.createTempFile("script", ".pig"); + scriptFile.deleteOnExit(); + FileOutputStream fos = new FileOutputStream(scriptFile); + PrintStream ps = new PrintStream(fos); + File outputFolder = File.createTempFile("output", ".data"); + outputFolder.delete(); + + ps.println("a = load 'file:" + tmpFile + "';"); + ps.println("b = filter a by $0>'g';"); // successful statement + ps.println("store b into '" + outputFolder + "';"); + ps.close(); + fos.close(); + + Process p = Runtime.getRuntime().exec("java -cp pig.jar org.apache.pig.Main -f " + scriptFile); + p.waitFor(); + InputStreamReader isr = new InputStreamReader(p.getInputStream()); + BufferedReader br = new BufferedReader(isr); + String line; + String message = null; + while ((line= br.readLine())!=null) + { + message+=line; + message+="\n"; + } + System.out.println(message); + + File o = new File(outputFolder.toString()); + assertTrue(o.exists()); + deleteDirectory(o); + } + + @Test + public void testTwoSuccStore() throws Throwable { + File scriptFile = File.createTempFile("script", ".pig"); + scriptFile.deleteOnExit(); + FileOutputStream fos = new FileOutputStream(scriptFile); + PrintStream ps = new PrintStream(fos); + File outputFolder1 = File.createTempFile("output", ".data"); + outputFolder1.delete(); + File outputFolder2 = File.createTempFile("output", ".data"); + outputFolder2.delete(); + + ps.println("a = load 'file:" + tmpFile + "';"); + ps.println("b = filter a by $0>'g';"); + ps.println("store b into '" + outputFolder1 + "';"); // successful statement + ps.println("store b into '" + outputFolder2 + "';"); // successful statement + ps.close(); + fos.close(); + + Process p = Runtime.getRuntime().exec("java -cp pig.jar org.apache.pig.Main -f " + scriptFile); + p.waitFor(); + InputStreamReader isr = new InputStreamReader(p.getInputStream()); + BufferedReader br = new BufferedReader(isr); + String line; + String message = null; + while ((line= br.readLine())!=null) + { + message+=line; + message+="\n"; + } + System.out.println(message); + + File o1 = new File(outputFolder1.toString()); + assertTrue(o1.exists()); + File o2 = new File(outputFolder1.toString()); + assertTrue(o2.exists()); + deleteDirectory(o1); + deleteDirectory(o2); + } + + @Test + public void testOneFailStore() throws Throwable { + File scriptFile = File.createTempFile("script", ".pig"); + scriptFile.deleteOnExit(); + FileOutputStream fos = new FileOutputStream(scriptFile); + PrintStream ps = new PrintStream(fos); + File outputFolder = File.createTempFile("output", ".data"); + outputFolder.delete(); + + ps.println("a = load 'file:" + tmpFile + "';"); + ps.println("b = filter a by $0>3;"); // failed statement + ps.println("store b into '" + outputFolder + "';"); + ps.close(); + fos.close(); + + Process p = Runtime.getRuntime().exec("java -cp pig.jar org.apache.pig.Main -f " + scriptFile); + p.waitFor(); + InputStreamReader isr = new InputStreamReader(p.getInputStream()); + BufferedReader br = new BufferedReader(isr); + String line; + String message = null; + while ((line= br.readLine())!=null) + { + message+=line; + message+="\n"; + } + System.out.println(message); + + File o = new File(outputFolder.toString()); + assertFalse(o.exists()); + } + + @Test + public void testTwoFailStore() throws Throwable { + File scriptFile = File.createTempFile("script", ".pig"); + scriptFile.deleteOnExit(); + FileOutputStream fos = new FileOutputStream(scriptFile); + PrintStream ps = new PrintStream(fos); + File outputFolder1 = File.createTempFile("output", ".data"); + outputFolder1.delete(); + File outputFolder2 = File.createTempFile("output", ".data"); + outputFolder2.delete(); + + ps.println("a = load 'file:" + tmpFile + "';"); + ps.println("b = filter a by $0>3;"); + ps.println("store b into '" + outputFolder1 + "';"); // failed statement + ps.println("store b into '" + outputFolder2 + "';"); // failed statement + ps.close(); + fos.close(); + + Process p = Runtime.getRuntime().exec("java -cp pig.jar org.apache.pig.Main -f " + scriptFile); + p.waitFor(); + InputStreamReader isr = new InputStreamReader(p.getInputStream()); + BufferedReader br = new BufferedReader(isr); + String line; + String message = null; + while ((line= br.readLine())!=null) + { + message+=line; + message+="\n"; + } + System.out.println(message); + + File o1 = new File(outputFolder1.toString()); + assertFalse(o1.exists()); + File o2 = new File(outputFolder2.toString()); + assertFalse(o2.exists()); + } + + @Test + public void testOneSuccOneFailStore() throws Throwable { + File scriptFile = File.createTempFile("script", ".pig"); + scriptFile.deleteOnExit(); + FileOutputStream fos = new FileOutputStream(scriptFile); + PrintStream ps = new PrintStream(fos); + File outputFolder1 = File.createTempFile("output", ".data"); + outputFolder1.delete(); + File outputFolder2 = File.createTempFile("output", ".data"); + outputFolder2.delete(); + + ps.println("a = load 'file:" + tmpFile + "';"); + ps.println("b = filter a by $0>3;"); // failed statement + ps.println("c = filter a by $0>'g';"); // successful statement + ps.println("store b into '" + outputFolder1 + "';"); + ps.println("store c into '" + outputFolder2 + "';"); + ps.close(); + fos.close(); + + Process p = Runtime.getRuntime().exec("java -cp pig.jar org.apache.pig.Main -f " + scriptFile); + p.waitFor(); + InputStreamReader isr = new InputStreamReader(p.getInputStream()); + BufferedReader br = new BufferedReader(isr); + String line; + String message = null; + while ((line= br.readLine())!=null) + { + message+=line; + message+="\n"; + } + System.out.println(message); + + File o1 = new File(outputFolder1.toString()); + assertFalse(o1.exists()); + File o2 = new File(outputFolder2.toString()); + assertTrue(o2.exists()); + deleteDirectory(o2); + } +}