Author: olga Date: Tue Sep 29 22:31:21 2009 New Revision: 820112 URL: http://svn.apache.org/viewvc?rev=820112&view=rev Log: PIG-956: 10 minute commit tests (olgan)
Added: hadoop/pig/branches/branch-0.5/test/all-tests hadoop/pig/branches/branch-0.5/test/commit-tests hadoop/pig/branches/branch-0.5/test/org/apache/pig/test/TestCommit.java Modified: hadoop/pig/branches/branch-0.5/CHANGES.txt hadoop/pig/branches/branch-0.5/build.xml Modified: hadoop/pig/branches/branch-0.5/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.5/CHANGES.txt?rev=820112&r1=820111&r2=820112&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.5/CHANGES.txt (original) +++ hadoop/pig/branches/branch-0.5/CHANGES.txt Tue Sep 29 22:31:21 2009 @@ -30,8 +30,9 @@ PIG-963: Join in local mode matches null keys (pradeepkth) PIG-660: Integration with Hadoop 20 (sms via olgan) +PIG-956: 10 minute commit tests (olgan) -Release 0.4.0 - Unreleased +Release 0.4.0 - 2009-09-26 INCOMPATIBLE CHANGES Modified: hadoop/pig/branches/branch-0.5/build.xml URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.5/build.xml?rev=820112&r1=820111&r2=820112&view=diff ============================================================================== --- hadoop/pig/branches/branch-0.5/build.xml (original) +++ hadoop/pig/branches/branch-0.5/build.xml Tue Sep 29 22:31:21 2009 @@ -76,6 +76,9 @@ <property name="test.log.dir" value="${test.build.dir}/logs" /> <property name="test.timeout" value="2700000" /> <property name="test.junit.output.format" value="plain" /> + <property name="test.commit.file" value="${test.src.dir}/commit-tests"/> + <property name="test.all.file" value="${test.src.dir}/all-tests"/> + <!-- test configuration, use ${user.home}/build.properties to configure values --> <property name="ssh.gateway" value="" /> @@ -421,7 +424,17 @@ <!-- ================================================================== --> <!-- Run unit tests --> <!-- ================================================================== --> - <target name="test-core" depends="compile-test,jar"> + <target name="test-core" depends="compile-test,jar" description="Run full set of unit tests"> + <macro-test-runner test.file="${test.all.file}" /> + </target> + + <target name="test-commit" depends="compile-test,jar" description="Run approximate 10-minute set of unit tests prior to commiting"> + <macro-test-runner test.file="${test.commit.file}" /> + </target> + + <macrodef name="macro-test-runner"> + <attribute name="test.file" /> + <sequential> <delete dir="${test.log.dir}"/> <mkdir dir="${test.log.dir}"/> <junit showoutput="${test.output}" printsummary="yes" haltonfailure="no" fork="yes" maxmemory="256m" dir="${basedir}" timeout="${test.timeout}" errorProperty="tests.failed" failureProperty="tests.failed"> @@ -442,7 +455,11 @@ <batchtest fork="yes" todir="${test.log.dir}" unless="testcase"> <fileset dir="test"> - <include name="**/*Test*.java" /> + <patternset> + <includesfile name="@{test.file}"/> + </patternset> + + <!--include name="**/*Test*.java" /--> <!-- Excluced because they are end-to-end, don't work yet. --> <!-- <exclude name="**/TestFilterOpNumeric.java" /> @@ -470,7 +487,8 @@ </batchtest> </junit> <fail if="tests.failed">Tests failed!</fail> - </target> + </sequential> + </macrodef> <target name="test" description="to call the test-core and test-contrib target"> <antcall target="test-core" inheritRefs="true" inheritall="true"/> Added: hadoop/pig/branches/branch-0.5/test/all-tests URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.5/test/all-tests?rev=820112&view=auto ============================================================================== --- hadoop/pig/branches/branch-0.5/test/all-tests (added) +++ hadoop/pig/branches/branch-0.5/test/all-tests Tue Sep 29 22:31:21 2009 @@ -0,0 +1 @@ +**/Test*.java Added: hadoop/pig/branches/branch-0.5/test/commit-tests URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.5/test/commit-tests?rev=820112&view=auto ============================================================================== --- hadoop/pig/branches/branch-0.5/test/commit-tests (added) +++ hadoop/pig/branches/branch-0.5/test/commit-tests Tue Sep 29 22:31:21 2009 @@ -0,0 +1,70 @@ +**/TestAdd.java +**/TestBagFormat.java +**/TestBinaryStorage.java +**/TestBoolean.java +**/TestBuiltin.java +**/TestCmdLineParser.java +**/TestCommit.java +**/TestConstExpr.java +**/TestConversions.java +**/TestDataBag.java +**/TestDataModel.java +**/TestDeleteOnFail.java +**/TestDivide.java +**/TestEqualTo.java +**/TestEvalPipelineLocal.java +**/TestExampleGenerator.java +**/TestTestFilter.java +**/TestForEach.java +**/TestForEachNestedPlanLocal.java +**/TestFuncSpec.java +**/TestGTOrEqual.java +**/TestGreaterThan.java +**/TestImplicitSplit.java +**/TestInputOutputFileValidator.java +**/TestInstantiateFunc.java +**/TestLTOrEqual.java +**/TestLessThan.java +**/TestLoad.java +**/TestLocal.java +**/TestLocal2.java +**/TestLocalPOSplit.java +**/TestLocalRearrange.java +**/TestLogToPhyCompiler.java +**/TestLogicalOptimizer.java +**/TestLogicalPlanBuilder.java +**/TestMRCompiler.java +**/TestMod.java +**/TestMultiQueryLocal.java +**/TestMultiply.java +**/TestNotEqualTo.java +**/TestNull.java +**/TestOperatorPlan.java +**/TestPOBinCond.java +**/TestPOCast.java +**/TestPOCogroup.java +**/TestPOCros.java +**/TestPODistinct.java +**/TestPOGenerate.java +**/TestPOMapLookUp.java +**/TestPOSort.java +**/TestPOUserFunc.java +**/TestParamSubPreproc.java +**/TestPhyOp.java +**/TestPigScriptParser.java +**/TestPigServer.java +**/TestPigSplit.java +**/TestPoissonSampleLoader.java +**/TestProject.java +**/TestRegexp.java +**/TestSample.java +**/TestSchema.java +**/TestSchemaParser.java +**/TestSchemaUtil.java +**/TestStore.java +**/TestStreaming.java +**/TestStreamingLocal.java +**/TestTextDataParse.java +**/TestTupleFormat.java +**/TestTypeChecking.java +**/TestTypeCheckingValidatorNoSchema.java Added: hadoop/pig/branches/branch-0.5/test/org/apache/pig/test/TestCommit.java URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.5/test/org/apache/pig/test/TestCommit.java?rev=820112&view=auto ============================================================================== --- hadoop/pig/branches/branch-0.5/test/org/apache/pig/test/TestCommit.java (added) +++ hadoop/pig/branches/branch-0.5/test/org/apache/pig/test/TestCommit.java Tue Sep 29 22:31:21 2009 @@ -0,0 +1,155 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.pig.test; + +import static org.apache.pig.ExecType.MAPREDUCE; + +import java.io.File; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.PrintStream; +import java.io.PrintWriter; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Random; +import java.util.StringTokenizer; + +import org.junit.Before; +import org.junit.Test; + +import org.apache.pig.ComparisonFunc; +import org.apache.pig.EvalFunc; +import org.apache.pig.ExecType; +import org.apache.pig.PigServer; +import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.builtin.BinStorage; +import org.apache.pig.builtin.Distinct; +import org.apache.pig.builtin.PigStorage; +import org.apache.pig.builtin.TextLoader; +import org.apache.pig.data.*; +import org.apache.pig.impl.io.FileLocalizer; +import org.apache.pig.impl.io.PigFile; +import org.apache.pig.impl.logicalLayer.schema.Schema; +import org.apache.pig.impl.logicalLayer.FrontendException; +import org.apache.pig.impl.util.Pair; +import org.apache.pig.test.utils.GenRandomData; +import org.apache.pig.test.utils.Identity; + +import junit.framework.TestCase; + +public class TestCommit extends TestCase { + + MiniCluster cluster = MiniCluster.buildCluster(); + private PigServer pigServer; + + TupleFactory mTf = TupleFactory.getInstance(); + + @Before + @Override + public void setUp() throws Exception{ + pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties()); + } + + @Test + public void testCheckin1() throws Exception{ + Tuple expected1 = mTf.newTuple(2); + Tuple expected2 = mTf.newTuple(2); + expected1.set(0, "independent"); + expected1.set(1, 50.0); + expected2.set(0, "democrat"); + expected2.set(1, 125.5); + File student = Util.createFile(new String[]{"joe smith:18:3.5","amy brown:25:2.5","jim fox:20:4.0","leo fu:55:3.0"}); + File voter = Util.createFile(new String[]{"amy brown,25,democrat,25.50","amy brown,25,democrat,100","jim fox,20,independent,50.0"}); + + pigServer.registerQuery("a = load '" + Util.generateURI(student.toString()) + "' using " + PigStorage.class.getName() + "(':') as (name, age, gpa);"); + pigServer.registerQuery("b = load '" + Util.generateURI(voter.toString()) + "' using " + PigStorage.class.getName() + "(',') as (name, age, registration, contributions);"); + pigServer.registerQuery("c = filter a by age < 50;"); + pigServer.registerQuery("d = filter b by age < 50;"); + pigServer.registerQuery("e = cogroup c by (name, age), d by (name, age);"); + pigServer.registerQuery("f = foreach e generate flatten(c), flatten(d);"); + pigServer.registerQuery("g = group f by registration;"); + pigServer.registerQuery("h = foreach g generate (chararray)group, SUM(f.d::contributions);"); + pigServer.registerQuery("i = order h by $1;"); + + Iterator<Tuple> iter = pigServer.openIterator("i"); + int count = 0; + while(iter.hasNext()){ + Tuple t = iter.next(); + count++; + if (count == 1) { + assertTrue(t.get(0).equals(expected1.get(0))); + assertTrue(t.get(1).equals(expected1.get(1))); + } else if (count == 2){ + assertTrue(t.get(0).equals(expected2.get(0))); + assertTrue(t.get(1).equals(expected2.get(1))); + } + } + assertEquals(count, 2); + } + + @Test + public void testCheckin2() throws Exception{ + Tuple expected1 = mTf.newTuple(4); + Tuple expected2 = mTf.newTuple(4); + String tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null, pigServer.getPigContext()).toString() + "'"; + File student = Util.createFile(new String[]{"joe smith:18:3.5","amy brown:18:2.5","jim fox:20:4.0","leo fu:55:3.0", "amy smith:20:3.0"}); + expected1.set(0, 18); + expected1.set(1, 1L); + expected1.set(2, "joe smith"); + expected1.set(3, 18); + expected2.set(0, 55); + expected2.set(1, 1L); + expected2.set(2, "leo fu"); + expected2.set(3, 55); + + pigServer.registerQuery("a = load '" + Util.generateURI(student.toString()) + + "' using " + PigStorage.class.getName() + "(':') as (name: chararray, age: int, gpa: float);"); + pigServer.registerQuery("b = group a by age;"); + //pigServer.registerQuery("c = foreach b { d = order a by $0; generate group, COUNT(d), MAX (d), MIN(d.$0);}; "); + pigServer.registerQuery("c = foreach b { d = filter a by gpa > 2.5; " + + "e = order a by name; f = a.age; g = distinct f; " + + " generate group, COUNT(d), MAX (e.name), MIN(g.$0);};"); + pigServer.registerQuery("h = order c by $1;"); + pigServer.registerQuery("i = limit h 2;"); + pigServer.store("i", tmpFile1); + pigServer.registerQuery("x = load " + tmpFile1 + " as (age: int, cnt: long, max: chararray, min: int);"); + pigServer.registerQuery("y = foreach x generate age, cnt, max, min;"); + Iterator<Tuple> iter = pigServer.openIterator("y"); + int count = 0; + while(iter.hasNext()){ + Tuple t = iter.next(); + count++; + if (count == 1) { + assertTrue(t.get(0).equals(expected1.get(0))); + assertTrue(t.get(1).equals(expected1.get(1))); + assertTrue(t.get(2).equals(expected1.get(2))); + assertTrue(t.get(3).equals(expected1.get(3))); + } else if (count == 2){ + assertTrue(t.get(0).equals(expected2.get(0))); + assertTrue(t.get(1).equals(expected2.get(1))); + assertTrue(t.get(2).equals(expected2.get(2))); + assertTrue(t.get(3).equals(expected2.get(3))); + } + } + assertEquals(count, 2); + } +}