Author: olga
Date: Tue Sep 29 22:28:27 2009
New Revision: 820111
URL: http://svn.apache.org/viewvc?rev=820111&view=rev
Log:
PIG-956: 10 minute commit tests (olgan)
Added:
hadoop/pig/trunk/test/all-tests
hadoop/pig/trunk/test/commit-tests
hadoop/pig/trunk/test/org/apache/pig/test/TestCommit.java
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/build.xml
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=820111&r1=820110&r2=820111&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Tue Sep 29 22:28:27 2009
@@ -28,6 +28,8 @@
PIG-891: Fixing dfs statement for Pig (zjffdu via daijy).
+PIG-956: 10 minute commit tests (olgan)
+
OPTIMIZATIONS
BUG FIXES
@@ -55,7 +57,7 @@
PIG-963: Join in local mode matches null keys (pradeepkth)
PIG-660: Integration with Hadoop 20 (sms via olgan)
-Release 0.4.0 - Unreleased
+Release 0.4.0 - 2009-09-26
INCOMPATIBLE CHANGES
Modified: hadoop/pig/trunk/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/build.xml?rev=820111&r1=820110&r2=820111&view=diff
==============================================================================
--- hadoop/pig/trunk/build.xml (original)
+++ hadoop/pig/trunk/build.xml Tue Sep 29 22:28:27 2009
@@ -76,6 +76,9 @@
<property name="test.log.dir" value="${test.build.dir}/logs" />
<property name="test.timeout" value="2700000" />
<property name="test.junit.output.format" value="plain" />
+ <property name="test.commit.file" value="${test.src.dir}/commit-tests"/>
+ <property name="test.all.file" value="${test.src.dir}/all-tests"/>
+
<!-- test configuration, use ${user.home}/build.properties to configure
values -->
<property name="ssh.gateway" value="" />
@@ -421,7 +424,17 @@
<!-- ================================================================== -->
<!-- Run unit tests -->
<!-- ================================================================== -->
- <target name="test-core" depends="compile-test,jar">
+ <target name="test-core" depends="compile-test,jar" description="Run full
set of unit tests">
+ <macro-test-runner test.file="${test.all.file}" />
+ </target>
+
+ <target name="test-commit" depends="compile-test,jar" description="Run
approximate 10-minute set of unit tests prior to commiting">
+ <macro-test-runner test.file="${test.commit.file}" />
+ </target>
+
+ <macrodef name="macro-test-runner">
+ <attribute name="test.file" />
+ <sequential>
<delete dir="${test.log.dir}"/>
<mkdir dir="${test.log.dir}"/>
<junit showoutput="${test.output}" printsummary="yes"
haltonfailure="no" fork="yes" maxmemory="256m" dir="${basedir}"
timeout="${test.timeout}" errorProperty="tests.failed"
failureProperty="tests.failed">
@@ -442,7 +455,11 @@
<batchtest fork="yes" todir="${test.log.dir}" unless="testcase">
<fileset dir="test">
- <include name="**/*Test*.java" />
+ <patternset>
+ <includesfile name="@{test.file}"/>
+ </patternset>
+
+ <!--include name="**/*Test*.java" /-->
<!-- Excluced because they are end-to-end, don't work yet.
-->
<!--
<exclude name="**/TestFilterOpNumeric.java" />
@@ -470,7 +487,8 @@
</batchtest>
</junit>
<fail if="tests.failed">Tests failed!</fail>
- </target>
+ </sequential>
+ </macrodef>
<target name="test" description="to call the test-core and test-contrib
target">
<antcall target="test-core" inheritRefs="true" inheritall="true"/>
Added: hadoop/pig/trunk/test/all-tests
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/all-tests?rev=820111&view=auto
==============================================================================
--- hadoop/pig/trunk/test/all-tests (added)
+++ hadoop/pig/trunk/test/all-tests Tue Sep 29 22:28:27 2009
@@ -0,0 +1 @@
+**/Test*.java
Added: hadoop/pig/trunk/test/commit-tests
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/commit-tests?rev=820111&view=auto
==============================================================================
--- hadoop/pig/trunk/test/commit-tests (added)
+++ hadoop/pig/trunk/test/commit-tests Tue Sep 29 22:28:27 2009
@@ -0,0 +1,70 @@
+**/TestAdd.java
+**/TestBagFormat.java
+**/TestBinaryStorage.java
+**/TestBoolean.java
+**/TestBuiltin.java
+**/TestCmdLineParser.java
+**/TestCommit.java
+**/TestConstExpr.java
+**/TestConversions.java
+**/TestDataBag.java
+**/TestDataModel.java
+**/TestDeleteOnFail.java
+**/TestDivide.java
+**/TestEqualTo.java
+**/TestEvalPipelineLocal.java
+**/TestExampleGenerator.java
+**/TestTestFilter.java
+**/TestForEach.java
+**/TestForEachNestedPlanLocal.java
+**/TestFuncSpec.java
+**/TestGTOrEqual.java
+**/TestGreaterThan.java
+**/TestImplicitSplit.java
+**/TestInputOutputFileValidator.java
+**/TestInstantiateFunc.java
+**/TestLTOrEqual.java
+**/TestLessThan.java
+**/TestLoad.java
+**/TestLocal.java
+**/TestLocal2.java
+**/TestLocalPOSplit.java
+**/TestLocalRearrange.java
+**/TestLogToPhyCompiler.java
+**/TestLogicalOptimizer.java
+**/TestLogicalPlanBuilder.java
+**/TestMRCompiler.java
+**/TestMod.java
+**/TestMultiQueryLocal.java
+**/TestMultiply.java
+**/TestNotEqualTo.java
+**/TestNull.java
+**/TestOperatorPlan.java
+**/TestPOBinCond.java
+**/TestPOCast.java
+**/TestPOCogroup.java
+**/TestPOCros.java
+**/TestPODistinct.java
+**/TestPOGenerate.java
+**/TestPOMapLookUp.java
+**/TestPOSort.java
+**/TestPOUserFunc.java
+**/TestParamSubPreproc.java
+**/TestPhyOp.java
+**/TestPigScriptParser.java
+**/TestPigServer.java
+**/TestPigSplit.java
+**/TestPoissonSampleLoader.java
+**/TestProject.java
+**/TestRegexp.java
+**/TestSample.java
+**/TestSchema.java
+**/TestSchemaParser.java
+**/TestSchemaUtil.java
+**/TestStore.java
+**/TestStreaming.java
+**/TestStreamingLocal.java
+**/TestTextDataParse.java
+**/TestTupleFormat.java
+**/TestTypeChecking.java
+**/TestTypeCheckingValidatorNoSchema.java
Added: hadoop/pig/trunk/test/org/apache/pig/test/TestCommit.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestCommit.java?rev=820111&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestCommit.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestCommit.java Tue Sep 29
22:28:27 2009
@@ -0,0 +1,155 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import static org.apache.pig.ExecType.MAPREDUCE;
+
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.io.PrintWriter;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.StringTokenizer;
+
+import org.junit.Before;
+import org.junit.Test;
+
+import org.apache.pig.ComparisonFunc;
+import org.apache.pig.EvalFunc;
+import org.apache.pig.ExecType;
+import org.apache.pig.PigServer;
+import org.apache.pig.backend.executionengine.ExecException;
+import org.apache.pig.builtin.BinStorage;
+import org.apache.pig.builtin.Distinct;
+import org.apache.pig.builtin.PigStorage;
+import org.apache.pig.builtin.TextLoader;
+import org.apache.pig.data.*;
+import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.impl.io.PigFile;
+import org.apache.pig.impl.logicalLayer.schema.Schema;
+import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.impl.util.Pair;
+import org.apache.pig.test.utils.GenRandomData;
+import org.apache.pig.test.utils.Identity;
+
+import junit.framework.TestCase;
+
+public class TestCommit extends TestCase {
+
+ MiniCluster cluster = MiniCluster.buildCluster();
+ private PigServer pigServer;
+
+ TupleFactory mTf = TupleFactory.getInstance();
+
+ @Before
+ @Override
+ public void setUp() throws Exception{
+ pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+ }
+
+ @Test
+ public void testCheckin1() throws Exception{
+ Tuple expected1 = mTf.newTuple(2);
+ Tuple expected2 = mTf.newTuple(2);
+ expected1.set(0, "independent");
+ expected1.set(1, 50.0);
+ expected2.set(0, "democrat");
+ expected2.set(1, 125.5);
+ File student = Util.createFile(new String[]{"joe smith:18:3.5","amy
brown:25:2.5","jim fox:20:4.0","leo fu:55:3.0"});
+ File voter = Util.createFile(new String[]{"amy
brown,25,democrat,25.50","amy brown,25,democrat,100","jim
fox,20,independent,50.0"});
+
+ pigServer.registerQuery("a = load '" +
Util.generateURI(student.toString()) + "' using " + PigStorage.class.getName()
+ "(':') as (name, age, gpa);");
+ pigServer.registerQuery("b = load '" +
Util.generateURI(voter.toString()) + "' using " + PigStorage.class.getName() +
"(',') as (name, age, registration, contributions);");
+ pigServer.registerQuery("c = filter a by age < 50;");
+ pigServer.registerQuery("d = filter b by age < 50;");
+ pigServer.registerQuery("e = cogroup c by (name, age), d by (name,
age);");
+ pigServer.registerQuery("f = foreach e generate flatten(c),
flatten(d);");
+ pigServer.registerQuery("g = group f by registration;");
+ pigServer.registerQuery("h = foreach g generate (chararray)group,
SUM(f.d::contributions);");
+ pigServer.registerQuery("i = order h by $1;");
+
+ Iterator<Tuple> iter = pigServer.openIterator("i");
+ int count = 0;
+ while(iter.hasNext()){
+ Tuple t = iter.next();
+ count++;
+ if (count == 1) {
+ assertTrue(t.get(0).equals(expected1.get(0)));
+ assertTrue(t.get(1).equals(expected1.get(1)));
+ } else if (count == 2){
+ assertTrue(t.get(0).equals(expected2.get(0)));
+ assertTrue(t.get(1).equals(expected2.get(1)));
+ }
+ }
+ assertEquals(count, 2);
+ }
+
+ @Test
+ public void testCheckin2() throws Exception{
+ Tuple expected1 = mTf.newTuple(4);
+ Tuple expected2 = mTf.newTuple(4);
+ String tmpFile1 = "'" + FileLocalizer.getTemporaryPath(null,
pigServer.getPigContext()).toString() + "'";
+ File student = Util.createFile(new String[]{"joe smith:18:3.5","amy
brown:18:2.5","jim fox:20:4.0","leo fu:55:3.0", "amy smith:20:3.0"});
+ expected1.set(0, 18);
+ expected1.set(1, 1L);
+ expected1.set(2, "joe smith");
+ expected1.set(3, 18);
+ expected2.set(0, 55);
+ expected2.set(1, 1L);
+ expected2.set(2, "leo fu");
+ expected2.set(3, 55);
+
+ pigServer.registerQuery("a = load '" +
Util.generateURI(student.toString()) +
+ "' using " + PigStorage.class.getName() +
"(':') as (name: chararray, age: int, gpa: float);");
+ pigServer.registerQuery("b = group a by age;");
+ //pigServer.registerQuery("c = foreach b { d = order a by $0;
generate group, COUNT(d), MAX (d), MIN(d.$0);}; ");
+ pigServer.registerQuery("c = foreach b { d = filter a by gpa > 2.5; "
+
+ "e = order a by name; f = a.age; g = distinct
f; " +
+ " generate group, COUNT(d), MAX (e.name),
MIN(g.$0);};");
+ pigServer.registerQuery("h = order c by $1;");
+ pigServer.registerQuery("i = limit h 2;");
+ pigServer.store("i", tmpFile1);
+ pigServer.registerQuery("x = load " + tmpFile1 + " as (age: int, cnt:
long, max: chararray, min: int);");
+ pigServer.registerQuery("y = foreach x generate age, cnt, max, min;");
+ Iterator<Tuple> iter = pigServer.openIterator("y");
+ int count = 0;
+ while(iter.hasNext()){
+ Tuple t = iter.next();
+ count++;
+ if (count == 1) {
+ assertTrue(t.get(0).equals(expected1.get(0)));
+ assertTrue(t.get(1).equals(expected1.get(1)));
+ assertTrue(t.get(2).equals(expected1.get(2)));
+ assertTrue(t.get(3).equals(expected1.get(3)));
+ } else if (count == 2){
+ assertTrue(t.get(0).equals(expected2.get(0)));
+ assertTrue(t.get(1).equals(expected2.get(1)));
+ assertTrue(t.get(2).equals(expected2.get(2)));
+ assertTrue(t.get(3).equals(expected2.get(3)));
+ }
+ }
+ assertEquals(count, 2);
+ }
+}
