svn commit: r955682 - in /hadoop/pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/ src/org/apache/pig/dat
Author: dvryaboy
Date: Thu Jun 17 17:24:00 2010
New Revision: 955682
URL: http://svn.apache.org/viewvc?rev=955682&view=rev
Log:
PIG-1428: Make a StatusReporter singleton available for incrementing counters
(dvryaboy)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReducePOStoreImpl.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigHadoopLogger.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapBase.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigMapReduce.java
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PigLogger.java
hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java
hadoop/pig/trunk/src/org/apache/pig/tools/pigstats/PigStatusReporter.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=955682&r1=955681&r2=955682&view=diff
==
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Thu Jun 17 17:24:00 2010
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1428: Make a StatusReporter singleton available for incrementing counters
(dvryaboy)
+
PIG-972: Make describe work with nested foreach (aniket486 via daijy)
PIG-1438: [Performance] MultiQueryOptimizer should also merge DISTINCT jobs
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReducePOStoreImpl.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReducePOStoreImpl.java?rev=955682&r1=955681&r2=955682&view=diff
==
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReducePOStoreImpl.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReducePOStoreImpl.java
Thu Jun 17 17:24:00 2010
@@ -51,8 +51,8 @@ public class MapReducePOStoreImpl extend
// configuration below (like setting the output location) do
// not affect the caller's copy
Configuration outputConf = new
Configuration(context.getConfiguration());
-
-reporter = new PigStatusReporter(context);
+PigStatusReporter.setContext(context);
+reporter = PigStatusReporter.getInstance();
// make a copy of the Context to use here - since in the same
// task (map or reduce) we could have multiple stores, we should
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java?rev=955682&r1=955681&r2=955682&view=diff
==
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigCombiner.java
Thu Jun 17 17:24:00 2010
@@ -135,7 +135,8 @@ public class PigCombiner {
PigHadoopLogger pigHadoopLogger =
PigHadoopLogger.getInstance();
pigHadoopLogger.setAggregate(aggregateWarning);
-pigHadoopLogger.setReporter(new PigStatusReporter(context));
+PigStatusReporter.setContext(context);
+pigHadoopLogger.setReporter(PigStatusReporter.getInstance());
PhysicalOperator.setPigLogger(pigHadoopLogger);
}
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigHadoopLogger.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigHadoopLogger.java?rev=955682&r1=955681&r2=955682&view=diff
==
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigHadoopLogger.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PigHadoopLogger.java
Thu Jun 17 17:24:00 2010
@@ -19,8 +19,6 @@ package org.apache.pig.backend.hadoop.ex
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.TaskInputOutputContext;
import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigLogger;
import org.apache.pig.tools.pigstats.Pig
svn commit: r955696 - in /hadoop/pig/trunk: CHANGES.txt build.xml
Author: daijy Date: Thu Jun 17 18:21:28 2010 New Revision: 955696 URL: http://svn.apache.org/viewvc?rev=955696&view=rev Log: PIG-1450: TestAlgebraicEvalLocal failures due to OOM Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/build.xml Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=955696&r1=955695&r2=955696&view=diff == --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Jun 17 18:21:28 2010 @@ -86,6 +86,8 @@ PIG-1309: Map-side Cogroup (ashutoshc) BUG FIXES +PIG-1450: TestAlgebraicEvalLocal failures due to OOM (daijy) + PIG-1433: pig should create success file if mapreduce.fileoutputcommitter.marksuccessfuljobs is true (pradeepkth) Modified: hadoop/pig/trunk/build.xml URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/build.xml?rev=955696&r1=955695&r2=955696&view=diff == --- hadoop/pig/trunk/build.xml (original) +++ hadoop/pig/trunk/build.xml Thu Jun 17 18:21:28 2010 @@ -553,7 +553,7 @@ - +
svn commit: r955701 - in /hadoop/pig/trunk/contrib/zebra: CHANGES.txt build.xml
Author: yanz Date: Thu Jun 17 18:44:31 2010 New Revision: 955701 URL: http://svn.apache.org/viewvc?rev=955701&view=rev Log: PIG-1455 Addition of test-unit as an ant target (yanz) Modified: hadoop/pig/trunk/contrib/zebra/CHANGES.txt hadoop/pig/trunk/contrib/zebra/build.xml Modified: hadoop/pig/trunk/contrib/zebra/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/CHANGES.txt?rev=955701&r1=955700&r2=955701&view=diff == --- hadoop/pig/trunk/contrib/zebra/CHANGES.txt (original) +++ hadoop/pig/trunk/contrib/zebra/CHANGES.txt Thu Jun 17 18:44:31 2010 @@ -4,6 +4,8 @@ Trunk (unreleased changes) INCOMPATIBLE CHANGES +PIG-1455 Addition of test-unit as an ant target (yanz) + PIG-1451 Change the build.test property in build to test.build.dir to be consistent with PIG (yanz) PIG-1444 Addition of test-smoke ant target (gauravj via yanz) Modified: hadoop/pig/trunk/contrib/zebra/build.xml URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/contrib/zebra/build.xml?rev=955701&r1=955700&r2=955701&view=diff == --- hadoop/pig/trunk/contrib/zebra/build.xml (original) +++ hadoop/pig/trunk/contrib/zebra/build.xml Thu Jun 17 18:44:31 2010 @@ -176,6 +176,49 @@ to call at top-level: ant deploy-contrib Tests failed! + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Tests failed! +
svn commit: r955702 - in /hadoop/pig/branches/branch-0.7/contrib/zebra: CHANGES.txt build.xml
Author: yanz Date: Thu Jun 17 18:45:27 2010 New Revision: 955702 URL: http://svn.apache.org/viewvc?rev=955702&view=rev Log: PIG-1455 Addition of test-unit as an ant target (yanz) Modified: hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt hadoop/pig/branches/branch-0.7/contrib/zebra/build.xml Modified: hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt?rev=955702&r1=955701&r2=955702&view=diff == --- hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt (original) +++ hadoop/pig/branches/branch-0.7/contrib/zebra/CHANGES.txt Thu Jun 17 18:45:27 2010 @@ -4,6 +4,8 @@ Trunk (unreleased changes) INCOMPATIBLE CHANGES +PIG-1455 Addition of test-unit as an ant target (yanz) + PIG-1451 Change the build.test property in build to test.build.dir to be consistent with PIG (yanz) PIG-1444 Addition of test-smoke ant target (gauravj via yanz) Modified: hadoop/pig/branches/branch-0.7/contrib/zebra/build.xml URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.7/contrib/zebra/build.xml?rev=955702&r1=955701&r2=955702&view=diff == --- hadoop/pig/branches/branch-0.7/contrib/zebra/build.xml (original) +++ hadoop/pig/branches/branch-0.7/contrib/zebra/build.xml Thu Jun 17 18:45:27 2010 @@ -176,6 +176,49 @@ to call at top-level: ant deploy-contrib Tests failed! + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Tests failed! +
svn commit: r955703 - in /hadoop/pig/branches/branch-0.6/contrib/zebra: CHANGES.txt build.xml
Author: yanz Date: Thu Jun 17 18:46:24 2010 New Revision: 955703 URL: http://svn.apache.org/viewvc?rev=955703&view=rev Log: PIG-1455 Addition of test-unit as an ant target (yanz) Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt hadoop/pig/branches/branch-0.6/contrib/zebra/build.xml Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt?rev=955703&r1=955702&r2=955703&view=diff == --- hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt (original) +++ hadoop/pig/branches/branch-0.6/contrib/zebra/CHANGES.txt Thu Jun 17 18:46:24 2010 @@ -4,6 +4,8 @@ Trunk (unreleased changes) INCOMPATIBLE CHANGES +PIG-1455 Addition of test-unit as an ant target (yanz) + PIG-1451 Change the build.test property in build to test.build.dir to be consistent with PIG (yanz) PIG-1444 Addition of test-smoke ant target (gauravj via yanz) Modified: hadoop/pig/branches/branch-0.6/contrib/zebra/build.xml URL: http://svn.apache.org/viewvc/hadoop/pig/branches/branch-0.6/contrib/zebra/build.xml?rev=955703&r1=955702&r2=955703&view=diff == --- hadoop/pig/branches/branch-0.6/contrib/zebra/build.xml (original) +++ hadoop/pig/branches/branch-0.6/contrib/zebra/build.xml Thu Jun 17 18:46:24 2010 @@ -144,6 +144,49 @@ to call at top-level: ant deploy-contrib Tests failed! + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +Tests failed! +
svn commit: r955733 - in /hadoop/pig/trunk: CHANGES.txt build.xml
Author: daijy Date: Thu Jun 17 20:43:41 2010 New Revision: 955733 URL: http://svn.apache.org/viewvc?rev=955733&view=rev Log: PIG-1457: Pig will run complete zebra test even we give -Dtestcase=xxx Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/build.xml Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=955733&r1=955732&r2=955733&view=diff == --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Jun 17 20:43:41 2010 @@ -86,6 +86,8 @@ PIG-1309: Map-side Cogroup (ashutoshc) BUG FIXES +PIG-1457: Pig will run complete zebra test even we give -Dtestcase=xxx (daijy) + PIG-1450: TestAlgebraicEvalLocal failures due to OOM (daijy) PIG-1433: pig should create success file if Modified: hadoop/pig/trunk/build.xml URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/build.xml?rev=955733&r1=955732&r2=955733&view=diff == --- hadoop/pig/trunk/build.xml (original) +++ hadoop/pig/trunk/build.xml Thu Jun 17 20:43:41 2010 @@ -612,9 +612,13 @@ - + + + + +
svn commit: r955753 - /hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java
Author: daijy
Date: Thu Jun 17 21:34:40 2010
New Revision: 955753
URL: http://svn.apache.org/viewvc?rev=955753&view=rev
Log:
Fix TestDataBag failure for PIG-1428
Modified:
hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java
Modified: hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java?rev=955753&r1=955752&r2=955753&view=diff
==
--- hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java (original)
+++ hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java Thu Jun 17
21:34:40 2010
@@ -375,7 +375,7 @@ public abstract class DefaultAbstractBag
protected void incSpillCount(Enum counter) {
PigStatusReporter reporter = PigStatusReporter.getInstance();
-if (reporter != null) {
+if (reporter != null && reporter.getCounter(counter)!=null) {
reporter.getCounter(counter).increment(1);
} else {
PigHadoopLogger.getInstance().warn(this, "Spill counter
incremented", counter);
svn commit: r955763 [1/4] - in /hadoop/pig/trunk: CHANGES.txt test/org/apache/pig/test/TestMultiQuery.java test/org/apache/pig/test/TestMultiQueryBasic.java test/org/apache/pig/test/TestMultiQueryComp
Author: rding Date: Thu Jun 17 22:05:21 2010 New Revision: 955763 URL: http://svn.apache.org/viewvc?rev=955763&view=rev Log: PIG-1456: TestMultiQuery takes a long time to run Added: hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryCompiler.java Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQuery.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=955763&r1=955762&r2=955763&view=diff == --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Jun 17 22:05:21 2010 @@ -86,6 +86,8 @@ PIG-1309: Map-side Cogroup (ashutoshc) BUG FIXES +PIG-1456: TestMultiQuery takes a long time to run (rding) + PIG-1457: Pig will run complete zebra test even we give -Dtestcase=xxx (daijy) PIG-1450: TestAlgebraicEvalLocal failures due to OOM (daijy)
svn commit: r955763 [3/4] - in /hadoop/pig/trunk: CHANGES.txt test/org/apache/pig/test/TestMultiQuery.java test/org/apache/pig/test/TestMultiQueryBasic.java test/org/apache/pig/test/TestMultiQueryComp
Added: hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java?rev=955763&view=auto
==
--- hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestMultiQueryBasic.java Thu Jun
17 22:05:21 2010
@@ -0,0 +1,764 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.File;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.PrintWriter;
+import java.io.StringReader;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+
+import junit.framework.Assert;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.io.WritableComparable;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.JobContext;
+import org.apache.hadoop.mapreduce.OutputCommitter;
+import org.apache.hadoop.mapreduce.OutputFormat;
+import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.pig.ExecType;
+import org.apache.pig.LoadFunc;
+import org.apache.pig.PigServer;
+import org.apache.pig.ResourceSchema;
+import org.apache.pig.StoreFunc;
+import org.apache.pig.backend.executionengine.ExecJob;
+import org.apache.pig.data.DataBag;
+import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.io.FileLocalizer;
+import org.apache.pig.tools.grunt.GruntParser;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.junit.runners.JUnit4;
+...@runwith(JUnit4.class)
+public class TestMultiQueryBasic {
+
+private static final MiniCluster cluster = MiniCluster.buildCluster();
+
+private PigServer myPig;
+
+@BeforeClass
+public static void setUpBeforeClass() throws IOException {
+Util.copyFromLocalToCluster(cluster,
+"test/org/apache/pig/test/data/passwd", "passwd");
+Util.copyFromLocalToCluster(cluster,
+"test/org/apache/pig/test/data/passwd2", "passwd2");
+}
+
+@AfterClass
+public static void tearDownAfterClass() throws IOException {
+Util.deleteFile(cluster, "passwd");
+Util.deleteFile(cluster, "passwd2");
+cluster.shutDown();
+}
+
+@Before
+public void setUp() throws Exception {
+cluster.setProperty("opt.multiquery", ""+true);
+myPig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
+deleteOutputFiles();
+}
+
+@After
+public void tearDown() throws Exception {
+myPig = null;
+}
+
+
+@Test
+public void testMultiQueryWithTwoStores2() {
+
+System.out.println("= multi-query with 2 stores (2) =");
+
+try {
+myPig.setBatchOn();
+
+myPig.registerQuery("a = load 'passwd' " +
+"using PigStorage(':') as (uname:chararray,
passwd:chararray, uid:int,gid:int);");
+myPig.registerQuery("b = filter a by uid > 5;");
+myPig.registerQuery("store b into '/tmp/output1';");
+myPig.registerQuery("c = group b by gid;");
+myPig.registerQuery("store c into '/tmp/output2';");
+
+List jobs = myPig.executeBatch();
+assertTrue(jobs.size() == 2);
+
+for (ExecJob job : jobs) {
+assertTrue(job.getStatus() == ExecJob.JOB_STATUS.COMPLETED);
+}
+
+} catch (Exception e) {
+e.printStackTrace();
+Assert.fail();
+}
+}
+
+@Test
+public void testMultiQueryWithTwoLoads2() {
+
+System.out.println("= multi-query with two loads (2) =");
+
+try {
+myPig.setBatchOn();
+
+myPig.registerQuery("a = load 'passwd'
