Author: rding
Date: Mon Jul 26 17:02:24 2010
New Revision: 979362
URL: http://svn.apache.org/viewvc?rev=979362&view=rev
Log:
PIG-1489: Pig MapReduceLauncher does not use jars in register statement
Added:
hadoop/pig/trunk/test/org/apache/pig/test/PigTestLoader.java
hadoop/pig/trunk/test/org/apache/pig/test/data/pigtestloader.jar (with
props)
Modified:
hadoop/pig/trunk/CHANGES.txt
hadoop/pig/trunk/build.xml
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java
Modified: hadoop/pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=979362&r1=979361&r2=979362&view=diff
==============================================================================
--- hadoop/pig/trunk/CHANGES.txt (original)
+++ hadoop/pig/trunk/CHANGES.txt Mon Jul 26 17:02:24 2010
@@ -106,6 +106,9 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1489: Pig MapReduceLauncher does not use jars in register statement
+(rding)
+
PIG-1435: make sure dependent jobs fail when a jon in multiquery fails (niraj
via rding)
Modified: hadoop/pig/trunk/build.xml
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/build.xml?rev=979362&r1=979361&r2=979362&view=diff
==============================================================================
--- hadoop/pig/trunk/build.xml (original)
+++ hadoop/pig/trunk/build.xml Mon Jul 26 17:02:24 2010
@@ -302,12 +302,16 @@
<param name="sources" value="${test.src.dir}" />
<param name="dist" value="${test.build.classes}" />
<param name="cp" value="test.classpath" />
+ <!-- don't compile PigTestLoader -->
+ <param name="excludes" value="**/PigTestLoader.java" />
</antcall>
<antcall target="compile-sources-all-warnings" inheritRefs="true"
inheritall="true">
<param name="sources" value="${test.src.dir}" />
<param name="dist" value="${test.build.classes}" />
<param name="cp" value="test.classpath" />
+ <!-- don't compile PigTestLoader -->
+ <param name="excludes" value="**/PigTestLoader.java" />
</antcall>
<copy file="${basedir}/test/hbase-site.xml"
tofile="${test.build.classes}/hbase-site.xml"/>
@@ -315,7 +319,7 @@
<!-- This target is for default compilation -->
<target name="compile-sources" unless="all.warnings">
- <javac encoding="${build.encoding}" srcdir="${sources}"
+ <javac encoding="${build.encoding}" srcdir="${sources}"
excludes="${excludes}"
includes="**/*.java" destdir="${dist}" debug="${javac.debug}"
optimize="${javac.optimize}" target="${javac.version}"
source="${javac.version}" deprecation="${javac.deprecation}">
@@ -328,7 +332,7 @@
<!-- this target is for compilation with all warnings enabled -->
<target name="compile-sources-all-warnings" if="all.warnings">
- <javac encoding="${build.encoding}" srcdir="${sources}"
+ <javac encoding="${build.encoding}" srcdir="${sources}"
excludes="${excludes}"
includes="**/*.java" destdir="${dist}" debug="${javac.debug}"
optimize="${javac.optimize}" target="${javac.version}"
source="${javac.version}" deprecation="${javac.deprecation}">
Modified:
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=979362&r1=979361&r2=979362&view=diff
==============================================================================
---
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++
hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Mon Jul 26 17:02:24 2010
@@ -168,6 +168,8 @@ public class MapReduceLauncher extends L
Thread jcThread = new Thread(jc);
jcThread.setUncaughtExceptionHandler(jctExceptionHandler);
+ jcThread.setContextClassLoader(pc.createCl(null));
+
//All the setup done, now lets launch the jobs.
jcThread.start();
Added: hadoop/pig/trunk/test/org/apache/pig/test/PigTestLoader.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/PigTestLoader.java?rev=979362&view=auto
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/PigTestLoader.java (added)
+++ hadoop/pig/trunk/test/org/apache/pig/test/PigTestLoader.java Mon Jul 26
17:02:24 2010
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.pig.test;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.mapreduce.Job;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
+import org.apache.pig.builtin.PigStorage;
+
+public class PigTestLoader extends PigStorage {
+
+ @Override
+ public void setLocation(String location, Job job) throws IOException {
+ super.setLocation(location, job);
+ FileInputFormat.setInputPathFilter(job, TestPathFilter.class);
+ }
+
+ public static class TestPathFilter implements PathFilter {
+ @Override
+ public boolean accept(Path p) {
+ String name = p.getName();
+ return !name.endsWith(".xml");
+ }
+ }
+
+}
Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=979362&r1=979361&r2=979362&view=diff
==============================================================================
--- hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ hadoop/pig/trunk/test/org/apache/pig/test/TestPigRunner.java Mon Jul 26
17:02:24 2010
@@ -24,14 +24,14 @@ import static org.junit.Assert.assertTru
import java.io.File;
import java.io.FileWriter;
import java.io.PrintWriter;
-import java.util.Properties;
-import java.util.List;
import java.util.Iterator;
-
+import java.util.List;
+import java.util.Properties;
import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.PigRunner.ReturnCode;
+import org.apache.pig.experimental.plan.Operator;
import org.apache.pig.impl.PigContext;
import org.apache.pig.impl.io.FileLocalizer;
import org.apache.pig.tools.pigstats.JobStats;
@@ -39,7 +39,6 @@ import org.apache.pig.tools.pigstats.Out
import org.apache.pig.tools.pigstats.PigProgressNotificationListener;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.PigStatsUtil;
-import org.apache.pig.experimental.plan.Operator;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -328,6 +327,24 @@ public class TestPigRunner {
assertEquals(PigStatsUtil.MULTI_INPUTS_RECORD_COUNTER + "batchtest",
name);
}
+ @Test
+ public void classLoaderTest() throws Exception {
+ PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
+ w.println("register test/org/apache/pig/test/data/pigtestloader.jar");
+ w.println("A = load '" + INPUT_FILE + "' using
org.apache.pig.test.PigTestLoader();");
+ w.println("store A into '" + OUTPUT_FILE + "';");
+ w.close();
+
+ try {
+ String[] args = { PIG_FILE };
+ PigStats stats = PigRunner.run(args, new
TestNotificationListener());
+ assertTrue(stats.isSuccessful());
+ } finally {
+ new File(PIG_FILE).delete();
+ Util.deleteFile(cluster, OUTPUT_FILE);
+ }
+ }
+
private static class TestNotificationListener implements
PigProgressNotificationListener {
private int numJobsToLaunch = 0;
@@ -385,4 +402,5 @@ public class TestPigRunner {
}
}
+
}
Added: hadoop/pig/trunk/test/org/apache/pig/test/data/pigtestloader.jar
URL:
http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/data/pigtestloader.jar?rev=979362&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/pig/trunk/test/org/apache/pig/test/data/pigtestloader.jar
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
