svn commit: r1553787 - in /pig/trunk: src/org/apache/pig/impl/util/avro/AvroMapWrapper.java test/org/apache/pig/builtin/TestAvroStorage.java
Author: rding
Date: Sat Dec 28 04:49:44 2013
New Revision: 1553787
URL: http://svn.apache.org/r1553787
Log:
PIG-3608: ClassCastException when looking up a value from AvroMapWrapper using
a Utf8 key (rding)
Modified:
pig/trunk/src/org/apache/pig/impl/util/avro/AvroMapWrapper.java
pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java
Modified: pig/trunk/src/org/apache/pig/impl/util/avro/AvroMapWrapper.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/avro/AvroMapWrapper.java?rev=1553787r1=1553786r2=1553787view=diff
==
--- pig/trunk/src/org/apache/pig/impl/util/avro/AvroMapWrapper.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/avro/AvroMapWrapper.java Sat Dec 28
04:49:44 2013
@@ -76,12 +76,12 @@ public final class AvroMapWrapper implem
@Override
public Object get(final Object key) {
Object v = null;
-if (isUtf8key) {
+if (isUtf8key !(key instanceof Utf8)) {
v = innerMap.get(new Utf8((String) key));
} else {
v = innerMap.get(key);
}
-
+
if (v instanceof Utf8) {
return v.toString();
} else {
Modified: pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java?rev=1553787r1=1553786r2=1553787view=diff
==
--- pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java (original)
+++ pig/trunk/test/org/apache/pig/builtin/TestAvroStorage.java Sat Dec 28
04:49:44 2013
@@ -24,6 +24,7 @@ import org.apache.avro.tool.DataFileWrit
import org.apache.avro.tool.Tool;
import org.apache.avro.tool.TrevniCreateRandomTool;
import org.apache.avro.tool.TrevniToJsonTool;
+import org.apache.avro.util.Utf8;
import org.apache.commons.io.FileUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -40,6 +41,7 @@ import org.apache.pig.backend.executione
import org.apache.pig.builtin.mock.Storage.Data;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.util.avro.AvroBagWrapper;
+import org.apache.pig.impl.util.avro.AvroMapWrapper;
import org.apache.pig.impl.util.avro.AvroTupleWrapper;
import org.apache.pig.test.Util;
import org.junit.AfterClass;
@@ -59,6 +61,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Set;
+import java.util.TreeMap;
import java.util.TreeSet;
import static junit.framework.Assert.assertEquals;
@@ -808,6 +811,19 @@ public class TestAvroStorage {
assertTrue(size1.compareTo(size5) 0);
}
+@Test
+public void testUtf8KeyLookupFromMap() throws Exception {
+MapCharSequence, Object tm = new TreeMapCharSequence, Object ();
+tm.put(new Utf8(foo), foo);
+tm.put(new Utf8(bar), bar);
+
+AvroMapWrapper wrapper = new AvroMapWrapper(tm);
+String v = (String)wrapper.get(new Utf8(foo));
+assertEquals(foo, v);
+v = (String)wrapper.get(new Utf8(bar));
+assertEquals(bar, v);
+}
+
private void testAvroStorage(boolean expectedToSucceed, String scriptFile,
MapString,String parameterMap) throws IOException {
pigServerLocal.setBatchOn();
svn commit: r1125512 - in /pig/trunk: CHANGES.txt src/org/apache/pig/parser/DryRunGruntParser.java test/org/apache/pig/test/TestMacroExpansion.java
Author: rding
Date: Fri May 20 19:05:02 2011
New Revision: 1125512
URL: http://svn.apache.org/viewvc?rev=1125512view=rev
Log:
PIG-2081: Dryrun gives wrong line numbers in error message for scripts
containing macro
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1125512r1=1125511r2=1125512view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 20 19:05:02 2011
@@ -227,6 +227,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2081: Dryrun gives wrong line numbers in error message for scripts
containing macro (rding)
+
PIG-2078: POProject.getNext(DataBag) does not handle null (daijy)
PIG-2029: Inconsistency in Pig Stats reports (rding)
Modified: pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java?rev=1125512r1=1125511r2=1125512view=diff
==
--- pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java Fri May 20
19:05:02 2011
@@ -228,10 +228,20 @@ public class DryRunGruntParser extends P
@Override
protected void processPig(String cmd) throws IOException {
+int start = getLineNumber();
+
+StringBuilder blder = new StringBuilder();
+for (int i = 1; i start; i++) {
+blder.append(\n);
+}
+
if (cmd.charAt(cmd.length() - 1) != ';') {
cmd += ;;
}
+blder.append(cmd);
+cmd = blder.toString();
+
CommonTokenStream tokenStream = QueryParserDriver.tokenize(cmd,
source);
Tree ast = null;
Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1125512r1=1125511r2=1125512view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Fri May 20
19:05:02 2011
@@ -31,7 +31,7 @@ import java.util.Properties;
import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.parser.DryRunGruntParser;
import org.apache.pig.tools.grunt.Grunt;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.ScriptState;
@@ -1190,6 +1190,29 @@ public class TestMacroExpansion {
validateFailure(sb.toString(), expectedErr, at);
}
+@Test // PIG-2081
+public void lineNumberTest3() throws Throwable {
+StringBuilder sb = new StringBuilder();
+
+sb.append(DEFINE my_macro (X,key) returns Y\n +
+{\n +
+tmp1 = foreach $X generate TOKENIZE((chararray)$key) as
tokens;\n +
+tmp2 = foreach tmp1 generate flatten(tokens);\n +
+tmp3 = order tmp2 by $0;\n +
+$Y = distinct tmp3;\n +
+};\n);
+
+sb.append(A3 = load 'sometext3' using TextLoader() as (row3);\n);
+
+sb.append(E3 = my_macro(A3,$0);\n);
+
+
+String expectedErr =
+file myscript.pig, line 9, column 17 mismatched input '$0'
expecting set null;
+
+validateDryrunFailure(sb.toString(), expectedErr, file);
+}
+
@Test
public void recursiveMacrosTest3() throws Exception {
String macro1 = define group_and_partition (A, group_key, reducers)
returns B, D {\n +
@@ -2156,6 +2179,39 @@ public class TestMacroExpansion {
}
}
+private void validateDryrunFailure(String piglatin, String expectedErr,
+String keyword) throws Throwable {
+String scriptFile = myscript.pig;
+
+try {
+BufferedReader br = new BufferedReader(new StringReader(piglatin));
+DryRunGruntParser parser = new DryRunGruntParser(br, scriptFile,
+new PigContext(ExecType.LOCAL, new Properties()));
+
+PrintWriter w = new PrintWriter(new FileWriter(scriptFile));
+w.print(piglatin);
+w.close();
+
+parser.parseStopOnError();
+
+Assert.fail(Expected exception isn't thrown);
+} catch (Exception e) {
+String msg = e.getMessage();
+int pos = msg.indexOf(keyword);
+if (pos 0) {
+Throwable cause = e.getCause();
+if (cause != null
svn commit: r1125519 - in /pig/branches/branch-0.9: CHANGES.txt src/org/apache/pig/parser/DryRunGruntParser.java test/org/apache/pig/test/TestMacroExpansion.java
Author: rding
Date: Fri May 20 19:23:18 2011
New Revision: 1125519
URL: http://svn.apache.org/viewvc?rev=1125519view=rev
Log:
PIG-2081: Dryrun gives wrong line numbers in error message for scripts
containing macro
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/parser/DryRunGruntParser.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1125519r1=1125518r2=1125519view=diff
==
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Fri May 20 19:23:18 2011
@@ -182,6 +182,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2081: Dryrun gives wrong line numbers in error message for scripts
containing macro (rding)
+
PIG-2078: POProject.getNext(DataBag) does not handle null (daijy)
PIG-2029: Inconsistency in Pig Stats reports (rding)
Modified:
pig/branches/branch-0.9/src/org/apache/pig/parser/DryRunGruntParser.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/DryRunGruntParser.java?rev=1125519r1=1125518r2=1125519view=diff
==
--- pig/branches/branch-0.9/src/org/apache/pig/parser/DryRunGruntParser.java
(original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/DryRunGruntParser.java
Fri May 20 19:23:18 2011
@@ -228,10 +228,20 @@ public class DryRunGruntParser extends P
@Override
protected void processPig(String cmd) throws IOException {
+int start = getLineNumber();
+
+StringBuilder blder = new StringBuilder();
+for (int i = 1; i start; i++) {
+blder.append(\n);
+}
+
if (cmd.charAt(cmd.length() - 1) != ';') {
cmd += ;;
}
+blder.append(cmd);
+cmd = blder.toString();
+
CommonTokenStream tokenStream = QueryParserDriver.tokenize(cmd,
source);
Tree ast = null;
Modified:
pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java?rev=1125519r1=1125518r2=1125519view=diff
==
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
(original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
Fri May 20 19:23:18 2011
@@ -31,7 +31,7 @@ import java.util.Properties;
import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.logicalLayer.FrontendException;
+import org.apache.pig.parser.DryRunGruntParser;
import org.apache.pig.tools.grunt.Grunt;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.ScriptState;
@@ -1190,6 +1190,29 @@ public class TestMacroExpansion {
validateFailure(sb.toString(), expectedErr, at);
}
+@Test // PIG-2081
+public void lineNumberTest3() throws Throwable {
+StringBuilder sb = new StringBuilder();
+
+sb.append(DEFINE my_macro (X,key) returns Y\n +
+{\n +
+tmp1 = foreach $X generate TOKENIZE((chararray)$key) as
tokens;\n +
+tmp2 = foreach tmp1 generate flatten(tokens);\n +
+tmp3 = order tmp2 by $0;\n +
+$Y = distinct tmp3;\n +
+};\n);
+
+sb.append(A3 = load 'sometext3' using TextLoader() as (row3);\n);
+
+sb.append(E3 = my_macro(A3,$0);\n);
+
+
+String expectedErr =
+file myscript.pig, line 9, column 17 mismatched input '$0'
expecting set null;
+
+validateDryrunFailure(sb.toString(), expectedErr, file);
+}
+
@Test
public void recursiveMacrosTest3() throws Exception {
String macro1 = define group_and_partition (A, group_key, reducers)
returns B, D {\n +
@@ -2156,6 +2179,39 @@ public class TestMacroExpansion {
}
}
+private void validateDryrunFailure(String piglatin, String expectedErr,
+String keyword) throws Throwable {
+String scriptFile = myscript.pig;
+
+try {
+BufferedReader br = new BufferedReader(new StringReader(piglatin));
+DryRunGruntParser parser = new DryRunGruntParser(br, scriptFile,
+new PigContext(ExecType.LOCAL, new Properties()));
+
+PrintWriter w = new PrintWriter(new FileWriter(scriptFile));
+w.print(piglatin);
+w.close();
+
+parser.parseStopOnError();
+
+Assert.fail(Expected exception isn't thrown);
+} catch
svn commit: r1125537 - in /pig/trunk: ./ src/org/apache/pig/impl/ src/org/apache/pig/impl/util/ src/org/apache/pig/scripting/jython/ test/org/apache/pig/test/
Author: rding
Date: Fri May 20 20:04:28 2011
New Revision: 1125537
URL: http://svn.apache.org/viewvc?rev=1125537view=rev
Log:
PIG-1824: Support import modules in Jython UDF
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/PigContext.java
pig/trunk/src/org/apache/pig/impl/util/JarManager.java
pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java
pig/trunk/test/org/apache/pig/test/TestPigContext.java
pig/trunk/test/org/apache/pig/test/TestScriptUDF.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1125537r1=1125536r2=1125537view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 20 20:04:28 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1824: Support import modules in Jython UDF (woody via rding)
+
PIG-1994: e2e test harness deployment implementation for existing cluster
(gates)
Modified: pig/trunk/src/org/apache/pig/impl/PigContext.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=1125537r1=1125536r2=1125537view=diff
==
--- pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/PigContext.java Fri May 20 20:04:28 2011
@@ -28,6 +28,7 @@ import java.net.URLClassLoader;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
+import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -91,7 +92,9 @@ public class PigContext implements Seria
private Properties properties;
// script files that are needed to run a job
+@Deprecated
public ListString scriptFiles = new ArrayListString();
+private MapString,File aliasedScriptFiles = new
LinkedHashMapString,File();
// script jars that are needed to run a script - jython.jar etc
public ListString scriptJars = new ArrayListString(2);
@@ -211,10 +214,28 @@ public class PigContext implements Seria
log.error(Failed to set tracker at: + newLocation);
}
}
-
+
+/**
+ * calls: addScriptFile(path, new File(path)), ensuring that a given path
is
+ * added to the jar at most once.
+ * @param path
+ * @throws MalformedURLException
+ */
public void addScriptFile(String path) throws MalformedURLException {
if (path != null) {
-scriptFiles.add(path);
+aliasedScriptFiles.put(path, new File(path));
+}
+}
+
+/**
+ * this method adds script files that must be added to the shipped jar
+ * named differently from their local fs path.
+ * @param name name in the jar
+ * @param path path on the local fs
+ */
+public void addScriptFile(String name, String path) {
+if (path != null) {
+aliasedScriptFiles.put(name, new File(path));
}
}
@@ -232,6 +253,14 @@ public class PigContext implements Seria
}
}
+/**
+ * script files as name/file pairs to be added to the job jar
+ * @return
+ */
+public MapString,File getScriptFiles() {
+return aliasedScriptFiles;
+}
+
public void rename(String oldName, String newName) throws IOException {
if (oldName.equals(newName)) {
return;
Modified: pig/trunk/src/org/apache/pig/impl/util/JarManager.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/JarManager.java?rev=1125537r1=1125536r2=1125537view=diff
==
--- pig/trunk/src/org/apache/pig/impl/util/JarManager.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/JarManager.java Fri May 20 20:04:28
2011
@@ -97,6 +97,7 @@ public class JarManager {
* @throws ClassNotFoundException
* @throws IOException
*/
+@SuppressWarnings(deprecation)
public static void createJar(OutputStream os, SetString funcs,
PigContext pigContext) throws ClassNotFoundException, IOException {
VectorJarListEntry jarList = new VectorJarListEntry();
for(String toSend: pigPackagesToSend) {
@@ -124,10 +125,12 @@ public class JarManager {
// log.error(Adding extra + pigContext.extraJars.get(i));
mergeJar(jarFile, extraJar, null, contents);
}
-for (int i = 0; i pigContext.scriptFiles.size(); i++) {
- String path = pigContext.scriptFiles.get(i);
+for (String path: pigContext.scriptFiles) {
addStream(jarFile, path, new FileInputStream(new
File(path)),contents);
}
+for (Map.EntryString, File entry :
pigContext.getScriptFiles().entrySet()) {
+ addStream(jarFile, entry.getKey(), new
FileInputStream(entry.getValue()),contents
svn commit: r1125183 - /pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java
Author: rding
Date: Thu May 19 23:57:26 2011
New Revision: 1125183
URL: http://svn.apache.org/viewvc?rev=1125183view=rev
Log:
Replace RuntimeException with ParserException
Modified:
pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java?rev=1125183r1=1125182r2=1125183view=diff
==
--- pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java Thu May 19
23:57:26 2011
@@ -111,7 +111,7 @@ class PigMacro {
Expected number of parameters: + params.size()
+ actual number of inputs:
+ ((inputs == null) ? 0 : inputs.length));
-throw new RuntimeException(msg);
+throw new ParserException(msg);
}
boolean isVoidReturn = false;
if (rets.isEmpty()) {
svn commit: r1125185 - /pig/trunk/src/org/apache/pig/parser/PigMacro.java
Author: rding
Date: Thu May 19 23:59:24 2011
New Revision: 1125185
URL: http://svn.apache.org/viewvc?rev=1125185view=rev
Log:
Replace RuntimeException with ParserException
Modified:
pig/trunk/src/org/apache/pig/parser/PigMacro.java
Modified: pig/trunk/src/org/apache/pig/parser/PigMacro.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/PigMacro.java?rev=1125185r1=1125184r2=1125185view=diff
==
--- pig/trunk/src/org/apache/pig/parser/PigMacro.java (original)
+++ pig/trunk/src/org/apache/pig/parser/PigMacro.java Thu May 19 23:59:24 2011
@@ -111,7 +111,7 @@ class PigMacro {
Expected number of parameters: + params.size()
+ actual number of inputs:
+ ((inputs == null) ? 0 : inputs.length));
-throw new RuntimeException(msg);
+throw new ParserException(msg);
}
boolean isVoidReturn = false;
if (rets.isEmpty()) {
svn commit: r1103796 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java test/org/apache/pig/test/TestMacroExpansion.java
Author: rding
Date: Mon May 16 17:06:18 2011
New Revision: 1103796
URL: http://svn.apache.org/viewvc?rev=1103796view=rev
Log:
PIG-2069: LoadFunc jar does not ship to backend in MultiQuery case
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1103796r1=1103795r2=1103796view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon May 16 17:06:18 2011
@@ -224,6 +224,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2069: LoadFunc jar does not ship to backend in MultiQuery case (rding)
+
PIG-2076: update documentation, help command with correct default value
of pig.cachedbag.memusage (thejas)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java?rev=1103796r1=1103795r2=1103796view=diff
==
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
Mon May 16 17:06:18 2011
@@ -329,6 +329,11 @@ class MultiQueryOptimizer extends MROpPl
}
}
}
+
+// PIG-2069: LoadFunc jar does not ship to backend in MultiQuery
case
+if (!mr.UDFs.isEmpty()) {
+succ.UDFs.addAll(mr.UDFs);
+}
}
// finally, remove the splitter from the MR plan
Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1103796r1=1103795r2=1103796view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Mon May 16
17:06:18 2011
@@ -1095,9 +1095,9 @@ public class TestMacroExpansion {
public void negativeTest9() throws Throwable {
String macro = define test( out1,out2 ){ A = load 'x' as (u:int,
v:int); $B = filter A by u 3 and v 20; };
-String expectedErr = Error during parsing. file myscript.pig, line
1, column 24 mismatched input '{' expecting RETURNS;
+String expectedErr = file myscript.pig, line 1, column 24
mismatched input '{' expecting RETURNS;
-validateFailure(macro, expectedErr, Error);
+validateFailure(macro, expectedErr, file);
}
@Test
svn commit: r1103799 - in /pig/branches/branch-0.9: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java test/org/apache/pig/test/TestMacroExpansion.ja
Author: rding
Date: Mon May 16 17:14:27 2011
New Revision: 1103799
URL: http://svn.apache.org/viewvc?rev=1103799view=rev
Log:
PIG-2069: LoadFunc jar does not ship to backend in MultiQuery case
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1103799r1=1103798r2=1103799view=diff
==
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Mon May 16 17:14:27 2011
@@ -182,6 +182,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2069: LoadFunc jar does not ship to backend in MultiQuery case (rding)
+
PIG-2076: update documentation, help command with correct default value
of pig.cachedbag.memusage (thejas)
Modified:
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java?rev=1103799r1=1103798r2=1103799view=diff
==
---
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
(original)
+++
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MultiQueryOptimizer.java
Mon May 16 17:14:27 2011
@@ -329,6 +329,11 @@ class MultiQueryOptimizer extends MROpPl
}
}
}
+
+// PIG-2069: LoadFunc jar does not ship to backend in MultiQuery
case
+if (!mr.UDFs.isEmpty()) {
+succ.UDFs.addAll(mr.UDFs);
+}
}
// finally, remove the splitter from the MR plan
Modified:
pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java?rev=1103799r1=1103798r2=1103799view=diff
==
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
(original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
Mon May 16 17:14:27 2011
@@ -1095,9 +1095,9 @@ public class TestMacroExpansion {
public void negativeTest9() throws Throwable {
String macro = define test( out1,out2 ){ A = load 'x' as (u:int,
v:int); $B = filter A by u 3 and v 20; };
-String expectedErr = Error during parsing. file myscript.pig, line
1, column 24 mismatched input '{' expecting RETURNS;
+String expectedErr = file myscript.pig, line 1, column 24
mismatched input '{' expecting RETURNS;
-validateFailure(macro, expectedErr, Error);
+validateFailure(macro, expectedErr, file);
}
@Test
svn commit: r1102821 - in /pig/branches/branch-0.9: CHANGES.txt src/org/apache/pig/scripting/Pig.java test/org/apache/pig/test/TestScriptLanguage.java
Author: rding
Date: Fri May 13 17:22:42 2011
New Revision: 1102821
URL: http://svn.apache.org/viewvc?rev=1102821view=rev
Log:
PIG-1827: When passing a parameter to Pig, if the value contains $ it has to be
escaped for no apparent reason
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1102821r1=1102820r2=1102821view=diff
==
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Fri May 13 17:22:42 2011
@@ -178,6 +178,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1827: When passing a parameter to Pig, if the value contains $ it has to
be escaped for no apparent reason (rding)
+
PIG-1938: support project-range as udf argument (thejas)
PIG-2056: Jython error messages should show script name (rding)
Modified: pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java?rev=1102821r1=1102820r2=1102821view=diff
==
--- pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/scripting/Pig.java Fri May 13
17:22:42 2011
@@ -24,6 +24,7 @@ import java.io.LineNumberReader;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -276,7 +277,8 @@ public class Pig {
throws IOException {
ArrayListString plist = new ArrayListString();
for (EntryString, Object entry : vars.entrySet()) {
-plist.add(entry.getKey() + = + entry.getValue().toString());
+plist.add(entry.getKey() + =
++ fixNonEscapedDollarSign(entry.getValue().toString()));
}
ParameterSubstitutionPreprocessor psp =
@@ -294,6 +296,31 @@ public class Pig {
return writer.toString();
}
+// Escape the $ so that we can use the parameter substitution
+// to perform bind operation. Parameter substitution will un-escape $
+private static String fixNonEscapedDollarSign(String s) {
+String[] tkns = s.split(\\$, -1);
+
+if (tkns.length == 1) return s;
+
+StringBuilder sb = new StringBuilder();
+
+for (int i = 0; i tkns.length -1; i++) {
+if (tkns[i].isEmpty()) {
+sb.append();
+} else {
+sb.append(tkns[i]);
+if (tkns[i].charAt(tkns[i].length()-1) != '\\') {
+sb.append();
+}
+}
+sb.append($);
+}
+sb.append(tkns[tkns.length - 1]);
+
+return sb.toString();
+}
+
//-
private static String getScriptFromFile(String filename) throws
IOException {
Modified:
pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java?rev=1102821r1=1102820r2=1102821view=diff
==
--- pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java
(original)
+++ pig/branches/branch-0.9/test/org/apache/pig/test/TestScriptLanguage.java
Fri May 13 17:22:42 2011
@@ -21,6 +21,7 @@ import static org.junit.Assert.assertFal
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
+import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@@ -29,7 +30,11 @@ import org.apache.pig.ExecType;
import org.apache.pig.PigRunner;
import org.apache.pig.PigServer;
import org.apache.pig.data.Tuple;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.scripting.BoundScript;
+import org.apache.pig.scripting.Pig;
import org.apache.pig.scripting.ScriptEngine;
+import org.apache.pig.scripting.ScriptPigContext;
import org.apache.pig.tools.pigstats.JobStats;
import org.apache.pig.tools.pigstats.OutputStats;
import org.apache.pig.tools.pigstats.PigStats;
@@ -357,6 +362,11 @@ public class TestScriptLanguage {
Pig.fs(\rmr simple_out\),
input = 'simple_table_5',
output = 'simple_out',
+testvar = 'abcd$py',
+testvar2 = '$',
+testvar3 = '$',
+testvar4 = 'abcd\\$py$',
+testvar5 = 'abcd\\$py
svn commit: r1101259 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/parser/ test/org/apache/pig/test/
Author: rding
Date: Mon May 9 22:49:40 2011
New Revision: 1101259
URL: http://svn.apache.org/viewvc?rev=1101259view=rev
Log:
PIG-2012: Comments at the begining of the file throws off line numbers in errors
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/parser/PigMacro.java
pig/trunk/src/org/apache/pig/parser/PigParserNode.java
pig/trunk/src/org/apache/pig/parser/PigParserNodeAdaptor.java
pig/trunk/src/org/apache/pig/parser/QueryParser.g
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
pig/trunk/src/org/apache/pig/parser/SourceLocation.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1101259r1=1101258r2=1101259view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon May 9 22:49:40 2011
@@ -205,6 +205,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2012: Comments at the begining of the file throws off line numbers in
errors (rding)
+
PIG-2043: Ship antlr-runtime.jar to backend (daijy)
PIG-2049: Pig should display TokenMgrError message consistently across all
parsers (rding)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1101259r1=1101258r2=1101259view=diff
==
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Mon May 9 22:49:40 2011
@@ -1575,8 +1575,13 @@ public class PigServer {
scriptCache.add( );
currentLineNum++;
}
-scriptCache.add( query );
-currentLineNum = startLine;
+BufferedReader br = new BufferedReader(new
StringReader(query));
+String line = br.readLine();
+while (line != null) {
+scriptCache.add(line);
+currentLineNum++;
+line = br.readLine();
+}
}
} else {
scriptCache.add( query );
Modified: pig/trunk/src/org/apache/pig/parser/PigMacro.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/PigMacro.java?rev=1101259r1=1101258r2=1101259view=diff
==
--- pig/trunk/src/org/apache/pig/parser/PigMacro.java (original)
+++ pig/trunk/src/org/apache/pig/parser/PigMacro.java Mon May 9 22:49:40 2011
@@ -36,6 +36,7 @@ import org.antlr.runtime.tree.CommonTree
import org.antlr.runtime.tree.Tree;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.pig.parser.PigParserNode.InvocationPoint;
import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor;
class PigMacro {
@@ -50,6 +51,9 @@ class PigMacro {
private MapString, PigMacro seen;
private SetString macroStack;
private long idx = 0;
+
+// The start line number of this macro in the script
+private int startLine = 0;
PigMacro(String name, String file, ListString params,
ListString returns, String body, MapString, PigMacro seen) {
@@ -71,9 +75,17 @@ class PigMacro {
SetString getStack() { return macroStack; }
-private CommonTree inline(String[] inputs, String[] outputs, int
lineNumber,
+void setStartLine(int start) {
+this.startLine = start;
+}
+
+int getStartLine() {
+return startLine;
+}
+
+private CommonTree inline(String[] inputs, String[] outputs, CommonTree t,
String file) throws ParserException {
-String in = substituteParams(inputs, outputs, lineNumber, file);
+String in = substituteParams(inputs, outputs, t.getLine(), file);
SetString masks = new HashSetString();
if (inputs != null) {
@@ -86,7 +98,7 @@ class PigMacro {
masks.add(s);
}
-return maskAlias(in, masks, lineNumber, file);
+return maskAlias(in, masks, t, file);
}
@@ -155,8 +167,13 @@ class PigMacro {
return writer.toString();
}
-private CommonTree maskAlias(String in, SetString masks, int line,
+private CommonTree maskAlias(String in, SetString masks, CommonTree tree,
String file) throws ParserException {
+
+// this is the MACRO_INLINE node. the real line number is in the
+// macro name node
+int line = tree.getChild(0).getLine();
+
CharStream input = null;
try {
// parse macro
svn commit: r1101260 - in /pig/branches/branch-0.9: ./ src/org/apache/pig/ src/org/apache/pig/parser/ test/org/apache/pig/test/
Author: rding
Date: Mon May 9 22:51:34 2011
New Revision: 1101260
URL: http://svn.apache.org/viewvc?rev=1101260view=rev
Log:
PIG-2012: Comments at the begining of the file throws off line numbers in errors
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java
pig/branches/branch-0.9/src/org/apache/pig/parser/PigParserNode.java
pig/branches/branch-0.9/src/org/apache/pig/parser/PigParserNodeAdaptor.java
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParser.g
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserDriver.java
pig/branches/branch-0.9/src/org/apache/pig/parser/QueryParserUtils.java
pig/branches/branch-0.9/src/org/apache/pig/parser/SourceLocation.java
pig/branches/branch-0.9/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1101260r1=1101259r2=1101260view=diff
==
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Mon May 9 22:51:34 2011
@@ -174,6 +174,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2012: Comments at the begining of the file throws off line numbers in
errors (rding)
+
PIG-2043: Ship antlr-runtime.jar to backend (daijy)
PIG-2049: Pig should display TokenMgrError message consistently across all
parsers (rding)
Modified: pig/branches/branch-0.9/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/PigServer.java?rev=1101260r1=1101259r2=1101260view=diff
==
--- pig/branches/branch-0.9/src/org/apache/pig/PigServer.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/PigServer.java Mon May 9
22:51:34 2011
@@ -1575,8 +1575,13 @@ public class PigServer {
scriptCache.add( );
currentLineNum++;
}
-scriptCache.add( query );
-currentLineNum = startLine;
+BufferedReader br = new BufferedReader(new
StringReader(query));
+String line = br.readLine();
+while (line != null) {
+scriptCache.add(line);
+currentLineNum++;
+line = br.readLine();
+}
}
} else {
scriptCache.add( query );
Modified: pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java?rev=1101260r1=1101259r2=1101260view=diff
==
--- pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java (original)
+++ pig/branches/branch-0.9/src/org/apache/pig/parser/PigMacro.java Mon May 9
22:51:34 2011
@@ -36,6 +36,7 @@ import org.antlr.runtime.tree.CommonTree
import org.antlr.runtime.tree.Tree;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.pig.parser.PigParserNode.InvocationPoint;
import org.apache.pig.tools.parameters.ParameterSubstitutionPreprocessor;
class PigMacro {
@@ -50,6 +51,9 @@ class PigMacro {
private MapString, PigMacro seen;
private SetString macroStack;
private long idx = 0;
+
+// The start line number of this macro in the script
+private int startLine = 0;
PigMacro(String name, String file, ListString params,
ListString returns, String body, MapString, PigMacro seen) {
@@ -71,9 +75,17 @@ class PigMacro {
SetString getStack() { return macroStack; }
-private CommonTree inline(String[] inputs, String[] outputs, int
lineNumber,
+void setStartLine(int start) {
+this.startLine = start;
+}
+
+int getStartLine() {
+return startLine;
+}
+
+private CommonTree inline(String[] inputs, String[] outputs, CommonTree t,
String file) throws ParserException {
-String in = substituteParams(inputs, outputs, lineNumber, file);
+String in = substituteParams(inputs, outputs, t.getLine(), file);
SetString masks = new HashSetString();
if (inputs != null) {
@@ -86,7 +98,7 @@ class PigMacro {
masks.add(s);
}
-return maskAlias(in, masks, lineNumber, file);
+return maskAlias(in, masks, t, file);
}
@@ -155,8 +167,13 @@ class PigMacro {
return writer.toString();
}
-private CommonTree maskAlias(String in, SetString masks, int line,
+private CommonTree maskAlias(String in, SetString masks
svn commit: r1100411 - in /pig/trunk: CHANGES.txt src/org/apache/pig/impl/util/LogUtils.java
Author: rding
Date: Fri May 6 22:52:21 2011
New Revision: 1100411
URL: http://svn.apache.org/viewvc?rev=1100411view=rev
Log:
PIG-2049: Pig should display TokenMgrError message consistently across all
parsers
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/util/LogUtils.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1100411r1=1100410r2=1100411view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri May 6 22:52:21 2011
@@ -202,6 +202,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-2049: Pig should display TokenMgrError message consistently across all
parsers (rding)
+
PIG-2041: Minicluster should make each run independent (daijy)
PIG-2040: Move classloader from QueryParserDriver to PigContext (daijy)
Modified: pig/trunk/src/org/apache/pig/impl/util/LogUtils.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/LogUtils.java?rev=1100411r1=1100410r2=1100411view=diff
==
--- pig/trunk/src/org/apache/pig/impl/util/LogUtils.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/LogUtils.java Fri May 6 22:52:21
2011
@@ -106,7 +106,8 @@ public class LogUtils {
} else {
if((t instanceof ParseException
|| t instanceof
org.apache.pig.tools.pigscript.parser.TokenMgrError
-|| t instanceof
org.apache.pig.impl.logicalLayer.parser.TokenMgrError)) {
+|| t instanceof
org.apache.pig.impl.logicalLayer.parser.TokenMgrError
+|| t instanceof
org.apache.pig.tools.parameters.TokenMgrError)) {
message = ERROR 1000: Error during parsing. +
t.getMessage();
} else if (t instanceof IOException) {
message = ERROR 2997: Encountered IOException. +
t.getMessage();
svn commit: r1100418 - in /pig/branches/branch-0.9: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Author: rding
Date: Sat May 7 00:12:29 2011
New Revision: 1100418
URL: http://svn.apache.org/viewvc?rev=1100418view=rev
Log:
PIG-2033: Pig returns sucess for the failed Pig script
Modified:
pig/branches/branch-0.9/CHANGES.txt
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Modified: pig/branches/branch-0.9/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/CHANGES.txt?rev=1100418r1=1100417r2=1100418view=diff
==
--- pig/branches/branch-0.9/CHANGES.txt (original)
+++ pig/branches/branch-0.9/CHANGES.txt Sat May 7 00:12:29 2011
@@ -535,6 +535,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-2033: Pig returns sucess for the failed Pig script (rding)
+
PIG-1993: PigStorageSchema throw NPE with ColumnPruning (daijy)
PIG-1935: New logical plan: Should not push up filter in front of Bincond
(daijy)
Modified:
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=1100418r1=1100417r2=1100418view=diff
==
---
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++
pig/branches/branch-0.9/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Sat May 7 00:12:29 2011
@@ -354,6 +354,10 @@ public class MapReduceLauncher extends L
// stats collection is done, log the results
PigStatsUtil.stopCollection(true);
+// PigStatsUtil.stopCollection also computes the return code based on
+// total jobs to run, jobs successful and jobs failed
+failed = failed || !PigStats.get().isSuccessful();
+
MapEnum, Long warningAggMap = new HashMapEnum, Long();
if (succJobs != null) {
svn commit: r1100419 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Author: rding
Date: Sat May 7 00:12:55 2011
New Revision: 1100419
URL: http://svn.apache.org/viewvc?rev=1100419view=rev
Log:
PIG-2033: Pig returns sucess for the failed Pig script
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1100419r1=1100418r2=1100419view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Sat May 7 00:12:55 2011
@@ -557,6 +557,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-2033: Pig returns sucess for the failed Pig script (rding)
+
PIG-1993: PigStorageSchema throw NPE with ColumnPruning (daijy)
PIG-1935: New logical plan: Should not push up filter in front of Bincond
(daijy)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java?rev=1100419r1=1100418r2=1100419view=diff
==
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
Sat May 7 00:12:55 2011
@@ -354,6 +354,10 @@ public class MapReduceLauncher extends L
// stats collection is done, log the results
PigStatsUtil.stopCollection(true);
+// PigStatsUtil.stopCollection also computes the return code based on
+// total jobs to run, jobs successful and jobs failed
+failed = failed || !PigStats.get().isSuccessful();
+
MapEnum, Long warningAggMap = new HashMapEnum, Long();
if (succJobs != null) {
svn commit: r1095228 - in /pig/trunk: CHANGES.txt src/org/apache/pig/Main.java src/org/apache/pig/parser/DryRunGruntParser.java src/org/apache/pig/parser/QueryParserDriver.java test/org/apache/pig/tes
Author: rding
Date: Tue Apr 19 22:22:42 2011
New Revision: 1095228
URL: http://svn.apache.org/viewvc?rev=1095228view=rev
Log:
PIG-1987: -dryrun does not work with set
Added:
pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1095228r1=1095227r2=1095228view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Apr 19 22:22:42 2011
@@ -158,6 +158,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1987: -dryrun does not work with set (rding)
+
PIG-1871: Dont throw exception if partition filters cannot be pushed up.
(rding)
PIG-1870: HBaseStorage doesn't project correctly (dvryaboy)
Modified: pig/trunk/src/org/apache/pig/Main.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1095228r1=1095227r2=1095228view=diff
==
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Tue Apr 19 22:22:42 2011
@@ -35,7 +35,6 @@ import java.util.AbstractList;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
@@ -48,6 +47,7 @@ import jline.ConsoleReader;
import jline.ConsoleReaderInputStream;
import jline.History;
+import org.antlr.runtime.RecognitionException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -66,6 +66,7 @@ import org.apache.pig.impl.util.LogUtils
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.PropertiesUtil;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.parser.DryRunGruntParser;
import org.apache.pig.parser.QueryParserDriver;
import org.apache.pig.scripting.ScriptEngine;
import org.apache.pig.scripting.ScriptEngine.SupportedScriptLang;
@@ -389,9 +390,7 @@ static int run(String args[], PigProgres
pin = runParamPreprocessor(properties, in, params, paramFiles,
substFile, debug || dryrun || checkScriptOnly);
if (dryrun) {
-QueryParserDriver driver = new QueryParserDriver(
-pigContext, 0, new HashMapString, String());
-if (driver.dryrun(substFile)) {
+if (dryrun(substFile, pigContext)) {
log.info(Dry run completed. Substituted pig script is at
+ substFile
+ . Expanded pig script is at
@@ -518,9 +517,7 @@ static int run(String args[], PigProgres
substFile = remainders[0] + .substituted;
pin = runParamPreprocessor(properties, in, params, paramFiles,
substFile, debug || dryrun || checkScriptOnly);
if (dryrun) {
-QueryParserDriver driver = new QueryParserDriver(
-pigContext, 0, new HashMapString, String());
-if (driver.dryrun(substFile)) {
+if (dryrun(substFile, pigContext)) {
log.info(Dry run completed. Substituted pig script is at
+ substFile
+ . Expanded pig script is at
@@ -605,6 +602,25 @@ private static int getReturnCodeForStats
: ReturnCode.PARTIAL_FAILURE; // some jobs have
failed
}
+public static boolean dryrun(String scriptFile, PigContext pigContext)
+throws RecognitionException, IOException {
+BufferedReader rd = new BufferedReader(new FileReader(scriptFile));
+
+DryRunGruntParser dryrun = new DryRunGruntParser(rd, scriptFile,
+pigContext);
+
+boolean hasMacro = dryrun.parseStopOnError();
+
+if (hasMacro) {
+String expandedFile = scriptFile.replace(.substituted,
+.expanded);
+BufferedWriter fw = new BufferedWriter(new FileWriter(expandedFile));
+fw.append(dryrun.getResult());
+fw.close();
+}
+return hasMacro;
+}
+
//TODO jz: log4j.properties should be used instead
private static void configureLog4J(Properties properties, PigContext
pigContext) {
// TODO Add a file appender for the logs
Added: pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java?rev=1095228view=auto
==
--- pig/trunk/src/org/apache/pig/parser/DryRunGruntParser.java (added)
+++ pig/trunk/src/org
svn commit: r1091123 - in /pig/branches/branch-0.8: CHANGES.txt src/org/apache/pig/impl/io/TFileRecordReader.java test/org/apache/pig/test/TestTmpFileCompression.java
Author: rding
Date: Mon Apr 11 16:52:11 2011
New Revision: 1091123
URL: http://svn.apache.org/viewvc?rev=1091123view=rev
Log:
PIG-1977: 'Stream closed' error while reading Pig temp files
Modified:
pig/branches/branch-0.8/CHANGES.txt
pig/branches/branch-0.8/src/org/apache/pig/impl/io/TFileRecordReader.java
pig/branches/branch-0.8/test/org/apache/pig/test/TestTmpFileCompression.java
Modified: pig/branches/branch-0.8/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.8/CHANGES.txt?rev=1091123r1=1091122r2=1091123view=diff
==
--- pig/branches/branch-0.8/CHANGES.txt (original)
+++ pig/branches/branch-0.8/CHANGES.txt Mon Apr 11 16:52:11 2011
@@ -34,6 +34,8 @@ PIG-1886: Add zookeeper jar to list of j
BUG FIXES
+PIG-1977: Stream closed error while reading Pig temp files (results of
intermediate jobs) (rding)
+
PIG-1911: Infinite loop with accumulator function in nested foreach (thejas)
PIG-1964: PigStorageSchema fails if a column value is null (thejas)
Modified:
pig/branches/branch-0.8/src/org/apache/pig/impl/io/TFileRecordReader.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.8/src/org/apache/pig/impl/io/TFileRecordReader.java?rev=1091123r1=1091122r2=1091123view=diff
==
--- pig/branches/branch-0.8/src/org/apache/pig/impl/io/TFileRecordReader.java
(original)
+++ pig/branches/branch-0.8/src/org/apache/pig/impl/io/TFileRecordReader.java
Mon Apr 11 16:52:11 2011
@@ -78,12 +78,13 @@ public class TFileRecordReader extends R
// if we got here, we have seen
RECORD_1-RECORD_2-RECORD_3-TUPLE_MARKER
// sequence - lets now read the contents of the tuple
value = (Tuple) sedes.readDatum(in);
-scanner.advance();
-return true;
}
finally {
in.close();
}
+
+scanner.advance();
+return true;
}
@Override
Modified:
pig/branches/branch-0.8/test/org/apache/pig/test/TestTmpFileCompression.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.8/test/org/apache/pig/test/TestTmpFileCompression.java?rev=1091123r1=1091122r2=1091123view=diff
==
---
pig/branches/branch-0.8/test/org/apache/pig/test/TestTmpFileCompression.java
(original)
+++
pig/branches/branch-0.8/test/org/apache/pig/test/TestTmpFileCompression.java
Mon Apr 11 16:52:11 2011
@@ -20,19 +20,26 @@ package org.apache.pig.test;
import static org.junit.Assert.*;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintWriter;
import java.util.HashMap;
import java.util.Iterator;
+import java.util.Properties;
import java.io.BufferedReader;
import java.io.FileReader;
import org.apache.pig.ExecType;
+import org.apache.pig.PigRunner;
import org.apache.pig.PigServer;
+import org.apache.pig.data.DataBag;
import org.apache.pig.data.Tuple;
import org.apache.pig.impl.io.TFileStorage;
import org.apache.pig.impl.io.InterStorage;
+import org.apache.pig.tools.pigstats.OutputStats;
+import org.apache.pig.tools.pigstats.PigStats;
import org.junit.After;
import org.junit.AfterClass;
import org.junit.Before;
@@ -267,4 +274,57 @@ public class TestTmpFileCompression {
}));
}
+
+// PIG-1977
+@Test
+public void testTFileRecordReader() throws Exception {
+PrintWriter w = new PrintWriter(new FileWriter(1.txt));
+for (int i = 0; i 30; i++) {
+w.println(1\tthis is a test for compression of temp files);
+}
+w.close();
+
+Util.copyFromLocalToCluster(cluster, 1.txt, 1.txt);
+
+PrintWriter w1 = new PrintWriter(new FileWriter(tfile.pig));
+w1.println(A = load '1.txt' as (a0:int, a1:chararray););
+w1.println(B = group A by a0;);
+w1.println(store B into 'tfile' using
org.apache.pig.impl.io.TFileStorage(););
+w1.close();
+
+PrintWriter w2 = new PrintWriter(new FileWriter(tfile2.pig));
+w2.println(A = load 'tfile' using
org.apache.pig.impl.io.TFileStorage() as (a:int, b:bag{(b0:int,
b1:chararray)}););
+w2.println(B = foreach A generate flatten($1););
+w2.println(store B into '2.txt';);
+w2.close();
+
+try {
+String[] args = { -Dpig.tmpfilecompression.codec=gz,
+-Dtfile.io.chunk.size=100, tfile.pig };
+PigStats stats = PigRunner.run(args, null);
+
+assertTrue(stats.isSuccessful());
+
+String[] args2 = { -Dpig.tmpfilecompression.codec=gz,
+-Dtfile.io.chunk.size=100, tfile2.pig };
+PigStats stats2 = PigRunner.run(args2, null);
+
+assertTrue(stats2.isSuccessful());
+
+OutputStats
svn commit: r1091242 - in /pig/trunk: CHANGES.txt src/org/apache/pig/parser/QueryParserUtils.java test/org/apache/pig/test/TestMacroExpansion.java
Author: rding
Date: Mon Apr 11 22:45:50 2011
New Revision: 1091242
URL: http://svn.apache.org/viewvc?rev=1091242view=rev
Log:
PIG-1988: Importing an empty macro file causing NPE
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1091242r1=1091241r2=1091242view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Apr 11 22:45:50 2011
@@ -142,6 +142,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1988: Importing an empty macro file causing NPE (rding)
+
PIG-1977: Stream closed error while reading Pig temp files (results of
intermediate jobs) (rding)
PIG-1963: in nested foreach, accumutive udf taking input from order-by does
not get results in order (thejas)
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java?rev=1091242r1=1091241r2=1091242view=diff
==
--- pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserUtils.java Mon Apr 11
22:45:50 2011
@@ -191,7 +191,10 @@ public class QueryParserUtils {
for (int i = 0; i count; i++) {
if (i == idx) {
-parent.addChildren(macroList);
+// add only there is something to add
+if (macroList != null) {
+parent.addChildren(macroList);
+}
} else {
parent.addChild((Tree) childList.get(i));
}
Modified: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1091242r1=1091241r2=1091242view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Mon Apr 11
22:45:50 2011
@@ -1663,6 +1663,31 @@ public class TestMacroExpansion {
testMacro( query, expected );
}
+// PIG-1988
+@Test
+public void test36() throws Exception {
+File f = new File(mymacro.pig);
+f.deleteOnExit();
+
+FileWriter fw = new FileWriter(f);
+fw.append( );
+fw.close();
+
+String query = import 'mymacro.pig'; +
+define macro1() returns dummy {}; +
+A = load '1.txt' as (a0:int, a1:chararray); +
+dummy = macro1(); +
+B = group A by a0; +
+store B into 'output';;
+
+String expected =
+A = load '1.txt' as (a0:int, a1:chararray);\n +
+B = group A by (a0);\n +
+store B INTO 'output';\n;
+
+verify(query, expected);
+}
+
@Test
public void testCommentInMacro() throws Exception {
String query = a = load 'testComplexCast' as (m);\n +
@@ -1734,7 +1759,11 @@ public class TestMacroExpansion {
String[] args = { -Dpig.import.search.path=/tmp, -x, local,
-c, myscript.pig };
PigStats stats = PigRunner.run(args, null);
-
+
+if (!stats.isSuccessful()) {
+System.out.println(error msg: + stats.getErrorMessage());
+}
+
assertTrue(stats.isSuccessful());
String[] args2 = { -Dpig.import.search.path=/tmp, -x, local,
-r, myscript.pig };
svn commit: r1089537 - in /pig/trunk: ./ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ src/org/a
Author: rding
Date: Wed Apr 6 17:10:57 2011
New Revision: 1089537
URL: http://svn.apache.org/viewvc?rev=1089537view=rev
Log:
PIG-1955: PhysicalOperator has a member variable (non-static) Log object that
is non-transient, this causes serialization errors
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POCast.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserComparisonFunc.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCollectedGroup.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POCombinerPackage.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PODemux.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/PODistinct.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFRJoin.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POFilter.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POForEach.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLimit.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLoad.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POLocalRearrange.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMergeJoin.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POMultiQueryPackage.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POPreCombinerLocalRearrange.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSkewedJoin.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSort.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/relationalOperators/POSplit.java
pig/trunk/test/org/apache/pig/test/data/GoldenFiles/MRC18.gld
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1089537r1=1089536r2=1089537view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Apr 6 17:10:57 2011
@@ -136,6 +136,9 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1955: PhysicalOperator has a member variable (non-static) Log object that
+is non-transient, this causes serialization errors (woody via rding)
+
PIG-1964: PigStorageSchema fails if a column value is null (thejas))
PIG-1866: Dereference a bag within a tuple does not work (daijy)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java?rev=1089537r1=1089536r2=1089537view=diff
==
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/PhysicalOperator.java
Wed Apr 6 17:10:57 2011
@@ -61,8 +61,7 @@ import org.apache.pig.pen.Illustrable;
*
*/
public abstract class PhysicalOperator extends OperatorPhyPlanVisitor
implements Illustrable, Cloneable {
-
-private final Log log = LogFactory.getLog(getClass());
+private static final Log log = LogFactory.getLog(PhysicalOperator.class);
protected static final long serialVersionUID = 1L;
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java?rev=1089537r1=1089536r2=1089537view=diff
==
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/ExpressionOperator.java
(original)
+++
pig/trunk/src/org
svn commit: r1085612 [4/4] - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/parser/ src/org/apache/pig/scripting/ src/org/apache/pig/tools/parameters/ src/org/apache/pig/tools/pigscript/pars
Modified: pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java?rev=1085612r1=1085611r2=1085612view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestParamSubPreproc.java Sat Mar 26
00:07:41 2011
@@ -1482,4 +1482,50 @@ public class TestParamSubPreproc extends
log.info(Done);
}
+@Test
+public void testMacroDef() throws Exception{
+log.info(Starting test testMacroDef() ...);
+try {
+ParameterSubstitutionPreprocessor ps = new
ParameterSubstitutionPreprocessor(50);
+pigIStream = new BufferedReader(new FileReader(basedir +
/input6.pig));
+pigOStream = new FileWriter(basedir + /output1.pig);
+
+String[] arg = {date=20080228};
+String[] argFiles = null;
+ps.genSubstitutedFile(pigIStream , pigOStream , arg , argFiles);
+
+FileInputStream pigResultStream = new FileInputStream(basedir +
/output1.pig);
+pigExResultStream = new FileInputStream(basedir +
/ExpectedResult6.pig);
+BufferedReader inExpected = new BufferedReader(new
InputStreamReader(pigExResultStream));
+BufferedReader inResult = new BufferedReader(new
InputStreamReader(pigResultStream));
+
+String exLine;
+String resLine;
+int lineNum=0;
+
+while (true) {
+lineNum++;
+exLine = inExpected.readLine();
+resLine = inResult.readLine();
+if (exLine==null || resLine==null)
+break;
+assertEquals(Command line parameter substitution failed. +
Expected : +exLine+ , but got : +resLine+ in line num : +lineNum
,exLine.trim(), resLine.trim());
+}
+if (!(exLine==null resLine==null)) {
+fail (Command line parameter substitution failed. +
Expected : +exLine+ , but got : +resLine+ in line num : +lineNum);
+}
+
+inExpected.close();
+inResult.close();
+} catch (ParseException e) {
+fail (Got ParseException : + e.getMessage());
+} catch (RuntimeException e) {
+fail (Got RuntimeException : + e.getMessage());
+} catch (Error e) {
+fail (Got error : + e.getMessage());
+}
+
+log.info(Done);
+
+}
}
Added: pig/trunk/test/org/apache/pig/test/data/ExpectedResult6.pig
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/ExpectedResult6.pig?rev=1085612view=auto
==
--- pig/trunk/test/org/apache/pig/test/data/ExpectedResult6.pig (added)
+++ pig/trunk/test/org/apache/pig/test/data/ExpectedResult6.pig Sat Mar 26
00:07:41 2011
@@ -0,0 +1,15 @@
+define mymacro1(A, rate) returns B {
+C = group $A by $0 parallel $rate;
+B = foreach C generate group, COUNT($A);
+};
+
+aa = load
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/20080228'
using PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts
); }
+store countInactiveAcct into
'/user/kaleidoscope/pow_stats/20080228/acct_stats/InactiveAcctCount';
Added: pig/trunk/test/org/apache/pig/test/data/input6.pig
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/data/input6.pig?rev=1085612view=auto
==
--- pig/trunk/test/org/apache/pig/test/data/input6.pig (added)
+++ pig/trunk/test/org/apache/pig/test/data/input6.pig Sat Mar 26 00:07:41 2011
@@ -0,0 +1,15 @@
+define mymacro1(A, rate) returns B {
+C = group $A by $0 parallel $rate;
+B = foreach C generate group, COUNT($A);
+};
+
+aa = load
'/data/intermediate/pow/elcarobootstrap/account/full/weekly/data/$date' using
PigStorage('\x01');
+bb = filter aa by (ARITY == '16') and ( $4 eq '' or $4 eq 'NULL' or $4 eq
'ss') parallel 400;
+a = foreach bb generate $0,$12,$7;
+
+--generate inactive accts
+inactiveAccounts = filter a by ($1 neq '') and ($1 == '2') parallel 400;
+store inactiveAccounts into
'/user/kaleidoscope/pow_stats/20080228/acct/InactiveAcct';
+grpInactiveAcct = group inactiveAccounts all;
+countInactiveAcct = foreach grpInactiveAcct { generate COUNT( inactiveAccounts
); }
+store countInactiveAcct into
svn commit: r1085612 [2/4] - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/parser/ src/org/apache/pig/scripting/ src/org/apache/pig/tools/parameters/ src/org/apache/pig/tools/pigscript/pars
Modified: pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java?rev=1085612r1=1085611r2=1085612view=diff
==
--- pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java (original)
+++ pig/trunk/src/org/apache/pig/parser/QueryParserDriver.java Sat Mar 26
00:07:41 2011
@@ -18,31 +18,55 @@
package org.apache.pig.parser;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
+import java.io.FileNotFoundException;
+import java.io.FileReader;
+import java.io.FileWriter;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.antlr.runtime.BaseRecognizer;
import org.antlr.runtime.CharStream;
import org.antlr.runtime.CommonTokenStream;
import org.antlr.runtime.RecognitionException;
+import org.antlr.runtime.tree.CommonTree;
import org.antlr.runtime.tree.CommonTreeNodeStream;
import org.antlr.runtime.tree.Tree;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.pig.impl.PigContext;
import org.apache.pig.newplan.Operator;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
+import org.apache.pig.tools.pigstats.ScriptState;
public class QueryParserDriver {
public static ClassLoader classloader =
QueryParserDriver.class.getClassLoader();
+private static final Log LOG = LogFactory.getLog(QueryParserDriver.class);
+
+private static final String MACRO_DEF = MACRO_DEF;
+private static final String MACRO_INLINE = MACRO_INLINE;
+private static final String IMPORT_DEF = import;
+
private PigContext pigContext;
private String scope;
private MapString, StringfileNameMap;
private MapString, Operator operators;
+private SetString importSeen;
+private SetString macroSeen;
public QueryParserDriver(PigContext pigContext, String scope, MapString,
String fileNameMap) {
this.pigContext = pigContext;
this.scope = scope;
this.fileNameMap = fileNameMap;
+importSeen = new HashSetString();
+macroSeen = new HashSetString();
}
public LogicalPlan parse(String query) throws ParserException {
@@ -55,7 +79,9 @@ public class QueryParserDriver {
ast = parse( tokenStream );
} catch(RuntimeException ex) {
throw new ParserException( ex.getMessage() );
-}
+}
+
+ast = expandMacro(ast);
try{
ast = validateAst( ast );
@@ -76,6 +102,44 @@ public class QueryParserDriver {
return plan;
}
+public boolean dryrun(String scriptFile) throws ParserException,
IOException, RecognitionException {
+BufferedReader rd = new BufferedReader(new FileReader(scriptFile));
+StringBuilder sb = new StringBuilder();
+String line = rd.readLine();
+while (line != null) {
+sb.append(line).append(\n);
+line = rd.readLine();
+}
+
+CommonTokenStream tokenStream = tokenize(sb.toString());
+Tree ast = null;
+
+try {
+ast = parse( tokenStream );
+} catch(RuntimeException ex) {
+throw new ParserException( ex.getMessage() );
+}
+
+ListCommonTree importNodes = new ArrayListCommonTree();
+ListCommonTree macroNodes = new ArrayListCommonTree();
+ListCommonTree inlineNodes = new ArrayListCommonTree();
+
+traverseImport(ast, importNodes);
+traverse(ast, macroNodes, inlineNodes);
+
+if (importNodes.isEmpty() macroNodes.isEmpty()
+ inlineNodes.isEmpty()) {
+return false;
+}
+
+ast = expandMacro(ast);
+
+String expandedFile = scriptFile.replace(.substituted, .expanded);
+dryrun(ast, expandedFile);
+
+return true;
+}
+
public MapString, Operator getOperators() {
return operators;
}
@@ -83,39 +147,52 @@ public class QueryParserDriver {
private static CommonTokenStream tokenize(String query) throws
ParserException {
CharStream input;
try {
-input = new QueryParserStringStream( query );
-} catch(IOException ex) {
-throw new ParserException( Unexpected IOException: +
ex.getMessage() );
-}
-QueryLexer lexer = new QueryLexer( input );
-CommonTokenStream tokens = new CommonTokenStream( lexer );
-checkError( lexer );
+input = new QueryParserStringStream(query);
+} catch (IOException ex) {
+throw new ParserException(Unexpected IOException:
++ ex.getMessage());
+}
svn commit: r1080747 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/ src/org/apache/pig/impl/ src/org/apache/pig/impl/util/ src/org/apache/pig/
Author: rding
Date: Fri Mar 11 20:08:38 2011
New Revision: 1080747
URL: http://svn.apache.org/viewvc?rev=1080747view=rev
Log:
PIG-1874: Make PigServer work in a multithreading environment
Added:
pig/trunk/test/org/apache/pig/test/utils/UDFContextTestEvalFunc3.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/POStatus.java
pig/trunk/src/org/apache/pig/impl/PigContext.java
pig/trunk/src/org/apache/pig/impl/util/UDFContext.java
pig/trunk/src/org/apache/pig/scripting/BoundScript.java
pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1080747r1=1080746r2=1080747view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Fri Mar 11 20:08:38 2011
@@ -26,6 +26,8 @@ PIG-1680: HBaseStorage should work with
IMPROVEMENTS
+PIG-1874: Make PigServer work in a multithreading environment (rding)
+
PIG-1188: Padding nulls to the input tuple according to input schema (daijy)
PIG-1876: Typed map for Pig (daijy)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1080747r1=1080746r2=1080747view=diff
==
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Fri Mar 11 20:08:38 2011
@@ -41,6 +41,7 @@ import java.util.Properties;
import java.util.Queue;
import java.util.Set;
import java.util.Stack;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -172,7 +173,8 @@ public class PigServer {
private String jobPriority;
-private static int scopeCounter = 0;
+private final static AtomicInteger scopeCounter = new AtomicInteger(0);
+
private final String scope = constructScope();
@@ -189,7 +191,7 @@ public class PigServer {
// username-date. For now let's simplify the scope, if a real
// scope is needed again, we might need to update all the
// operators to not include scope in their name().
-return +(++scopeCounter);
+return + scopeCounter.incrementAndGet();
}
/**
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/POStatus.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/POStatus.java?rev=1080747r1=1080746r2=1080747view=diff
==
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/POStatus.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/POStatus.java
Fri Mar 11 20:08:38 2011
@@ -33,6 +33,4 @@ public class POStatus {
// successfully processing of a batch, used by accumulative UDFs
// this is used for accumulative UDFs
public static final byte STATUS_BATCH_OK = 5;
-
-public static Object result;
}
Modified: pig/trunk/src/org/apache/pig/impl/PigContext.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/PigContext.java?rev=1080747r1=1080746r2=1080747view=diff
==
--- pig/trunk/src/org/apache/pig/impl/PigContext.java (original)
+++ pig/trunk/src/org/apache/pig/impl/PigContext.java Fri Mar 11 20:08:38 2011
@@ -107,7 +107,8 @@ public class PigContext implements Seria
private MapString, StreamingCommand definedCommands =
new HashMapString, StreamingCommand();
-private static ArrayListString packageImportList = new
ArrayListString();
+private static ThreadLocalArrayListString packageImportList =
+new ThreadLocalArrayListString();
private Properties log4jProperties = new Properties();
@@ -154,23 +155,18 @@ public class PigContext implements Seria
skippedShipPaths.add(/usr/sbin);
skippedShipPaths.add(/usr/local/sbin);
}
-
-static{
-packageImportList.add();
-packageImportList.add(org.apache.pig.builtin.);
-packageImportList.add(org.apache.pig.impl.builtin.);
-}
public static void initializeImportList(String
importListCommandLineProperties)
{
StringTokenizer tokenizer = new
StringTokenizer(importListCommandLineProperties, :);
int pos = 0;
+ArrayListString importList = getPackageImportList();
while (tokenizer.hasMoreTokens())
{
String importItem = tokenizer.nextToken();
if (!importItem.endsWith(.))
importItem
svn commit: r1080384 - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/backend/hadoop/executionengine/ src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/ src/org/apache/pig/pen
Author: rding
Date: Thu Mar 10 22:50:55 2011
New Revision: 1080384
URL: http://svn.apache.org/viewvc?rev=1080384view=rev
Log:
PIG-1862: Pig returns exit code 0 for the failed Pig script due to non-existing
input directory
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/PigServer.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MapReduceLauncher.java
pig/trunk/src/org/apache/pig/pen/LocalMapReduceSimulator.java
pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
pig/trunk/src/org/apache/pig/tools/pigstats/JobStats.java
pig/trunk/test/org/apache/pig/test/TestGroupConstParallel.java
pig/trunk/test/org/apache/pig/test/TestJobSubmission.java
pig/trunk/test/org/apache/pig/test/TestMultiQueryLocal.java
pig/trunk/test/org/apache/pig/test/TestPigRunner.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1080384r1=1080383r2=1080384view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Mar 10 22:50:55 2011
@@ -108,6 +108,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1862: Pig returns exit code 0 for the failed Pig script due to
non-existing input directory (rding)
+
PIG-1888: Fix TestLogicalPlanGenerator not use hardcoded path (daijy)
PIG-1837: Error while using IsEmpty function (rding)
Modified: pig/trunk/src/org/apache/pig/PigServer.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/PigServer.java?rev=1080384r1=1080383r2=1080384view=diff
==
--- pig/trunk/src/org/apache/pig/PigServer.java (original)
+++ pig/trunk/src/org/apache/pig/PigServer.java Thu Mar 10 22:50:55 2011
@@ -54,8 +54,10 @@ import org.apache.pig.backend.executione
import org.apache.pig.backend.executionengine.ExecJob;
import org.apache.pig.backend.executionengine.ExecJob.JOB_STATUS;
import org.apache.pig.backend.hadoop.executionengine.HJob;
+import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStore;
+import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.builtin.PigStorage;
import org.apache.pig.classification.InterfaceAudience;
import org.apache.pig.classification.InterfaceStability;
@@ -975,7 +977,11 @@ public class PigServer {
currDAG.lp.explain(lps, format, verbose);
pp.explain(pps, format, verbose);
-pigContext.getExecutionEngine().explain(pp, eps, format, verbose);
+
+MapRedUtil.checkLeafIsStore(pp, pigContext);
+MapReduceLauncher launcher = new MapReduceLauncher();
+launcher.explain(pp, pigContext, eps, format, verbose);
+
if (markAsExecute) {
currDAG.markAsExecuted();
}
@@ -1216,25 +1222,40 @@ public class PigServer {
// discover pig features used in this script
ScriptState.get().setScriptFeatures( currDAG.lp );
PhysicalPlan pp = compilePp();
-// execute using appropriate engine
-ListExecJob jobs = pigContext.getExecutionEngine().execute(pp,
job_pigexec_);
+
+MapReduceLauncher launcher = new MapReduceLauncher();
PigStats stats = null;
-if (jobs.size() 0) {
-stats = jobs.get(0).getStatistics();
-} else {
-stats = PigStatsUtil.getEmptyPigStats();
+try {
+stats = launcher.launchPig(pp, job_pigexec_, pigContext);
+} catch (Exception e) {
+// There are a lot of exceptions thrown by the launcher. If this
+// is an ExecException, just let it through. Else wrap it.
+if (e instanceof ExecException){
+throw (ExecException)e;
+} else if (e instanceof FrontendException) {
+throw (FrontendException)e;
+} else {
+int errCode = 2043;
+String msg = Unexpected error during execution.;
+throw new ExecException(msg, errCode, PigException.BUG, e);
+}
+} finally {
+launcher.reset();
}
+
for (OutputStats output : stats.getOutputStats()) {
if (!output.isSuccessful()) {
POStore store = output.getPOStore();
try {
-
store.getStoreFunc().cleanupOnFailure(store.getSFile().getFileName(),
+store.getStoreFunc().cleanupOnFailure(
+store.getSFile().getFileName(),
new Job
svn commit: r1078875 - in /pig/trunk: CHANGES.txt src/org/apache/pig/tools/pigstats/ScriptState.java test/org/apache/pig/test/TestPigStats.java
Author: rding
Date: Mon Mar 7 18:09:02 2011
New Revision: 1078875
URL: http://svn.apache.org/viewvc?rev=1078875view=rev
Log:
The pig script stored in the Hadoop History logs is stored as a concatenated
string without whitespace this causes problems when attempting to extract and
execute the script
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
pig/trunk/test/org/apache/pig/test/TestPigStats.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1078875r1=1078874r2=1078875view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Mar 7 18:09:02 2011
@@ -102,6 +102,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1861: The pig script stored in the Hadoop History logs is stored as a
concatenated string without whitespace this causes problems when attempting to
extract and execute the script (rding)
+
PIG-1829: 0 value seen in PigStat's map/reduce runtime, even when the job is
successful (rding)
PIG-1856: Custom jar is not packaged with the new job created by LimitAdjuster
(rding)
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=1078875r1=1078874r2=1078875view=diff
==
--- pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Mon Mar 7
18:09:02 2011
@@ -386,9 +386,8 @@ public class ScriptState {
try {
String line = reader.readLine();
while (line != null) {
-line = line.trim();
-if (line.length() 0 !line.startsWith(--)) {
-sb.append(line);
+if (line.length() 0) {
+sb.append(line).append(\n);
}
line = reader.readLine();
}
Modified: pig/trunk/test/org/apache/pig/test/TestPigStats.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigStats.java?rev=1078875r1=1078874r2=1078875view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestPigStats.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigStats.java Mon Mar 7 18:09:02
2011
@@ -18,34 +18,107 @@
package org.apache.pig.test;
-import static org.junit.Assert.*;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
import java.io.File;
+import java.io.FileWriter;
import java.io.IOException;
+import java.io.PrintWriter;
import junit.framework.Assert;
+import org.apache.commons.codec.binary.Base64;
+import org.apache.hadoop.conf.Configuration;
import org.apache.pig.ExecType;
-import org.apache.pig.PigException;
import org.apache.pig.PigServer;
import org.apache.pig.backend.executionengine.ExecJob;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceLauncher;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.MapReduceOper;
import
org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.plans.MROperPlan;
import
org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan;
-import org.apache.pig.backend.hadoop.executionengine.util.MapRedUtil;
import org.apache.pig.impl.PigContext;
-import org.apache.pig.impl.util.LogUtils;
+import org.apache.pig.impl.plan.OperatorKey;
import org.apache.pig.newplan.logical.relational.LogicalPlan;
-import org.apache.pig.tools.pigscript.parser.ParseException;
import org.apache.pig.tools.pigstats.PigStats;
import org.apache.pig.tools.pigstats.ScriptState;
-import org.junit.Before;
import org.junit.Test;
public class TestPigStats {
@Test
+public void testPigScriptInConf() throws Exception {
+PrintWriter w = new PrintWriter(new FileWriter(test.pig));
+w.println(register /mydir/sath.jar);
+w.println(register /mydir/lib/hadoop-tools-0.20.201.0-SNAPSHOT.jar);
+w.println(register /mydir/lib/jackson-core-asl-1.4.2.jar);
+w.println(register /mydir/lib/jackson-mapper-asl-1.4.2.jar);
+w.close();
+
+ScriptState ss = ScriptState.get();
+ss.setScript(new File(test.pig));
+Configuration conf = new Configuration();
+MapReduceOper mro = new MapReduceOper(new OperatorKey());
+ss.addSettingsToConf(mro, conf);
+
+String s = conf.get(pig.script);
+String script = new String(Base64.decodeBase64(s.getBytes()));
+
+String expected =
+register /mydir/sath.jar\n +
+register /mydir/lib/hadoop-tools-0.20.201.0-SNAPSHOT.jar\n +
+register /mydir/lib
svn commit: r1075998 - in /pig/trunk: CHANGES.txt src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java test/org/apache/pig/test/TestMRCompiler.java
Author: rding
Date: Tue Mar 1 19:51:32 2011
New Revision: 1075998
URL: http://svn.apache.org/viewvc?rev=1075998view=rev
Log:
PIG-1856: Custom jar is not packaged with the new job created by LimitAdjuster
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1075998r1=1075997r2=1075998view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Mar 1 19:51:32 2011
@@ -100,6 +100,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1856: Custom jar is not packaged with the new job created by LimitAdjuster
(rding)
+
PIG-1872: Fix bug in AvroStorage (guolin2001, jghoman via daijy)
PIG-1536: use same logic for merging inner schemas in default union and
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java?rev=1075998r1=1075997r2=1075998view=diff
==
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/MRCompiler.java
Tue Mar 1 19:51:32 2011
@@ -2819,7 +2819,7 @@ public class MRCompiler extends PhyPlanV
public void visitMROp(MapReduceOper mr) throws VisitorException {
// Look for map reduce operators which contains limit operator.
// If so and the requestedParallelism 1, add one additional
map-reduce
-// operator with 1 reducer into the original plan
+// operator with 1 reducer into the original plan
if (mr.limit!=-1 mr.requestedParallelism!=1)
{
opsToAdjust.add(mr);
@@ -2895,6 +2895,13 @@ public class MRCompiler extends PhyPlanV
for (MapReduceOper op:successorList)
successors[i++] = op;
}
+
+// Process UDFs
+for (String udf : mr.UDFs) {
+if (!limitAdjustMROp.UDFs.contains(udf)) {
+limitAdjustMROp.UDFs.add(udf);
+}
+}
MRPlan.add(limitAdjustMROp);
MRPlan.connect(mr, limitAdjustMROp);
Modified: pig/trunk/test/org/apache/pig/test/TestMRCompiler.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMRCompiler.java?rev=1075998r1=1075997r2=1075998view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestMRCompiler.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestMRCompiler.java Tue Mar 1 19:51:32
2011
@@ -17,6 +17,8 @@
*/
package org.apache.pig.test;
+import static org.junit.Assert.assertTrue;
+
import java.io.ByteArrayOutputStream;
import java.io.FileInputStream;
import java.io.FileOutputStream;
@@ -995,6 +997,30 @@ public class TestMRCompiler extends juni
assertTrue(op.UDFs.contains(new
FuncSpec(PigStorageNoDefCtor.class.getName())+('\t')));
}
+@Test
+public void testLimitAdjusterFuncShipped() throws Exception{
+
+planTesterMR.buildPlan(a = load 'input';);
+planTesterMR.buildPlan(b = order a by $0 parallel 2;);
+planTesterMR.buildPlan(c = limit b 7;);
+LogicalPlan lp = planTesterMR.buildPlan(store c into '/tmp' using
++ PigStorageNoDefCtor.class.getName() + ('\t'););
+
+PhysicalPlan pp = Util.buildPhysicalPlan(lp, pc);
+MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
+MapReduceOper mrOper = mrPlan.getRoots().get(0);
+int count = 1;
+
+while(mrPlan.getSuccessors(mrOper) != null) {
+mrOper = mrPlan.getSuccessors(mrOper).get(0);
+++count;
+}
+assertTrue(count == 4);
+
+MapReduceOper op = mrPlan.getLeaves().get(0);
+assertTrue(op.UDFs.contains(new
FuncSpec(PigStorageNoDefCtor.class.getName())+('\t')));
+}
+
/**
* Test that POSortedDistinct gets printed as POSortedDistinct
* @throws Exception
svn commit: r1071119 - /pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java
Author: rding
Date: Wed Feb 16 01:18:45 2011
New Revision: 1071119
URL: http://svn.apache.org/viewvc?rev=1071119view=rev
Log:
disable a unit test due to PIG-1854
Modified:
pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java
Modified: pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java?rev=1071119r1=1071118r2=1071119view=diff
==
--- pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java (original)
+++ pig/branches/branch-0.8/test/org/apache/pig/test/TestGrunt.java Wed Feb 16
01:18:45 2011
@@ -812,7 +812,7 @@ public class TestGrunt extends TestCase
assertTrue(caught);
}
-@Test
+//@Test
public void testFsCommand(){
try {
svn commit: r1070662 - in /pig/trunk: CHANGES.txt lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java test/org/apache/pig/test/TestBZip.java
Author: rding
Date: Mon Feb 14 20:51:41 2011
New Revision: 1070662
URL: http://svn.apache.org/viewvc?rev=1070662view=rev
Log:
PIG-1304: Fail underlying M/R jobs when concatenated gzip and bz2 files are
provided as input
Modified:
pig/trunk/CHANGES.txt
pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java
pig/trunk/test/org/apache/pig/test/TestBZip.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1070662r1=1070661r2=1070662view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Feb 14 20:51:41 2011
@@ -96,6 +96,8 @@ PIG-1696: Performance: Use System.arrayc
BUG FIXES
+PIG-1304: Fail underlying M/R jobs when concatenated gzip and bz2 files are
provided as input (laukik via rding)
+
PIG-1852: Packaging antlr jar with pig.jar (rding via daijy)
PIG-1717 pig needs to call setPartitionFilter if schema is null but
Modified: pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java
URL:
http://svn.apache.org/viewvc/pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java?rev=1070662r1=1070661r2=1070662view=diff
==
--- pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java
(original)
+++ pig/trunk/lib-src/bzip2/org/apache/tools/bzip2r/CBZip2InputStream.java Mon
Feb 14 20:51:41 2011
@@ -220,10 +220,6 @@ public class CBZip2InputStream extends I
initBlock(blockSize != -1);
setupBlock();
}
-
-public CBZip2InputStream(FSDataInputStream zStream) throws IOException {
- this(zStream, -1, Long.MAX_VALUE);
-}
@Override
public int read() throws IOException {
@@ -408,7 +404,10 @@ public class CBZip2InputStream extends I
storedCombinedCRC != computedCombinedCRC) {
crcError();
}
-
+if (innerBsStream.getPos() endOffsetOfSplit) {
+ throw new IOException(Encountered additional bytes in the
filesplit past the crc block.
+ + Loading of concatenated bz2 files is not
supported);
+}
bsFinishedWithStream();
streamEnd = true;
}
Modified: pig/trunk/test/org/apache/pig/test/TestBZip.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBZip.java?rev=1070662r1=1070661r2=1070662view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestBZip.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBZip.java Mon Feb 14 20:51:41 2011
@@ -21,8 +21,12 @@ import static org.junit.Assert.*;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotSame;
+import java.io.BufferedReader;
+import java.io.BufferedWriter;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.FileWriter;
import java.io.IOException;
import java.util.HashMap;
import java.util.Iterator;
@@ -91,7 +95,7 @@ public class TestBZip {
pig.getPigContext().getProperties()));
FSDataInputStream is = fs.open(new Path(out.getAbsolutePath() +
/part-r-0.bz2));
-CBZip2InputStream cis = new CBZip2InputStream(is);
+CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
// Just a sanity check, to make sure it was a bzip file; we
// will do the value verification later
@@ -150,7 +154,7 @@ public class TestBZip {
pig.getPigContext().getProperties()));
FSDataInputStream is = fs.open(new Path(out.getAbsolutePath() +
/part-r-0.bz2));
-CBZip2InputStream cis = new CBZip2InputStream(is);
+CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
// Just a sanity check, to make sure it was a bzip file; we
// will do the value verification later
@@ -266,7 +270,7 @@ public class TestBZip {
pig.getPigContext().getProperties()));
FSDataInputStream is = fs.open(new Path(out.getAbsolutePath() +
/part-r-0.bz2));
-CBZip2InputStream cis = new CBZip2InputStream(is);
+CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
// Just a sanity check, to make sure it was a bzip file; we
// will do the value verification later
@@ -294,7 +298,7 @@ public class TestBZip {
assertNotSame(0, tmp.length());
FileSystem fs = FileSystem.getLocal(new Configuration(false));
CBZip2InputStream cis = new CBZip2InputStream(
-fs.open(new Path(tmp.getAbsolutePath(;
+fs.open(new Path(tmp.getAbsolutePath())), -1, tmp.length());
assertEquals(-1, cis.read(new byte[100]));
cis.close
svn commit: r1070705 - in /pig/trunk: CHANGES.txt build.xml ivy.xml ivy/libraries.properties ivy/pig.pom lib/antlr-3.2.jar
Author: rding
Date: Tue Feb 15 00:26:05 2011
New Revision: 1070705
URL: http://svn.apache.org/viewvc?rev=1070705view=rev
Log:
PIG-1853: Using ANTLR jars from maven repository
Removed:
pig/trunk/lib/antlr-3.2.jar
Modified:
pig/trunk/CHANGES.txt
pig/trunk/build.xml
pig/trunk/ivy.xml
pig/trunk/ivy/libraries.properties
pig/trunk/ivy/pig.pom
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1070705r1=1070704r2=1070705view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Feb 15 00:26:05 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1853: Using ANTLR jars from maven repository (rding)
+
PIG-1728: more doc updates (chandec via olgan)
PIG-1793: Add macro expansion to Pig Latin (rding)
Modified: pig/trunk/build.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1070705r1=1070704r2=1070705view=diff
==
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Tue Feb 15 00:26:05 2011
@@ -115,10 +115,6 @@
property name=grammar.package.dir value=org/apache/pig/parser/
property name=grammar.src.dir
value=${src.dir}/${grammar.package.dir}/
-!-- Antlr properties --
-property name=antlr.version value=3.2/
-property name=antlr.jar value=antlr-${antlr.version}.jar/
-
!-- rats properties --
property name=rat.reporting.classname value=rat.Report/
@@ -204,7 +200,6 @@
fileset file=${lib.dir}/${hbase.jarfile} /
fileset file=${lib.dir}/${hbase.test.jarfile} /
fileset file=${lib.dir}/${zookeeper.jarfile}/
- fileset file=${lib.dir}/${antlr.jar}/
/path
!-- javadoc-classpath --
@@ -213,7 +208,6 @@
fileset file=${lib.dir}/${guava.jarfile} /
fileset file=${lib.dir}/${automaton.jarfile} /
fileset file=${lib.dir}/${hbase.test.jarfile} /
- fileset file=${lib.dir}/${antlr.jar}/
path refid=javadoc.classpath/
/path
@@ -553,7 +547,8 @@
/manifest
zipfileset
src=${ivy.lib.dir}/hadoop-core-${hadoop-core.version}.jar /
zipfileset src=${lib.dir}/${automaton.jarfile} /
-zipfileset src=${lib.dir}/${antlr.jar} /
+zipfileset
src=${ivy.lib.dir}/antlr-runtime-${antlr.version}.jar /
+zipfileset
src=${ivy.lib.dir}/stringtemplate-${antlr.version}.jar /
zipfileset src=${ivy.lib.dir}/junit-${junit.version}.jar /
zipfileset src=${ivy.lib.dir}/jsch-${jsch.version}.jar /
zipfileset src=${ivy.lib.dir}/jline-${jline.version}.jar /
@@ -640,7 +635,8 @@
zipfileset
src=${ivy.lib.dir}/jackson-core-asl-${jackson.version}.jar /
zipfileset
src=${ivy.lib.dir}/joda-time-${joda-time.version}.jar /
zipfileset src=${lib.dir}/${automaton.jarfile} /
-zipfileset src=${lib.dir}/${antlr.jar} /
+zipfileset
src=${ivy.lib.dir}/antlr-runtime-${antlr.version}.jar /
+zipfileset
src=${ivy.lib.dir}/stringtemplate-${antlr.version}.jar /
fileset file=${basedir}/conf/pig-default.properties /
/jar
copy file=${output.jarfile.withouthadoop}
tofile=${output.jarfile.backcompat.withouthadoop}/
@@ -1046,8 +1042,6 @@
/target
--
path id=checkstyle-classpath
- path refid=checkstyle.classpath/
- fileset file=${lib.dir}/${antlr.jar}/
/path
target name=findbugs.check depends=check-for-findbugs
unless=findbugs.present
Modified: pig/trunk/ivy.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/ivy.xml?rev=1070705r1=1070704r2=1070705view=diff
==
--- pig/trunk/ivy.xml (original)
+++ pig/trunk/ivy.xml Tue Feb 15 00:26:05 2011
@@ -93,6 +93,8 @@
conf=compile-master/
dependency org=com.google.guava name=guava rev=${guava.version}
conf=compile-master /
dependency org=org.python name=jython rev=${jython.version}
conf=compile-master/
+ dependency org=org.antlr name=antlr rev=${antlr.version}
conf=compile-master/
+ dependency org=org.antlr name=antlr-runtime rev=${antlr.version}
conf=compile-default/
!--ATM hbase, hbase-test.jar, hadoop.jar are resolved from the lib dir--
dependency org=hsqldb name=hsqldb rev=${hsqldb.version}
Modified: pig/trunk/ivy/libraries.properties
URL:
http://svn.apache.org/viewvc/pig/trunk/ivy/libraries.properties?rev=1070705r1=1070704r2=1070705view=diff
==
--- pig/trunk/ivy/libraries.properties (original)
+++ pig/trunk/ivy/libraries.properties Tue Feb 15 00:26:05 2011
@@ -41,6 +41,7 @@ jsch.version=0.1.38
json-simple.version=1.1
junit.version=4.5
jython.version=2.5.0
+antlr.version=3.2
svn commit: r1070714 - in /pig/trunk: build.xml src/org/apache/pig/parser/AliasMasker.g src/org/apache/pig/parser/MacroExpansion.g src/org/apache/pig/parser/PigMacro.java test/org/apache/pig/test/Test
Author: rding
Date: Tue Feb 15 01:09:01 2011
New Revision: 1070714
URL: http://svn.apache.org/viewvc?rev=1070714view=rev
Log:
PIG-1793: Add macro expansion to Pig Latin
Modified:
pig/trunk/build.xml
pig/trunk/src/org/apache/pig/parser/AliasMasker.g
pig/trunk/src/org/apache/pig/parser/MacroExpansion.g
pig/trunk/src/org/apache/pig/parser/PigMacro.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified: pig/trunk/build.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1070714r1=1070713r2=1070714view=diff
==
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Tue Feb 15 01:09:01 2011
@@ -320,7 +320,7 @@
description=generates token parser class from an ANTLR grammar
java classname=org.antlr.Tool
classpathref=classpath fork=true
- arg line=-o ${src.gen.dir}/${grammar.package.dir}
${src.dir}/${grammar.package.dir}/MacroExpansion.g
${src.dir}/${grammar.package.dir}/MacroImport.g
${src.dir}/${grammar.package.dir}/${grammar.name}Parser.g/
+ arg line=-o ${src.gen.dir}/${grammar.package.dir}
${src.dir}/${grammar.package.dir}/MacroExpansion.g
${src.dir}/${grammar.package.dir}/MacroRecursion.g
${src.dir}/${grammar.package.dir}/MacroImport.g
${src.dir}/${grammar.package.dir}/${grammar.name}Parser.g/
/java
/target
Modified: pig/trunk/src/org/apache/pig/parser/AliasMasker.g
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/AliasMasker.g?rev=1070714r1=1070713r2=1070714view=diff
==
--- pig/trunk/src/org/apache/pig/parser/AliasMasker.g (original)
+++ pig/trunk/src/org/apache/pig/parser/AliasMasker.g Tue Feb 15 01:09:01 2011
@@ -75,6 +75,8 @@ private String getMask(String alias) {
private SetString params = new HashSetString();
+private SetString aliasSeen = new HashSetString();
+
private String macroName = ;
private long index = 0;
@@ -109,7 +111,8 @@ foreach_statement
: ^( STATEMENT ( alias { sb.append( = ); } )? foreach_clause {
sb.append(\n); } )
;
-alias : IDENTIFIER { sb.append(getMask($IDENTIFIER.text)); }
+alias
+: IDENTIFIER { sb.append(getMask($IDENTIFIER.text));
aliasSeen.add($IDENTIFIER.text); }
;
op_clause : define_clause
@@ -530,7 +533,13 @@ col_ref : alias_col_ref | dollar_col_ref
alias_col_ref
: GROUP { sb.append($GROUP.text); }
-| IDENTIFIER { sb.append($IDENTIFIER.text); }
+| name = IDENTIFIER {
+if (aliasSeen.contains($name.text)) {
+sb.append(getMask($name.text));
+} else {
+sb.append($name.text);
+}
+}
;
dollar_col_ref
@@ -689,3 +698,4 @@ rel_str_op
| STR_OP_LTE { sb.append($STR_OP_LTE.text); }
| STR_OP_MATCHES { sb.append($STR_OP_MATCHES.text); }
;
+
Modified: pig/trunk/src/org/apache/pig/parser/MacroExpansion.g
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/MacroExpansion.g?rev=1070714r1=1070713r2=1070714view=diff
==
--- pig/trunk/src/org/apache/pig/parser/MacroExpansion.g (original)
+++ pig/trunk/src/org/apache/pig/parser/MacroExpansion.g Tue Feb 15 01:09:01
2011
@@ -27,12 +27,11 @@ package org.apache.pig.parser;
import java.util.HashMap;
import java.util.List;
-import java.io.IOException;
import org.apache.pig.parser.PigMacro;
}
@members {
-private HashMapString, PigMacro memory = new HashMapString, PigMacro();
+private MapString, PigMacro memory = new HashMapString, PigMacro();
private StringBuilder sb = new StringBuilder();
public String getResultString() { return sb.toString(); }
@@ -42,7 +41,7 @@ MACRO
: 'define' WS name=ALIAS WS? '(' ( params+=ALIAS (',' WS* params+=ALIAS)*
)? ')' WS 'returns' WS rets+=ALIAS (',' WS* rets+=ALIAS)* WS '{' content=BLOCK
'};'
{
PigMacro macro = new PigMacro($name.text);
-macro.setBody($content.text);
+macro.setBody($content.text, memory);
if ($params != null) {
for (Object param : $params) {
macro.addParam(((Token)param).getText());
Modified: pig/trunk/src/org/apache/pig/parser/PigMacro.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/parser/PigMacro.java?rev=1070714r1=1070713r2=1070714view=diff
==
--- pig/trunk/src/org/apache/pig/parser/PigMacro.java (original)
+++ pig/trunk/src/org/apache/pig/parser/PigMacro.java Tue Feb 15 01:09:01 2011
@@ -18,22 +18,24 @@
package org.apache.pig.parser;
import java.io.BufferedReader;
+import java.io.IOException;
import java.io.StringReader;
import java.io.StringWriter;
import java.util.ArrayList;
-import java.util.Arrays
svn commit: r1070723 - in /pig/trunk: CHANGES.txt src/org/apache/pig/impl/util/LogUtils.java src/org/apache/pig/tools/grunt/GruntParser.java test/org/apache/pig/test/TestPigRunner.java
Author: rding
Date: Tue Feb 15 01:27:47 2011
New Revision: 1070723
URL: http://svn.apache.org/viewvc?rev=1070723view=rev
Log:
PIG-1854: Pig returns exit code 0 for the failed Pig script
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/impl/util/LogUtils.java
pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
pig/trunk/test/org/apache/pig/test/TestPigRunner.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1070723r1=1070722r2=1070723view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue Feb 15 01:27:47 2011
@@ -299,6 +299,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1854: Pig returns exit code 0 for the failed Pig script (rding)
+
PIG-1812: Problem with DID_NOT_FIND_LOAD_ONLY_MAP_PLAN (daijy)
PIG-1813: Pig 0.8 throws ERROR 1075 while trying to refer a map in the result
Modified: pig/trunk/src/org/apache/pig/impl/util/LogUtils.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/impl/util/LogUtils.java?rev=1070723r1=1070722r2=1070723view=diff
==
--- pig/trunk/src/org/apache/pig/impl/util/LogUtils.java (original)
+++ pig/trunk/src/org/apache/pig/impl/util/LogUtils.java Tue Feb 15 01:27:47
2011
@@ -99,6 +99,8 @@ public class LogUtils {
|| t instanceof
org.apache.pig.tools.pigscript.parser.TokenMgrError
|| t instanceof
org.apache.pig.impl.logicalLayer.parser.TokenMgrError)) {
message = ERROR 1000: Error during parsing. +
t.getMessage();
+} else if (t instanceof IOException) {
+message = ERROR 2997: Encountered IOException. +
t.getMessage();
} else if (t instanceof RuntimeException) {
message = ERROR 2999: Unexpected internal error. +
t.getMessage();
} else {
Modified: pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java?rev=1070723r1=1070722r2=1070723view=diff
==
--- pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java (original)
+++ pig/trunk/src/org/apache/pig/tools/grunt/GruntParser.java Tue Feb 15
01:27:47 2011
@@ -31,6 +31,8 @@ import java.io.PrintStream;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
+import java.util.AbstractList;
+import java.util.Arrays;
import java.util.Date;
import java.util.Iterator;
import java.util.List;
@@ -47,6 +49,7 @@ import org.apache.hadoop.mapred.JobClien
import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.JobID;
import org.apache.hadoop.mapred.RunningJob;
+import org.apache.pig.LoadFunc;
import org.apache.pig.PigServer;
import org.apache.pig.backend.datastorage.ContainerDescriptor;
import org.apache.pig.backend.datastorage.DataStorage;
@@ -935,11 +938,20 @@ public class GruntParser extends PigScri
executeBatch();
+int retCode = -1;
+
try {
-shell.run(cmdTokens);
+retCode = shell.run(cmdTokens);
} catch (Exception e) {
throw new IOException(e);
}
+
+if (retCode != 0 !mInteractive) {
+String s = LoadFunc.join(
+(AbstractListString) Arrays.asList(cmdTokens), );
+throw new IOException(fs command ' + s
++ ' failed. Please check output logs for details);
+}
} else {
log.warn('fs' statement is ignored while processing 'explain
-script' or '-check');
}
Modified: pig/trunk/test/org/apache/pig/test/TestPigRunner.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPigRunner.java?rev=1070723r1=1070722r2=1070723view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestPigRunner.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPigRunner.java Tue Feb 15 01:27:47
2011
@@ -509,6 +509,23 @@ public class TestPigRunner {
}
}
+@Test
+public void fsCommandTest() throws Exception {
+PrintWriter w = new PrintWriter(new FileWriter(PIG_FILE));
+w.println(fs -mv nonexist.file dummy.file);
+w.close();
+
+try {
+String[] args = { PIG_FILE };
+PigStats stats = PigRunner.run(args, new
TestNotificationListener());
+
+assertTrue(!stats.isSuccessful());
+assertTrue(stats.getReturnCode() ==
PigRunner.ReturnCode.IO_EXCEPTION);
+} finally {
+new File(PIG_FILE).delete
svn commit: r1069507 [1/2] - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/parser/ src/org/apache/pig/scripting/ test/org/apache/pig/test/
Author: rding
Date: Thu Feb 10 18:10:10 2011
New Revision: 1069507
URL: http://svn.apache.org/viewvc?rev=1069507view=rev
Log:
PIG-1793: Add macro expansion to Pig Latin
Added:
pig/trunk/src/org/apache/pig/parser/AliasMasker.g
pig/trunk/src/org/apache/pig/parser/MacroExpansion.g
pig/trunk/src/org/apache/pig/parser/MacroImport.g
pig/trunk/src/org/apache/pig/parser/ParserUtil.java
pig/trunk/src/org/apache/pig/parser/PigMacro.java
pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
Modified:
pig/trunk/CHANGES.txt
pig/trunk/build.xml
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/parser/QueryParser.g
pig/trunk/src/org/apache/pig/scripting/Pig.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1069507r1=1069506r2=1069507view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu Feb 10 18:10:10 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1793: Add macro expansion to Pig Latin (rding)
+
PIG-847: Setting twoLevelAccessRequired field in a bag schema should not be
required to access fields in the tuples of the bag (daijy)
PIG-1748: Add load/store function AvroStorage for avro data (guolin2001,
jghoman via daijy)
Modified: pig/trunk/build.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1069507r1=1069506r2=1069507view=diff
==
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Thu Feb 10 18:10:10 2011
@@ -326,7 +326,7 @@
description=generates token parser class from an ANTLR grammar
java classname=org.antlr.Tool
classpathref=classpath fork=true
- arg line=-o ${src.gen.dir}/${grammar.package.dir}
${src.dir}/${grammar.package.dir}/${grammar.name}Parser.g/
+ arg line=-o ${src.gen.dir}/${grammar.package.dir}
${src.dir}/${grammar.package.dir}/MacroExpansion.g
${src.dir}/${grammar.package.dir}/MacroImport.g
${src.dir}/${grammar.package.dir}/${grammar.name}Parser.g/
/java
/target
@@ -335,7 +335,7 @@
description=generates tree parser class from an ANTLR grammar
java classname=org.antlr.Tool
classpathref=classpath fork=true
- arg line=-o ${src.gen.dir}/${grammar.package.dir}
${src.dir}/${grammar.package.dir}/AstValidator.g
${src.dir}/${grammar.package.dir}/LogicalPlanGenerator.g/
+ arg line=-o ${src.gen.dir}/${grammar.package.dir}
${src.dir}/${grammar.package.dir}/AliasMasker.g
${src.dir}/${grammar.package.dir}/AstValidator.g
${src.dir}/${grammar.package.dir}/LogicalPlanGenerator.g/
/java
/target
Modified: pig/trunk/src/org/apache/pig/Main.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1069507r1=1069506r2=1069507view=diff
==
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Thu Feb 10 18:10:10 2011
@@ -67,6 +67,7 @@ import org.apache.pig.impl.util.LogUtils
import org.apache.pig.impl.util.ObjectSerializer;
import org.apache.pig.impl.util.PropertiesUtil;
import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.parser.ParserUtil;
import org.apache.pig.scripting.ScriptEngine;
import org.apache.pig.tools.cmdline.CmdLineParser;
import org.apache.pig.tools.grunt.Grunt;
@@ -380,9 +381,11 @@ static int run(String args[], PigProgres
.getPath(), type.name().toLowerCase());
}
}
-
-in = new BufferedReader(new FileReader(localFileRet.file));
-
+
+// run macro expansion
+FileReader fr = new FileReader(localFileRet.file);
+in = ParserUtil.getExpandedMacroAsBufferedReader(fr);
+
// run parameter substitution preprocessor first
substFile = file + .substituted;
pin = runParamPreprocessor(properties, in, params, paramFiles,
substFile, debug || dryrun || checkScriptOnly);
@@ -438,7 +441,10 @@ static int run(String args[], PigProgres
scriptState.setScript(sb.toString());
-in = new BufferedReader(new StringReader(sb.toString()));
+// run macro expansion
+StringReader sr = new StringReader(sb.toString());
+in = ParserUtil.getExpandedMacroAsBufferedReader(sr);
+
grunt = new Grunt(in, pigContext);
gruntCalled = true;
int results[] = grunt.exec();
@@ -495,8 +501,10 @@ static int run(String args[], PigProgres
.getPath(), type.name().toLowerCase
svn commit: r1069507 [2/2] - in /pig/trunk: ./ src/org/apache/pig/ src/org/apache/pig/parser/ src/org/apache/pig/scripting/ test/org/apache/pig/test/
Added: pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java?rev=1069507view=auto
==
--- pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java (added)
+++ pig/trunk/test/org/apache/pig/test/TestMacroExpansion.java Thu Feb 10
18:10:10 2011
@@ -0,0 +1,1075 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * License); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.pig.test;
+
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileWriter;
+import java.io.StringReader;
+import java.util.Properties;
+
+import junit.framework.Assert;
+
+import org.apache.pig.ExecType;
+import org.apache.pig.PigRunner;
+import org.apache.pig.impl.PigContext;
+import org.apache.pig.parser.ParserUtil;
+import org.apache.pig.tools.grunt.Grunt;
+import org.apache.pig.tools.pigstats.PigStats;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+public class TestMacroExpansion {
+
+private static final MiniCluster cluster = MiniCluster.buildCluster();
+
+@BeforeClass
+public static void setUpBeforeClass() throws Exception {
+}
+
+@AfterClass
+public static void tearDownAfterClass() throws Exception {
+cluster.shutDown();
+}
+
+@Before
+public void setUp() throws Exception {
+}
+
+@After
+public void tearDown() throws Exception {
+}
+
+@Test
+public void firstTest() throws Throwable {
+String macro = define group_and_count (A,group_key) returns B {\n +
+D = group $A by $group_key partition by
org.apache.pig.test.utils.SimpleCustomPartitioner parallel 50;\n +
+$B = foreach D generate group, COUNT($A);\n +
+};\n;
+
+String script =
+alpha = load 'users' as (user, age, zip);\n +
+gamma = group_and_count (alpha, user);\n +
+delta = group_and_count (alpha, age);\n +
+store gamma into 'byuser';\n +
+store delta into 'byage';\n;
+
+StringReader rd = new StringReader(macro + script);
+String s = ParserUtil.expandMacros(rd);
+
+validate(s);
+
+String expected =
+\nalpha = load 'users' as (user, age, zip);\n +
+macro_group_and_count_D_0 = group alpha by (user) partition BY
org.apache.pig.test.utils.SimpleCustomPartitioner parallel 50;\n +
+gamma = foreach macro_group_and_count_D_0 generate group,
COUNT(alpha);\n\n +
+macro_group_and_count_D_1 = group alpha by (age) partition BY
org.apache.pig.test.utils.SimpleCustomPartitioner parallel 50;\n +
+delta = foreach macro_group_and_count_D_1 generate group,
COUNT(alpha);\n\n +
+store gamma into 'byuser';\n +
+store delta into 'byage';\n;
+
+Assert.assertEquals(expected, s);
+}
+
+@Test
+public void distinctTest() throws Throwable {
+String macro = define group_and_count (A,group_key, reducers) returns
B {\n +
+$B = distinct $A partition by
org.apache.pig.test.utils.SimpleCustomPartitioner parallel $reducers;\n +
+};\n;
+
+String script =
+alpha = load 'users' as (user, age, zip);\n +
+gamma = group_and_count (alpha, user, 23);\n +
+delta = group_and_count (alpha, age, 32);\n +
+store gamma into 'byuser';\n +
+store delta into 'byage';\n;
+
+StringReader rd = new StringReader(macro + script);
+String s = ParserUtil.expandMacros(rd);
+
+validate(s);
+
+String expected =
+\nalpha = load 'users' as (user, age, zip);\n +
+gamma = distinct alpha partition BY
org.apache.pig.test.utils.SimpleCustomPartitioner parallel 23;\n\n +
+delta = distinct alpha partition BY
org.apache.pig.test.utils.SimpleCustomPartitioner parallel 32;\n\n +
+store gamma into 'byuser';\n +
+
svn commit: r1062074 - in /pig/branches/branch-0.8: CHANGES.txt build.xml
Author: rding
Date: Sat Jan 22 00:12:06 2011
New Revision: 1062074
URL: http://svn.apache.org/viewvc?rev=1062074view=rev
Log:
PIG-1800: Missing Signature for maven staging release
Modified:
pig/branches/branch-0.8/CHANGES.txt
pig/branches/branch-0.8/build.xml
Modified: pig/branches/branch-0.8/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.8/CHANGES.txt?rev=1062074r1=1062073r2=1062074view=diff
==
--- pig/branches/branch-0.8/CHANGES.txt (original)
+++ pig/branches/branch-0.8/CHANGES.txt Sat Jan 22 00:12:06 2011
@@ -213,6 +213,8 @@ PIG-1309: Map-side Cogroup (ashutoshc)
BUG FIXES
+PIG-1800: Missing Signature for maven staging release (rding)
+
PIG-1815: pig task retains used instances of PhysicalPlan (thejas)
PIG-1785: New logical plan: uid conflict in flattened fields (daijy)
Modified: pig/branches/branch-0.8/build.xml
URL:
http://svn.apache.org/viewvc/pig/branches/branch-0.8/build.xml?rev=1062074r1=1062073r2=1062074view=diff
==
--- pig/branches/branch-0.8/build.xml (original)
+++ pig/branches/branch-0.8/build.xml Sat Jan 22 00:12:06 2011
@@ -836,10 +836,10 @@
pom refid=pig/
attach file=${pig-core.jar}.asc type=jar.asc/
attach file=${pig.pom}.asc type=pom.asc/
- attach file=${output.jarfile.sources}.asc type=jar.asc /
- attach file=${output.jarfile.sources} classifier=sources /
- attach file=${output.jarfile.javadoc}.asc type=jar.asc /
- attach file=${output.jarfile.javadoc} classifier=javadoc /
+ attach file=${output.jarfile.sources}.asc type=jar.asc
classifier=sources/
+ attach file=${output.jarfile.sources} classifier=sources /
+ attach file=${output.jarfile.javadoc}.asc type=jar.asc
classifier=javadoc/
+ attach file=${output.jarfile.javadoc} classifier=javadoc /
/artifact:deploy
/target
svn commit: r1060894 - in /pig/trunk: CHANGES.txt src/org/apache/pig/scripting/Pig.java src/org/apache/pig/scripting/ScriptEngine.java src/org/apache/pig/scripting/jython/JythonScriptEngine.java test/
Author: rding
Date: Wed Jan 19 18:28:40 2011
New Revision: 1060894
URL: http://svn.apache.org/viewvc?rev=1060894view=rev
Log:
PIG-1806: Modify embedded Pig API for usability
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/scripting/Pig.java
pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java
pig/trunk/src/org/apache/pig/scripting/jython/JythonScriptEngine.java
pig/trunk/test/org/apache/pig/test/TestScriptLanguage.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1060894r1=1060893r2=1060894view=diff
==
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Wed Jan 19 18:28:40 2011
@@ -24,6 +24,8 @@ INCOMPATIBLE CHANGES
IMPROVEMENTS
+PIG-1806: Modify embedded Pig API for usability (rding)
+
PIG-1799: Provide deployable maven artifacts for pigunit and pig smoke tests
(cos via gates)
Modified: pig/trunk/src/org/apache/pig/scripting/Pig.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/Pig.java?rev=1060894r1=1060893r2=1060894view=diff
==
--- pig/trunk/src/org/apache/pig/scripting/Pig.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/Pig.java Wed Jan 19 18:28:40 2011
@@ -53,19 +53,21 @@ public class Pig {
* string.
* @throws IOException
*/
-public static void fs(String cmd) throws IOException {
+public static int fs(String cmd) throws IOException {
ScriptPigContext ctx = getScriptContext();
FsShell shell = new FsShell(ConfigurationUtil.toConfiguration(ctx
.getPigContext().getProperties()));
+int code = -1;
if (cmd != null) {
String[] cmdTokens = cmd.split(\\s+);
if (!cmdTokens[0].startsWith(-)) cmdTokens[0] = - +
cmdTokens[0];
try {
-shell.run(cmdTokens);
+code = shell.run(cmdTokens);
} catch (Exception e) {
throw new IOException(Run filesystem command failed, e);
}
}
+return code;
}
/**
@@ -201,7 +203,7 @@ public class Pig {
* @throws IOException if there is not a key for each
* Pig Latin parameter or if they contain unsupported types.
*/
-public BoundScript bind(MapString, String vars) throws IOException {
+public BoundScript bind(MapString, Object vars) throws IOException {
return new BoundScript(replaceParameters(script, vars), scriptContext,
name);
}
@@ -217,9 +219,9 @@ public class Pig {
* @throws IOException if there is not a key for each
* Pig Latin parameter or if they contain unsupported types.
*/
-public BoundScript bind(ListMapString, String vars) throws IOException
{
+public BoundScript bind(ListMapString, Object vars) throws IOException
{
ListString lst = new ArrayListString();
-for (MapString, String var : vars) {
+for (MapString, Object var : vars) {
lst.add(replaceParameters(script, var));
}
return new BoundScript(lst, scriptContext, name);
@@ -242,7 +244,7 @@ public class Pig {
*/
public BoundScript bind() throws IOException {
ScriptEngine engine = scriptContext.getScriptEngine();
-MapString, String vars = engine.getParamsFromVariables();
+MapString, Object vars = engine.getParamsFromVariables();
return bind(vars);
}
@@ -266,11 +268,11 @@ public class Pig {
* @param vars parameters and their values
* @return the modified version
*/
-private String replaceParameters(String qstr, MapString, String vars)
+private String replaceParameters(String qstr, MapString, Object vars)
throws IOException {
ArrayListString plist = new ArrayListString();
-for (EntryString, String entry : vars.entrySet()) {
-plist.add(entry.getKey() + = + entry.getValue());
+for (EntryString, Object entry : vars.entrySet()) {
+plist.add(entry.getKey() + = + entry.getValue().toString());
}
ParameterSubstitutionPreprocessor psp =
Modified: pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java?rev=1060894r1=1060893r2=1060894view=diff
==
--- pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java (original)
+++ pig/trunk/src/org/apache/pig/scripting/ScriptEngine.java Wed Jan 19
18:28:40 2011
@@ -89,7 +89,7 @@ public abstract class ScriptEngine {
* Returns a map from local variable names to their values
* @throws IOException
*/
-protected abstract MapString, String getParamsFromVariables()
+protected abstract
svn commit: r1051285 - in /pig/trunk: build.xml ivy.xml
Author: rding
Date: Mon Dec 20 20:53:20 2010
New Revision: 1051285
URL: http://svn.apache.org/viewvc?rev=1051285view=rev
Log:
fix checkstyle build target
Modified:
pig/trunk/build.xml
pig/trunk/ivy.xml
Modified: pig/trunk/build.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/build.xml?rev=1051285r1=1051284r2=1051285view=diff
==
--- pig/trunk/build.xml (original)
+++ pig/trunk/build.xml Mon Dec 20 20:53:20 2010
@@ -977,8 +977,8 @@
!--target name=checkstyle depends=checkstyle.check,
set-checkstyle-classpath if=checkstyle.home --
target name=checkstyle depends=ivy-checkstyle description=Run
optional third-party tool targets
taskdef resource=checkstyletask.properties
- classpath refid=checkstyle.classpath/
- !--classpath refid=checkstyle-classpath/ --
+ !--classpath refid=checkstyle.classpath/ --
+ classpath refid=checkstyle-classpath/
/taskdef
mkdir dir=${test.build.dir}/
checkstyle config=${test.src.dir}/checkstyle.xml
failOnViolation=false
@@ -994,13 +994,19 @@
to Ant on the command-line. /
/target
+!--
target name=set-checkstyle-classpath
path id=checkstyle-classpath
fileset dir=${checkstyle.home}
include name=**/*.jar/
/fileset
/path
-/target
+/target
+--
+path id=checkstyle-classpath
+ path refid=checkstyle.classpath/
+ fileset file=${lib.dir}/${antlr.jar}/
+/path
target name=findbugs.check depends=check-for-findbugs
unless=findbugs.present
fail message='findbugs.home' is not defined. Please pass
-Dfindbugs.home=lt;base of Findbugs installationgt;
Modified: pig/trunk/ivy.xml
URL:
http://svn.apache.org/viewvc/pig/trunk/ivy.xml?rev=1051285r1=1051284r2=1051285view=diff
==
--- pig/trunk/ivy.xml (original)
+++ pig/trunk/ivy.xml Mon Dec 20 20:53:20 2010
@@ -44,8 +44,8 @@
dependencies
dependency org=checkstyle name=checkstyle rev=${checkstyle.version}
conf=checkstyle-master/
-dependency org=antlr name=antlr rev=${antlr.version}
- conf=checkstyle-master/
+!-- dependency org=antlr name=antlr rev=${antlr.version}
+ conf=checkstyle-master/ --
dependency org=commons-beanutils name=commons-beanutils-core
rev=${commons-beanutils.version}
conf=checkstyle-master/
dependency org=commons-el name=commons-el rev=${commons-el.version}
svn commit: r1034603 - in /pig/trunk: src/org/apache/pig/Main.java src/org/apache/pig/builtin/PigStorage.java test/org/apache/pig/test/TestBZip.java
Author: rding
Date: Sat Nov 13 00:00:29 2010
New Revision: 1034603
URL: http://svn.apache.org/viewvc?rev=1034603view=rev
Log:
PIG-1714: Option mapred.output.compress doesn't work in Pig 0.8 but worked in
0.7
Modified:
pig/trunk/src/org/apache/pig/Main.java
pig/trunk/src/org/apache/pig/builtin/PigStorage.java
pig/trunk/test/org/apache/pig/test/TestBZip.java
Modified: pig/trunk/src/org/apache/pig/Main.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/Main.java?rev=1034603r1=1034602r2=1034603view=diff
==
--- pig/trunk/src/org/apache/pig/Main.java (original)
+++ pig/trunk/src/org/apache/pig/Main.java Sat Nov 13 00:00:29 2010
@@ -179,6 +179,16 @@ static int run(String args[], PigProgres
properties.setProperty(stop.on.failure, +false);
}
+if( true.equals( properties.getProperty( mapred.output.compress )
) ) {
+properties.setProperty( output.compression.enabled, true );
+String codec = properties.getProperty(
mapred.output.compression.codec );
+if( codec == null ) {
+throw new RuntimeException( 'mapred.output.compress' is set
but no value is specified for 'mapred.output.compression.codec'. );
+} else {
+properties.setProperty( output.compression.codec, codec );
+}
+}
+
// set up client side system properties in UDF context
UDFContext.getUDFContext().setClientSystemProps();
Modified: pig/trunk/src/org/apache/pig/builtin/PigStorage.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=1034603r1=1034602r2=1034603view=diff
==
--- pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original)
+++ pig/trunk/src/org/apache/pig/builtin/PigStorage.java Sat Nov 13 00:00:29
2010
@@ -28,6 +28,7 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.compress.BZip2Codec;
+import org.apache.hadoop.io.compress.CompressionCodec;
import org.apache.hadoop.io.compress.GzipCodec;
import org.apache.hadoop.mapreduce.InputFormat;
import org.apache.hadoop.mapreduce.Job;
@@ -232,14 +233,24 @@ LoadPushDown {
public void setStoreLocation(String location, Job job) throws IOException {
job.getConfiguration().set(mapred.textoutputformat.separator, );
FileOutputFormat.setOutputPath(job, new Path(location));
-if (location.endsWith(.bz2) || location.endsWith(.bz)) {
-FileOutputFormat.setCompressOutput(job, true);
-FileOutputFormat.setOutputCompressorClass(job, BZip2Codec.class);
-} else if (location.endsWith(.gz)) {
-FileOutputFormat.setCompressOutput(job, true);
-FileOutputFormat.setOutputCompressorClass(job, GzipCodec.class);
+if( true.equals( job.getConfiguration().get(
output.compression.enabled ) ) ) {
+FileOutputFormat.setCompressOutput( job, true );
+String codec = job.getConfiguration().get(
output.compression.codec );
+try {
+FileOutputFormat.setOutputCompressorClass( job, (Class?
extends CompressionCodec) Class.forName( codec ) );
+} catch (ClassNotFoundException e) {
+throw new RuntimeException(Class not found: + codec );
+}
} else {
-FileOutputFormat.setCompressOutput(job, false);
+if (location.endsWith(.bz2) || location.endsWith(.bz)) {
+FileOutputFormat.setCompressOutput(job, true);
+FileOutputFormat.setOutputCompressorClass(job,
BZip2Codec.class);
+} else if (location.endsWith(.gz)) {
+FileOutputFormat.setCompressOutput(job, true);
+FileOutputFormat.setOutputCompressorClass(job,
GzipCodec.class);
+} else {
+FileOutputFormat.setCompressOutput( job, false);
+}
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestBZip.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestBZip.java?rev=1034603r1=1034602r2=1034603view=diff
==
--- pig/trunk/test/org/apache/pig/test/TestBZip.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestBZip.java Sat Nov 13 00:00:29 2010
@@ -440,4 +440,35 @@ public class TestBZip {
stat = fs.getFileStatus(new Path(output.bz2/part-m-0.bz2));
assertTrue(stat.getLen() 0);
}
+
+@Test
+public void testBzipStoreInMultiQuery2() throws Exception {
+String[] inputData = new String[] {
+1\t2\r3\t4
+};
+
+String inputFileName = input2.txt;
+Util.createInputFile(cluster
svn commit: r1034605 - /pig/trunk/CHANGES.txt
Author: rding Date: Sat Nov 13 00:01:42 2010 New Revision: 1034605 URL: http://svn.apache.org/viewvc?rev=1034605view=rev Log: PIG-1714: Option mapred.output.compress doesn't work in Pig 0.8 but worked in 0.7 Modified: pig/trunk/CHANGES.txt Modified: pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1034605r1=1034604r2=1034605view=diff == --- pig/trunk/CHANGES.txt (original) +++ pig/trunk/CHANGES.txt Sat Nov 13 00:01:42 2010 @@ -220,6 +220,9 @@ PIG-1309: Map-side Cogroup (ashutoshc) BUG FIXES +PIG-1714: Option mapred.output.compress doesn't work in Pig 0.8 but worked in +0.7 (xuefuz via rding) + PIG-1715: pig-withouthadoop.jar missing automaton.jar (thejas) PIG-1706: New logical plan: PushDownFlattenForEach fail if flattened field has user defined schema (daijy)
svn commit: r1005953 - /pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
Author: rding
Date: Fri Oct 8 18:44:37 2010
New Revision: 1005953
URL: http://svn.apache.org/viewvc?rev=1005953view=rev
Log:
Increase size limit of script stored to job xml to 10k
Modified:
pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
Modified: pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java?rev=1005953r1=1005952r2=1005953view=diff
==
--- pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java (original)
+++ pig/trunk/src/org/apache/pig/tools/pigstats/ScriptState.java Fri Oct 8
18:44:37 2010
@@ -161,7 +161,7 @@ public class ScriptState {
/**
* Restricts the size of Pig script stored in job xml
*/
-public static final int MAX_SCRIPT_SIZE = 1024;
+public static final int MAX_SCRIPT_SIZE = 10240;
private static final Log LOG = LogFactory.getLog(ScriptState.class);
