Author: rohini
Date: Mon Oct 26 18:35:32 2015
New Revision: 1710672
URL: http://svn.apache.org/viewvc?rev=1710672&view=rev
Log:
PIG-4712: [Pig on Tez] NPE in Bloom UDF after Union (rohini)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
pig/trunk/test/e2e/pig/tests/nightly.conf
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1710672&r1=1710671&r2=1710672&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Mon Oct 26 18:35:32 2015
@@ -57,6 +57,8 @@ PIG-4639: Add better parser for Apache H
BUG FIXES
+PIG-4712: [Pig on Tez] NPE in Bloom UDF after Union (rohini)
+
PIG-4707: [Pig on Tez] Streaming job hangs with pig.exec.mapPartAgg=true
(rohini)
PIG-4703: TezOperator.stores shall not ship to backend (daijy)
Modified:
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java?rev=1710672&r1=1710671&r2=1710672&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
(original)
+++
pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/expressionOperators/POUserFunc.java
Mon Oct 26 18:35:32 2015
@@ -566,6 +566,8 @@ public class POUserFunc extends Expressi
requestedParallelism, null, funcSpec.clone());
clone.setResultType(resultType);
clone.signature = signature;
+ clone.cacheFiles = cacheFiles;
+ clone.shipFiles = shipFiles;
return clone;
}
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL:
http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1710672&r1=1710671&r2=1710672&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Mon Oct 26 18:35:32 2015
@@ -5148,6 +5148,30 @@ store C into ':OUTPATH:';\,
C = load ':INPATH:/singlefile/votertab10k'as
(name:chararray, age:int, reg:chararray, contrib:float);
D = join C by name, B by name;
store D into ':OUTPATH:';",
+ },{
+ 'num' => 4,
+ 'pig' => "set pig.optimizer.rules.disabled
PushUpFilter;
+ define bb BuildBloom('Hash.JENKINS_HASH',
'fixed', '128', '3');
+ A = LOAD ':INPATH:/singlefile/studenttab10k'
AS (name:chararray, age:int, gpa:double);
+ B = filter A by name == 'alice allen';
+ C = group B all;
+ D = foreach C generate bb(B.name);
+ store D into ':HDFSTMP:/mybloom_4';
+ exec;
+ define bloom Bloom(':HDFSTMP:/mybloom_4');
+ E = LOAD ':INPATH:/singlefile/studenttab10k'
AS (name:chararray, age:int, gpa:double);
+ F = LOAD ':INPATH:/singlefile/studenttab10k'
AS (name:chararray, age:int, gpa:double);
+ G = union E, F;
+ -- PushUpFilter is disabled to avoid filter
being pushed before union
+ H = filter G by bloom(name);
+ store H into ':OUTPATH:';",
+ 'notmq' => 1,
+ 'verify_pig_script' => "
+ A = LOAD ':INPATH:/singlefile/studenttab10k'
AS (name, age:int ,gpa:double);
+ B = LOAD ':INPATH:/singlefile/studenttab10k'
AS (name, age:int ,gpa:double);
+ C = UNION A,B;
+ D = filter C by name == 'alice allen';
+ store D into ':OUTPATH:';",
}
],
},{
