Author: daijy
Date: Tue May 19 23:49:03 2015
New Revision: 1680429
URL: http://svn.apache.org/r1680429
Log:
PIG-4559: Fix several new tez e2e test failures
Modified:
pig/trunk/CHANGES.txt
pig/trunk/test/e2e/pig/tests/nightly.conf
pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/DummyContextUDF.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1680429&r1=1680428&r2=1680429&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Tue May 19 23:49:03 2015
@@ -76,6 +76,8 @@ PIG-4333: Split BigData tests into multi
BUG FIXES
+PIG-4559: Fix several new tez e2e test failures (daijy)
+
PIG-4506: binstorage fails to write biginteger (ssavvides via daijy)
PIG-4556: Local mode is broken in some case by PIG-4247 (daijy)
Modified: pig/trunk/test/e2e/pig/tests/nightly.conf
URL:
http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/tests/nightly.conf?rev=1680429&r1=1680428&r2=1680429&view=diff
==============================================================================
--- pig/trunk/test/e2e/pig/tests/nightly.conf (original)
+++ pig/trunk/test/e2e/pig/tests/nightly.conf Tue May 19 23:49:03 2015
@@ -1515,7 +1515,8 @@ c = union a, b;
d = group c by name;
e = load ':INPATH:/singlefile/votertab10k' as (name, age, registration,
contributions);
f = join d by group, e by name using 'replicated';
-store f into ':OUTPATH:';\,
+g = foreach f generate group, flatten(c), name, age, registration,
contributions;
+store g into ':OUTPATH:';\,
},
{
# Group by with Secondary Key + Union
@@ -1524,14 +1525,14 @@ store f into ':OUTPATH:';\,
b = load ':INPATH:/singlefile/studentcolon10k' using PigStorage(':') as
(name:chararray, age, gpa);
c = group a by name;
d = foreach c {
- sorted = order a by name,age;
+ sorted = order a by name,age,gpa;
lmt = limit sorted 1;
generate lmt as c1;
};
e = foreach d generate flatten(c1) as (name:chararray, age, gpa);
f = group b by name;
g = foreach f {
- sorted = order b by name,age;
+ sorted = order b by name,age,gpa;
lmt = limit sorted 1;
generate lmt as f1;
};
@@ -2203,12 +2204,12 @@ store D into ':OUTPATH:';\,
{
'num' => 9,
'pig' =>q\a = load
':INPATH:/singlefile/studentnulltab10k';
-b = order a by $0, $1;
+b = order a by $0, $1, $2;
c = limit b 1000/10;
store c into ':OUTPATH:';\,
'verify_pig_script' => q\a = load
':INPATH:/singlefile/studentnulltab10k';
-b = order a by $0, $1;
+b = order a by $0, $1, $2;
c = limit b 100;
store c into ':OUTPATH:';\,
@@ -4938,34 +4939,39 @@ store C into ':OUTPATH:';\,
{
# PIG-2286
'num' => 1,
+ 'floatpostprocess' => 1,
+ 'delimiter' => ' ',
'pig' => q?A = LOAD
':INPATH:/singlefile/studenttab10k' AS (name, age:double, gpa:double);
B = group A all;
- C = foreach B generate group, COR(A.age,
A.gpa);
+ C = foreach B generate group,
flatten(COR(A.age, A.gpa));
store C into ':OUTPATH:';?,
'verify_pig_script' => q?set pig.exec.nocombiner true
A = LOAD ':INPATH:/singlefile/studenttab10k'
AS (name, age:double ,gpa:double);
B = group A all;
- C = foreach B generate group, COR(A.age,
A.gpa);
+ C = foreach B generate group,
flatten(COR(A.age, A.gpa));
store C into ':OUTPATH:';?,
}, {
# PIG-2286, with 3 inputs to COR
'num' => 2,
+ 'floatpostprocess' => 1,
+ 'delimiter' => ' ',
'pig' => q?A = LOAD
':INPATH:/singlefile/studenttab10k' AS (name, age:double ,gpa:double);
B = foreach A generate age, gpa, gpa*gpa as
gpa2;
C = group B all;
- D = foreach C generate group, COR(B.age,
B.gpa, B.gpa2);
+ D = foreach C generate group,
flatten(COR(B.age, B.gpa, B.gpa2));
store D into ':OUTPATH:';?,
'verify_pig_script' => q?set pig.exec.nocombiner true
A = LOAD ':INPATH:/singlefile/studenttab10k'
AS (name, age:double ,gpa:double);
B = foreach A generate age, gpa, gpa*gpa as
gpa2;
C = group B all;
- D = foreach C generate group, COR(B.age,
B.gpa, B.gpa2);
+ D = foreach C generate group,
flatten(COR(B.age, B.gpa, B.gpa2));
store D into ':OUTPATH:';?,
}, {
# PIG-2385
'num' => 3,
'pig_params' => ['-M'],
'floatpostprocess' => 1,
+ 'delimiter' => ' ',
'pig' => q?A = LOAD
':INPATH:/singlefile/studenttab10k' AS (name:chararray, age:int,gpa:double);
Z = group A all;
Z1 = foreach Z generate AVG(A.gpa) as avg;
@@ -5740,15 +5746,10 @@ store a into ':OUTPATH:';\,
'pig' => q\set mapred.max.split.size '100000000'
register :FUNCPATH:/testudf.jar;
define DummyContextUDF
HiveUDF('org.apache.pig.test.udf.evalfunc.DummyContextUDF');
- A = load ':INPATH:/singlefile/studenttab10k' as
(name:chararray, age:long, gpa:double);
- B = foreach A generate DummyContextUDF();
+ A = load ':INPATH:/singlefile/studenttab10k' as
(name:chararray, age:int, gpa:double);
+ B = foreach A generate DummyContextUDF(age);
store B into ':OUTPATH:';\,
- 'verify_pig_script' => q\set mapred.max.split.size '100000000'
- A = LOAD ':INPATH:/singlefile/studenttab10k' as
(name:chararray, age:long, gpa:double);
- B = foreach A generate UniqueID();
- C = foreach B generate (int)SUBSTRING($0, 2, 100);
- D = foreach C generate (chararray)($0+1);
- store D into ':OUTPATH:';\
+ 'expected_err_regex' => "Encountered Warning UDF_WARNING_1
4610 time.*",
}
]
}
Modified:
pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/DummyContextUDF.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/DummyContextUDF.java?rev=1680429&r1=1680428&r2=1680429&view=diff
==============================================================================
---
pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/DummyContextUDF.java
(original)
+++
pig/trunk/test/e2e/pig/udfs/java/org/apache/pig/test/udf/evalfunc/DummyContextUDF.java
Tue May 19 23:49:03 2015
@@ -25,31 +25,29 @@ import org.apache.hadoop.hive.ql.metadat
import org.apache.hadoop.hive.ql.udf.generic.GenericUDF;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
import
org.apache.hadoop.hive.serde2.objectinspector.primitive.PrimitiveObjectInspectorFactory;
-import org.apache.hadoop.io.LongWritable;
-import org.apache.hadoop.mapred.Counters;
+import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapred.Reporter;
-import org.apache.pig.impl.util.UDFContext;
+import org.apache.pig.PigWarning;
@Description(name = "dummycontextudf",
value = "_FUNC_(col) - UDF to report MR counter values")
public class DummyContextUDF extends GenericUDF {
private MapredContext context;
- private LongWritable result = new LongWritable();
+ private Text result = new Text();
public ObjectInspector initialize(ObjectInspector[] arguments) throws
UDFArgumentException {
- return PrimitiveObjectInspectorFactory.writableLongObjectInspector;
+ return PrimitiveObjectInspectorFactory.writableStringObjectInspector;
}
public Object evaluate(DeferredObject[] arguments) throws HiveException {
Reporter reporter = context.getReporter();
- Counters.Counter counter;
- if (UDFContext.getUDFContext().getJobConf().get("exectype").equals("TEZ"))
{
- counter =
reporter.getCounter("org.apache.tez.common.counters.TaskCounter",
"INPUT_RECORDS_PROCESSED");
- } else {
- counter = reporter.getCounter("org.apache.hadoop.mapred.Task$Counter",
"MAP_INPUT_RECORDS");
+ int age = (Integer)arguments[0].get();
+ if (age>50) {
+ // Here use PigWarning.UDF_WARNING_1, so it is easier to verify in e2e
test
+ reporter.incrCounter(PigWarning.UDF_WARNING_1, 1);
}
- result.set(counter.getValue());
+ result.set(arguments.toString());
return result;
}
