Author: knoguchi
Date: Thu May 25 15:11:07 2017
New Revision: 1796154
URL: http://svn.apache.org/viewvc?rev=1796154&view=rev
Log:
PIG-5235: Typecast with as-clause fails for tuple/bag with an empty schema
(knoguchi)
Modified:
pig/trunk/CHANGES.txt
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java
pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
Modified: pig/trunk/CHANGES.txt
URL:
http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1796154&r1=1796153&r2=1796154&view=diff
==============================================================================
--- pig/trunk/CHANGES.txt (original)
+++ pig/trunk/CHANGES.txt Thu May 25 15:11:07 2017
@@ -101,6 +101,8 @@ OPTIMIZATIONS
Â
BUG FIXES
+PIG-5235: Typecast with as-clause fails for tuple/bag with an empty schema
(knoguchi)
+
PIG-5238: Fix datetime related test issues after PIG-4748 (szita)
PIG-5185: Job name show "DefaultJobName" when running a Python script (daijy)
Modified:
pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java?rev=1796154&r1=1796153&r2=1796154&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java
(original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java Thu
May 25 15:11:07 2017
@@ -168,10 +168,8 @@ public class LOGenerate extends LogicalR
mergedSchema.mergeUid(expSchema);
}
+ setNullTypeToByteArrayType(mergedSchema);
for (LogicalFieldSchema fs : mergedSchema.getFields()) {
- if (fs.type==DataType.NULL) {
- fs.type = DataType.BYTEARRAY;
- }
planSchema.addField(fs);
}
} else {
@@ -323,4 +321,16 @@ public class LOGenerate extends LogicalR
public List<LogicalSchema> getExpSchemas() {
return expSchemas;
}
+
+ private void setNullTypeToByteArrayType (LogicalSchema s1) {
+ if( s1 != null ) {
+ for (LogicalFieldSchema fs : s1.getFields()) {
+ if( DataType.isSchemaType(fs.type) ) {
+ setNullTypeToByteArrayType(fs.schema);
+ } else if(fs.type == DataType.NULL) {
+ fs.type = DataType.BYTEARRAY;
+ }
+ }
+ }
+ }
}
Modified:
pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java?rev=1796154&r1=1796153&r2=1796154&view=diff
==============================================================================
--- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
(original)
+++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java
Thu May 25 15:11:07 2017
@@ -168,7 +168,7 @@ public class LogicalSchema {
if (s1==null && s2==null) {
return true;
}
- if (fs1==null || fs2==null) {
+ if (s1==null || s2==null) {
return false;
}
if (s1.size()!=s2.size()) {
Modified:
pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java
URL:
http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java?rev=1796154&r1=1796153&r2=1796154&view=diff
==============================================================================
---
pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java
(original)
+++
pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java
Thu May 25 15:11:07 2017
@@ -217,6 +217,10 @@ public class ForEachUserSchemaVisitor ex
for( LogicalSchema mUserDefinedSchema : mUserDefinedSchemas ) {
resetTypeToNull( mUserDefinedSchema );
}
+
+ // Given mUserDefinedSchema was changed, we should drop the cached
schema
+ foreach.resetSchema();
+ generate.resetSchema();
}
}
Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1796154&r1=1796153&r2=1796154&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Thu May 25
15:11:07 2017
@@ -535,4 +535,45 @@ public class TestPlanGeneration {
Assert.assertEquals(expectedRes[i], list.get(i).toString());
}
}
+
+ @Test
+ public void testAsWithTypeTuple() throws Exception {
+ Data data = Storage.resetData(ps);
+ data.set("input_testAsWithTypeTuple", "t1:tuple(), f2:chararray",
+ tuple(tuple(1.1,1.2), "a"), tuple(tuple(2.8, 3.1), "b"));
+
+ String query =
+ "A = load 'input_testAsWithTypeTuple' USING mock.Storage(); \n"
+ + "B = FOREACH A GENERATE t1 as (newt1:tuple(f1_1:int,
f1_2:double));\n"
+ + "store B into 'out' using mock.Storage;" ;
+
+ Util.registerMultiLineQuery(ps, query);
+ List<Tuple> list = data.get("out");
+ // making sure typecast from double to int worked.
+ // first field is 1 instead of 1.1, and 2 instead of 2.8.
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {"((1,1.2))", "((2,3.1))"});
+ Util.checkQueryOutputsAfterSort(list, expectedRes);
+ }
+
+ @Test
+ public void testAsWithTypeBag() throws Exception {
+ Data data = Storage.resetData(ps);
+ data.set("input_testAsWithTypeBag", "b1:bag{}, f2:chararray",
+ tuple(bag(tuple(1.1),tuple(1.2)), "a"), tuple(bag(tuple(2.8),
tuple(3.1)), "b"));
+
+ String query =
+ "A = load 'input_testAsWithTypeBag' USING mock.Storage(); \n"
+ + "B = FOREACH A GENERATE b1 as (newbag:bag{tuple(f1:int)}); \n"
+ + "store B into 'out' using mock.Storage;" ;
+
+ Util.registerMultiLineQuery(ps, query);
+ List<Tuple> list = data.get("out");
+ // making sure typecast from double to int worked.
+ List<Tuple> expectedRes =
+ Util.getTuplesFromConstantTupleStrings(
+ new String[] {"({(1),(1)})","({(2),(3)})"});
+ Util.checkQueryOutputsAfterSort(list, expectedRes);
+ }
}
Modified: pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java
URL:
http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java?rev=1796154&r1=1796153&r2=1796154&view=diff
==============================================================================
--- pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java (original)
+++ pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java Thu May
25 15:11:07 2017
@@ -301,7 +301,7 @@ public class TestProjectStarRangeInUdf
;
Schema sch = Utils.getSchemaFromString("tt : {(NullALias)}");
sch.getField(0).schema.getField(0).schema.getField(0).alias = null;
- sch.getField(0).schema.getField(0).schema.getField(0).type =
DataType.NULL;
+ sch.getField(0).schema.getField(0).schema.getField(0).type =
DataType.BYTEARRAY;
compileAndCompareSchema(sch, query, "f");
Iterator<Tuple> it = pigServer.openIterator("f");
