Author: knoguchi Date: Thu May 25 15:11:07 2017 New Revision: 1796154 URL: http://svn.apache.org/viewvc?rev=1796154&view=rev Log: PIG-5235: Typecast with as-clause fails for tuple/bag with an empty schema (knoguchi)
Modified: pig/trunk/CHANGES.txt pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java Modified: pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/pig/trunk/CHANGES.txt?rev=1796154&r1=1796153&r2=1796154&view=diff ============================================================================== --- pig/trunk/CHANGES.txt (original) +++ pig/trunk/CHANGES.txt Thu May 25 15:11:07 2017 @@ -101,6 +101,8 @@ OPTIMIZATIONS Â BUG FIXES +PIG-5235: Typecast with as-clause fails for tuple/bag with an empty schema (knoguchi) + PIG-5238: Fix datetime related test issues after PIG-4748 (szita) PIG-5185: Job name show "DefaultJobName" when running a Python script (daijy) Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java?rev=1796154&r1=1796153&r2=1796154&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java (original) +++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LOGenerate.java Thu May 25 15:11:07 2017 @@ -168,10 +168,8 @@ public class LOGenerate extends LogicalR mergedSchema.mergeUid(expSchema); } + setNullTypeToByteArrayType(mergedSchema); for (LogicalFieldSchema fs : mergedSchema.getFields()) { - if (fs.type==DataType.NULL) { - fs.type = DataType.BYTEARRAY; - } planSchema.addField(fs); } } else { @@ -323,4 +321,16 @@ public class LOGenerate extends LogicalR public List<LogicalSchema> getExpSchemas() { return expSchemas; } + + private void setNullTypeToByteArrayType (LogicalSchema s1) { + if( s1 != null ) { + for (LogicalFieldSchema fs : s1.getFields()) { + if( DataType.isSchemaType(fs.type) ) { + setNullTypeToByteArrayType(fs.schema); + } else if(fs.type == DataType.NULL) { + fs.type = DataType.BYTEARRAY; + } + } + } + } } Modified: pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java?rev=1796154&r1=1796153&r2=1796154&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java (original) +++ pig/trunk/src/org/apache/pig/newplan/logical/relational/LogicalSchema.java Thu May 25 15:11:07 2017 @@ -168,7 +168,7 @@ public class LogicalSchema { if (s1==null && s2==null) { return true; } - if (fs1==null || fs2==null) { + if (s1==null || s2==null) { return false; } if (s1.size()!=s2.size()) { Modified: pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java URL: http://svn.apache.org/viewvc/pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java?rev=1796154&r1=1796153&r2=1796154&view=diff ============================================================================== --- pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java (original) +++ pig/trunk/src/org/apache/pig/newplan/logical/visitor/ForEachUserSchemaVisitor.java Thu May 25 15:11:07 2017 @@ -217,6 +217,10 @@ public class ForEachUserSchemaVisitor ex for( LogicalSchema mUserDefinedSchema : mUserDefinedSchemas ) { resetTypeToNull( mUserDefinedSchema ); } + + // Given mUserDefinedSchema was changed, we should drop the cached schema + foreach.resetSchema(); + generate.resetSchema(); } } Modified: pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java?rev=1796154&r1=1796153&r2=1796154&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java (original) +++ pig/trunk/test/org/apache/pig/test/TestPlanGeneration.java Thu May 25 15:11:07 2017 @@ -535,4 +535,45 @@ public class TestPlanGeneration { Assert.assertEquals(expectedRes[i], list.get(i).toString()); } } + + @Test + public void testAsWithTypeTuple() throws Exception { + Data data = Storage.resetData(ps); + data.set("input_testAsWithTypeTuple", "t1:tuple(), f2:chararray", + tuple(tuple(1.1,1.2), "a"), tuple(tuple(2.8, 3.1), "b")); + + String query = + "A = load 'input_testAsWithTypeTuple' USING mock.Storage(); \n" + + "B = FOREACH A GENERATE t1 as (newt1:tuple(f1_1:int, f1_2:double));\n" + + "store B into 'out' using mock.Storage;" ; + + Util.registerMultiLineQuery(ps, query); + List<Tuple> list = data.get("out"); + // making sure typecast from double to int worked. + // first field is 1 instead of 1.1, and 2 instead of 2.8. + List<Tuple> expectedRes = + Util.getTuplesFromConstantTupleStrings( + new String[] {"((1,1.2))", "((2,3.1))"}); + Util.checkQueryOutputsAfterSort(list, expectedRes); + } + + @Test + public void testAsWithTypeBag() throws Exception { + Data data = Storage.resetData(ps); + data.set("input_testAsWithTypeBag", "b1:bag{}, f2:chararray", + tuple(bag(tuple(1.1),tuple(1.2)), "a"), tuple(bag(tuple(2.8), tuple(3.1)), "b")); + + String query = + "A = load 'input_testAsWithTypeBag' USING mock.Storage(); \n" + + "B = FOREACH A GENERATE b1 as (newbag:bag{tuple(f1:int)}); \n" + + "store B into 'out' using mock.Storage;" ; + + Util.registerMultiLineQuery(ps, query); + List<Tuple> list = data.get("out"); + // making sure typecast from double to int worked. + List<Tuple> expectedRes = + Util.getTuplesFromConstantTupleStrings( + new String[] {"({(1),(1)})","({(2),(3)})"}); + Util.checkQueryOutputsAfterSort(list, expectedRes); + } } Modified: pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java URL: http://svn.apache.org/viewvc/pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java?rev=1796154&r1=1796153&r2=1796154&view=diff ============================================================================== --- pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java (original) +++ pig/trunk/test/org/apache/pig/test/TestProjectStarRangeInUdf.java Thu May 25 15:11:07 2017 @@ -301,7 +301,7 @@ public class TestProjectStarRangeInUdf ; Schema sch = Utils.getSchemaFromString("tt : {(NullALias)}"); sch.getField(0).schema.getField(0).schema.getField(0).alias = null; - sch.getField(0).schema.getField(0).schema.getField(0).type = DataType.NULL; + sch.getField(0).schema.getField(0).schema.getField(0).type = DataType.BYTEARRAY; compileAndCompareSchema(sch, query, "f"); Iterator<Tuple> it = pigServer.openIterator("f");