Author: pradeepkth Date: Thu Mar 4 22:10:45 2010 New Revision: 919202 URL: http://svn.apache.org/viewvc?rev=919202&view=rev Log: PIG-1259: ResourceFieldSchema.setSchema should not allow a bag field without a Tuple as its only sub field (the tuple itself can have a schema with > 1 subfields) (pradeepkth)
Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=919202&r1=919201&r2=919202&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Thu Mar 4 22:10:45 2010 @@ -22,6 +22,10 @@ INCOMPATIBLE CHANGES +PIG-1259: ResourceFieldSchema.setSchema should not allow a bag field without a +Tuple as its only sub field (the tuple itself can have a schema with > 1 +subfields) (pradeepkth) + PIG-1265: Change LoadMetadata and StoreMetadata to use Job instead of Configuraiton and add a cleanupOnFailure method to StoreFuncInterface (pradeepkth) Modified: hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java?rev=919202&r1=919201&r2=919202&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/ResourceSchema.java Thu Mar 4 22:10:45 2010 @@ -18,6 +18,7 @@ package org.apache.pig; +import java.io.IOException; import java.io.Serializable; import java.util.Arrays; import java.util.List; @@ -25,6 +26,7 @@ import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; import org.apache.pig.data.DataType; +import org.apache.pig.impl.logicalLayer.FrontendException; import org.apache.pig.impl.logicalLayer.schema.Schema; import org.apache.pig.impl.logicalLayer.schema.Schema.FieldSchema; @@ -118,11 +120,35 @@ return schema; } - public ResourceFieldSchema setSchema(ResourceSchema schema) { + public ResourceFieldSchema setSchema(ResourceSchema schema) throws + IOException { + validateSchema(schema); this.schema = schema; return this; } + /** + * @param schema + */ + private void validateSchema(ResourceSchema schema) throws IOException { + if(type == DataType.BAG && schema != null) { + ResourceFieldSchema[] subFields = schema.getFields(); + if (subFields.length == 1) { + if (subFields[0].type != DataType.TUPLE) { + throwInvalidSchemaException(); + } + } else { + throwInvalidSchemaException(); + } + } + } + + public static void throwInvalidSchemaException() throws FrontendException { + int errCode = 2218; + throw new FrontendException("Invalid resource schema: " + + "bag schema must have tuple as its field", errCode, PigException.BUG); + } + @Override public String toString() { return getDescription(true); Modified: hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java?rev=919202&r1=919201&r2=919202&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/logicalLayer/schema/Schema.java Thu Mar 4 22:10:45 2010 @@ -1623,15 +1623,13 @@ if (fs.schema.size() == 1) { FieldSchema innerFs = fs.schema.getField(0); if (innerFs.type != DataType.TUPLE) { - throw new FrontendException("Invalide resource schema: " + - "bag schema must have tuple as its field."); + ResourceFieldSchema.throwInvalidSchemaException(); } if (innerFs.schema != null) { // allow partial schema fs.schema.setTwoLevelAccessRequired(true); } } else { - throw new FrontendException("Invalide resource schema: " + - "bag schema should have exact one field."); + ResourceFieldSchema.throwInvalidSchemaException(); } } } Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java?rev=919202&r1=919201&r2=919202&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestResourceSchema.java Thu Mar 4 22:10:45 2010 @@ -21,6 +21,7 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import java.io.IOException; import java.util.ArrayList; import java.util.List; @@ -124,9 +125,10 @@ * Test that Pig Schema is correctly created given a * ResourceSchema and vice versa. Test also that * TwoLevelAccess flag is set for Pig Schema when needed. + * @throws IOException */ @Test - public void testToPigSchemaWithTwoLevelAccess() throws FrontendException { + public void testToPigSchemaWithTwoLevelAccess() throws IOException { ResourceFieldSchema[] level0 = new ResourceFieldSchema[] { new ResourceFieldSchema() @@ -182,9 +184,10 @@ /** * Test invalid Resource Schema: multiple fields for a bag + * @throws IOException */ @Test(expected=FrontendException.class) - public void testToPigSchemaWithInvalidSchema() throws FrontendException { + public void testToPigSchemaWithInvalidSchema() throws IOException { ResourceFieldSchema[] level0 = new ResourceFieldSchema[] { new ResourceFieldSchema() .setName("fld0").setType(DataType.CHARARRAY), @@ -201,18 +204,14 @@ new ResourceFieldSchema() .setName("t2").setType(DataType.BAG).setSchema(rSchema0) }; - - ResourceSchema rSchema2 = new ResourceSchema() - .setFields(level2); - - Schema.getPigSchema(rSchema2); } /** * Test invalid Resource Schema: bag without tuple field + * @throws IOException */ @Test(expected=FrontendException.class) - public void testToPigSchemaWithInvalidSchema2() throws FrontendException { + public void testToPigSchemaWithInvalidSchema2() throws IOException { ResourceFieldSchema[] level0 = new ResourceFieldSchema[] { new ResourceFieldSchema() .setName("fld0").setType(DataType.CHARARRAY) @@ -225,11 +224,7 @@ new ResourceFieldSchema() .setName("t2").setType(DataType.BAG).setSchema(rSchema0) }; - - ResourceSchema rSchema2 = new ResourceSchema() - .setFields(level2); - - Schema.getPigSchema(rSchema2); + } /** Modified: hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java?rev=919202&r1=919201&r2=919202&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/TestTextDataParser.java Thu Mar 4 22:10:45 2010 @@ -17,6 +17,7 @@ */ package org.apache.pig.test; +import java.io.IOException; import java.util.Map; import org.junit.Test; @@ -43,7 +44,7 @@ PigStorage ps = new PigStorage(); - ResourceFieldSchema getTupleFieldSchema() { + ResourceFieldSchema getTupleFieldSchema() throws IOException { ResourceFieldSchema stringfs = new ResourceFieldSchema(); stringfs.setType(DataType.CHARARRAY); ResourceFieldSchema intfs = new ResourceFieldSchema(); @@ -58,7 +59,7 @@ return tuplefs; } - public ResourceFieldSchema getBagFieldSchema(){ + public ResourceFieldSchema getBagFieldSchema() throws IOException{ ResourceFieldSchema tuplefs = getTupleFieldSchema(); ResourceSchema outBagSchema = new ResourceSchema(); Modified: hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java?rev=919202&r1=919201&r2=919202&view=diff ============================================================================== --- hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java (original) +++ hadoop/pig/trunk/test/org/apache/pig/test/utils/GenRandomData.java Thu Mar 4 22:10:45 2010 @@ -17,6 +17,7 @@ */ package org.apache.pig.test.utils; +import java.io.IOException; import java.util.HashMap; import java.util.Map; import java.util.Random; @@ -77,7 +78,7 @@ return new DataByteArray(genRandString(r).getBytes()); } - public static ResourceFieldSchema getSmallTupleFieldSchema(){ + public static ResourceFieldSchema getSmallTupleFieldSchema() throws IOException{ ResourceFieldSchema stringfs = new ResourceFieldSchema(); stringfs.setType(DataType.CHARARRAY); ResourceFieldSchema intfs = new ResourceFieldSchema(); @@ -143,7 +144,7 @@ return db; } - public static ResourceFieldSchema getSmallTupDataBagFieldSchema() { + public static ResourceFieldSchema getSmallTupDataBagFieldSchema() throws IOException { ResourceFieldSchema tuplefs = getSmallTupleFieldSchema(); ResourceSchema bagSchema = new ResourceSchema(); @@ -190,7 +191,7 @@ return t; } - public static ResourceFieldSchema getSmallBagTextTupleFieldSchema(){ + public static ResourceFieldSchema getSmallBagTextTupleFieldSchema() throws IOException{ ResourceFieldSchema stringfs = new ResourceFieldSchema(); stringfs.setType(DataType.CHARARRAY); @@ -262,7 +263,7 @@ return db; } - public static ResourceFieldSchema getFullTupTextDataBagFieldSchema(){ + public static ResourceFieldSchema getFullTupTextDataBagFieldSchema() throws IOException{ ResourceFieldSchema tuplefs = getSmallBagTextTupleFieldSchema(); ResourceSchema outBagSchema = new ResourceSchema(); @@ -347,7 +348,7 @@ return db; } - public static ResourceFieldSchema getFloatDataBagFieldSchema(int column) { + public static ResourceFieldSchema getFloatDataBagFieldSchema(int column) throws IOException { ResourceFieldSchema intfs = new ResourceFieldSchema(); intfs.setType(DataType.INTEGER); @@ -391,7 +392,7 @@ return t; } - public static ResourceFieldSchema getMixedTupleToConvertFieldSchema() { + public static ResourceFieldSchema getMixedTupleToConvertFieldSchema() throws IOException { ResourceFieldSchema stringfs = new ResourceFieldSchema(); stringfs.setType(DataType.CHARARRAY); ResourceFieldSchema intfs = new ResourceFieldSchema();