Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBooleanWritable.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBooleanWritable.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBooleanWritable.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBooleanWritable.java Fri Sep 19 13:39:31 2008 @@ -3,97 +3,25 @@ */ package org.apache.pig.impl.io; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.hadoop.io.BooleanWritable; /** * */ -public class NullableBooleanWritable extends BooleanWritable { +public class NullableBooleanWritable extends PigNullableWritable { - private boolean isNull = false; - - public static byte NULL = 0x00; - public static byte NOTNULL = 0x01; - - - - /** - * - */ public NullableBooleanWritable() { - super(); + mValue = new BooleanWritable(); } /** * @param value */ public NullableBooleanWritable(boolean value) { - super(value); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object) - */ - @Override - public int compareTo(Object o) { - // if both are null they are equal only here! - if(isNull == true && ((NullableBooleanWritable)o).isNull()) - return 0; - else if(isNull == true) - return -1; - else if (((NullableBooleanWritable)o).isNull()) - return 1; - else - return super.compareTo(o); + mValue = new BooleanWritable(value); } - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput) - */ - @Override - public void readFields(DataInput in) throws IOException { - byte nullMarker = in.readByte(); - if(nullMarker == NULL) { - isNull = true; - } - else { - isNull = false; - super.readFields(in); - } - - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput) - */ - @Override - public void write(DataOutput out) throws IOException { - if(isNull()) { - out.writeByte(NULL); - } else { - out.writeByte(NOTNULL); - super.write(out); - } + public Object getValueAsPigType() { + return isNull() ? null : ((BooleanWritable)mValue).get(); } - - /** - * @return the isNull - */ - public boolean isNull() { - return isNull; - } - - /** - * @param isNull the isNull to set - */ - public void setNull(boolean isNull) { - this.isNull = isNull; - } - - - }
Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBytesWritable.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBytesWritable.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBytesWritable.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableBytesWritable.java Fri Sep 19 13:39:31 2008 @@ -17,96 +17,28 @@ */ package org.apache.pig.impl.io; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.hadoop.io.BytesWritable; +import org.apache.pig.data.DataByteArray; + /** * */ -public class NullableBytesWritable extends BytesWritable { +public class NullableBytesWritable extends PigNullableWritable { - private boolean isNull = false; - - public static byte NULL = 0x00; - public static byte NOTNULL = 0x01; - - - /** - * - */ public NullableBytesWritable() { - super(); + mValue = new BytesWritable(); } /** * @param bytes */ public NullableBytesWritable(byte[] bytes) { - super(bytes); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object) - */ - @Override - public int compareTo(Object o) { - // if both are null they are equal only here! - if(isNull == true && ((NullableBytesWritable)o).isNull()) - return 0; - else if(isNull == true) - return -1; - else if (((NullableBytesWritable)o).isNull()) - return 1; - else - return super.compareTo(o); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput) - */ - @Override - public void readFields(DataInput in) throws IOException { - byte nullMarker = in.readByte(); - if(nullMarker == NULL) { - isNull = true; - } - else { - isNull = false; - super.readFields(in); - } - + mValue = new BytesWritable(bytes); } - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput) - */ - @Override - public void write(DataOutput out) throws IOException { - if(isNull == true) { - out.writeByte(NULL); - } else { - out.writeByte(NOTNULL); - super.write(out); - } - } - - /** - * @return the isNull - */ - public boolean isNull() { - return isNull; + public Object getValueAsPigType() { + BytesWritable bw = (BytesWritable)mValue; + return isNull() ? null : new DataByteArray(bw.get(), 0, bw.getSize()); } - - /** - * @param isNull the isNull to set - */ - public void setNull(boolean isNull) { - this.isNull = isNull; - } - - - } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableDoubleWritable.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableDoubleWritable.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableDoubleWritable.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableDoubleWritable.java Fri Sep 19 13:39:31 2008 @@ -17,95 +17,25 @@ */ package org.apache.pig.impl.io; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.pig.backend.hadoop.DoubleWritable; /** * */ -public class NullableDoubleWritable extends DoubleWritable { +public class NullableDoubleWritable extends PigNullableWritable { - private boolean isNull = false; - - public static byte NULL = 0x00; - public static byte NOTNULL = 0x01; - - /** - * - */ public NullableDoubleWritable() { - super(); + mValue = new DoubleWritable(); } /** * @param value */ public NullableDoubleWritable(double value) { - super(value); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object) - */ - @Override - public int compareTo(Object o) { - // if both are null they are equal only here! - if(isNull == true && ((NullableDoubleWritable)o).isNull()) - return 0; - else if(isNull == true) - return -1; - else if (((NullableDoubleWritable)o).isNull()) - return 1; - else - return super.compareTo(o); + mValue = new DoubleWritable(value); } - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput) - */ - @Override - public void readFields(DataInput in) throws IOException { - byte nullMarker = in.readByte(); - if(nullMarker == NULL) { - isNull = true; - } - else { - isNull = false; - super.readFields(in); - } - - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput) - */ - @Override - public void write(DataOutput out) throws IOException { - if(isNull()) { - out.writeByte(NULL); - } else { - out.writeByte(NOTNULL); - super.write(out); - } + public Object getValueAsPigType() { + return isNull() ? null : ((DoubleWritable)mValue).get(); } - - /** - * @return the isNull - */ - public boolean isNull() { - return isNull; - } - - /** - * @param isNull the isNull to set - */ - public void setNull(boolean isNull) { - this.isNull = isNull; - } - - - } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableFloatWritable.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableFloatWritable.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableFloatWritable.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableFloatWritable.java Fri Sep 19 13:39:31 2008 @@ -17,95 +17,25 @@ */ package org.apache.pig.impl.io; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.hadoop.io.FloatWritable; /** * */ -public class NullableFloatWritable extends FloatWritable { +public class NullableFloatWritable extends PigNullableWritable { - private boolean isNull = false; - - public static byte NULL = 0x00; - public static byte NOTNULL = 0x01; - - /** - * - */ public NullableFloatWritable() { - super(); + mValue = new FloatWritable(); } /** * @param value */ public NullableFloatWritable(float value) { - super(value); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object) - */ - @Override - public int compareTo(Object o) { - // if both are null they are equal only here! - if(isNull == true && ((NullableFloatWritable)o).isNull()) - return 0; - else if(isNull == true) - return -1; - else if (((NullableFloatWritable)o).isNull()) - return 1; - else - return super.compareTo(o); + mValue = new FloatWritable(value); } - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput) - */ - @Override - public void readFields(DataInput in) throws IOException { - byte nullMarker = in.readByte(); - if(nullMarker == NULL) { - isNull = true; - } - else { - isNull = false; - super.readFields(in); - } - - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput) - */ - @Override - public void write(DataOutput out) throws IOException { - if(isNull()) { - out.writeByte(NULL); - } else { - out.writeByte(NOTNULL); - super.write(out); - } + public Object getValueAsPigType() { + return isNull() ? null : ((FloatWritable)mValue).get(); } - - /** - * @return the isNull - */ - public boolean isNull() { - return isNull; - } - - /** - * @param isNull the isNull to set - */ - public void setNull(boolean isNull) { - this.isNull = isNull; - } - - - } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableIntWritable.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableIntWritable.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableIntWritable.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableIntWritable.java Fri Sep 19 13:39:31 2008 @@ -17,104 +17,25 @@ */ package org.apache.pig.impl.io; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.hadoop.io.IntWritable; /** * */ -public class NullableIntWritable extends IntWritable { +public class NullableIntWritable extends PigNullableWritable { - private boolean isNull = false; - - /** - * Marker for indicating whether the value this object holds - * is a null - */ - public static byte NULL = 0x00; - - /** - * Marker for indicating whether the value this object holds - * is not a null - */ - public static byte NOTNULL = 0x01; - - /** - * - */ public NullableIntWritable() { - super(); + mValue = new IntWritable(); } /** * @param value */ public NullableIntWritable(int value) { - super(value); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object) - */ - @Override - public int compareTo(Object o) { - // if both are null they are equal only here! - if(isNull == true && ((NullableIntWritable)o).isNull()) - return 0; - else if(isNull == true) - return -1; - else if (((NullableIntWritable)o).isNull()) - return 1; - else - return super.compareTo(o); + mValue = new IntWritable(value); } - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput) - */ - @Override - public void readFields(DataInput in) throws IOException { - byte nullMarker = in.readByte(); - if(nullMarker == NULL) { - isNull = true; - } - else { - isNull = false; - super.readFields(in); - } - + public Object getValueAsPigType() { + return isNull() ? null : ((IntWritable)mValue).get(); } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput) - */ - @Override - public void write(DataOutput out) throws IOException { - if(isNull()) { - out.writeByte(NULL); - } else { - out.writeByte(NOTNULL); - super.write(out); - } - } - - /** - * @return the isNull - */ - public boolean isNull() { - return isNull; - } - - /** - * @param isNull the isNull to set - */ - public void setNull(boolean isNull) { - this.isNull = isNull; - } - - - } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableLongWritable.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableLongWritable.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableLongWritable.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableLongWritable.java Fri Sep 19 13:39:31 2008 @@ -17,95 +17,25 @@ */ package org.apache.pig.impl.io; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.hadoop.io.LongWritable; /** * */ -public class NullableLongWritable extends LongWritable { +public class NullableLongWritable extends PigNullableWritable { - private boolean isNull = false; - - public static byte NULL = 0x00; - public static byte NOTNULL = 0x01; - - /** - * - */ public NullableLongWritable() { - super(); + mValue = new LongWritable(); } /** * @param value */ public NullableLongWritable(long value) { - super(value); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object) - */ - @Override - public int compareTo(Object o) { - // if both are null they are equal only here! - if(isNull == true && ((NullableLongWritable)o).isNull()) - return 0; - else if(isNull == true) - return -1; - else if (((NullableLongWritable)o).isNull()) - return 1; - else - return super.compareTo(o); + mValue = new LongWritable(value); } - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput) - */ - @Override - public void readFields(DataInput in) throws IOException { - byte nullMarker = in.readByte(); - if(nullMarker == NULL) { - isNull = true; - } - else { - isNull = false; - super.readFields(in); - } - - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput) - */ - @Override - public void write(DataOutput out) throws IOException { - if(isNull()) { - out.writeByte(NULL); - } else { - out.writeByte(NOTNULL); - super.write(out); - } + public Object getValueAsPigType() { + return isNull() ? null : ((LongWritable)mValue).get(); } - - /** - * @return the isNull - */ - public boolean isNull() { - return isNull; - } - - /** - * @param isNull the isNull to set - */ - public void setNull(boolean isNull) { - this.isNull = isNull; - } - - - } Modified: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableText.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableText.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableText.java (original) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableText.java Fri Sep 19 13:39:31 2008 @@ -17,109 +17,32 @@ */ package org.apache.pig.impl.io; -import java.io.DataInput; -import java.io.DataOutput; -import java.io.IOException; - import org.apache.hadoop.io.Text; /** * */ -public class NullableText extends Text { +public class NullableText extends PigNullableWritable { - private boolean isNull = false; - - public static byte NULL = 0x00; - public static byte NOTNULL = 0x01; - - /** - * - */ public NullableText() { - super(); + mValue = new Text(); } /** * @param utf8 */ public NullableText(byte[] utf8) { - super(utf8); + mValue = new Text(utf8); } /** * @param string */ public NullableText(String string) { - super(string); - } - - /** - * @param utf8 - */ - public NullableText(Text utf8) { - super(utf8); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#compareTo(java.lang.Object) - */ - @Override - public int compareTo(Object o) { - // if both are null they are equal only here! - if(isNull == true && ((NullableText)o).isNull()) - return 0; - else if(isNull == true) - return -1; - else if (((NullableText)o).isNull()) - return 1; - else - return super.compareTo(o); - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput) - */ - @Override - public void readFields(DataInput in) throws IOException { - byte nullMarker = in.readByte(); - if(nullMarker == NULL) { - isNull = true; - } - else { - isNull = false; - super.readFields(in); - } - - } - - /* (non-Javadoc) - * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput) - */ - @Override - public void write(DataOutput out) throws IOException { - if(isNull()) { - out.writeByte(NULL); - } else { - out.writeByte(NOTNULL); - super.write(out); - } - } - - /** - * @return the isNull - */ - public boolean isNull() { - return isNull; + mValue = new Text(string); } - /** - * @param isNull the isNull to set - */ - public void setNull(boolean isNull) { - this.isNull = isNull; + public Object getValueAsPigType() { + return isNull() ? null : ((Text)mValue).toString(); } - - - } Added: incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableTuple.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableTuple.java?rev=697229&view=auto ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableTuple.java (added) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/NullableTuple.java Fri Sep 19 13:39:31 2008 @@ -0,0 +1,47 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.pig.impl.io; + +import org.apache.pig.data.Tuple; +import org.apache.pig.data.TupleFactory; + +/** + * + */ +public class NullableTuple extends PigNullableWritable { + + private TupleFactory mFactory = null; + + public NullableTuple() { + if (mFactory == null) { + mFactory = TupleFactory.getInstance(); + } + mValue = mFactory.newTuple(); + } + + /** + * @param bytes + */ + public NullableTuple(Tuple t) { + mValue = t; + } + + public Object getValueAsPigType() { + return isNull() ? null : (Tuple)mValue; + } +} Added: incubator/pig/branches/types/src/org/apache/pig/impl/io/PigNullableWritable.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/src/org/apache/pig/impl/io/PigNullableWritable.java?rev=697229&view=auto ============================================================================== --- incubator/pig/branches/types/src/org/apache/pig/impl/io/PigNullableWritable.java (added) +++ incubator/pig/branches/types/src/org/apache/pig/impl/io/PigNullableWritable.java Fri Sep 19 13:39:31 2008 @@ -0,0 +1,137 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.pig.impl.io; + +import java.io.DataInput; +import java.io.DataOutput; +import java.io.IOException; + +import org.apache.hadoop.io.WritableComparable; + +/** + * A base class for all types that pig uses to move data between map and + * reduce. It implements WritableComparable so that compareTo etc. can be + * called. It also wraps a WritableComparable 'value'. This is set by each + * different type to be an object of its specific type. + * It also provides a getIndex() and setIndex() calls that are used to get + * and set the index. These can be used by LocalRearrange, the partitioner, + * and Package to determine the index. + * + * Index and the null indicator are packed into one byte to save space. + */ +public abstract class PigNullableWritable implements WritableComparable { + + private boolean mNull; + + protected WritableComparable mValue; + + private byte mIndex; + + /** + * Compare two nullable objects. Step one is to check if either or both + * are null. If one is null and the other is not, then the one that is + * null is declared to be less. If both are null the indices are + * compared. If neither are null the indices are again compared. If + * these are equal, finally the values are compared. + * + * These comparators are used by hadoop as part of the post-map sort, when + * the data is still in object format. + */ + public int compareTo(Object o) { + PigNullableWritable w = (PigNullableWritable)o; + if (!mNull && !w.mNull) { + return mValue.compareTo(w.mValue); + } else if (mNull && w.mNull) { + // If they're both null, compare the indicies + if (mIndex < w.mIndex) return -1; + else if (mIndex > w.mIndex) return 1; + else return 0; + } + else if (mNull) return -1; + else return 1; + } + + /* (non-Javadoc) + * @see org.apache.hadoop.io.IntWritable#readFields(java.io.DataInput) + */ + public void readFields(DataInput in) throws IOException { + mNull = in.readBoolean(); + if (!mNull) mValue.readFields(in); + mIndex = in.readByte(); + } + + /* (non-Javadoc) + * @see org.apache.hadoop.io.IntWritable#write(java.io.DataOutput) + */ + public void write(DataOutput out) throws IOException { + out.writeBoolean(mNull); + if (!mNull) mValue.write(out); + out.writeByte(mIndex); + } + + /** + * @return the isNull + */ + public boolean isNull() { + return mNull; + } + + /** + * @param isNull the isNull to set + */ + public void setNull(boolean isNull) { + mNull = isNull; + } + + /** + * @return the index for this value + */ + public byte getIndex() { + return mIndex; + } + + /** + * @param index for this value. + */ + public void setIndex(byte index) { + mIndex = index; + } + + /** + * @return The wrapped value as a pig type, not as a WritableComparable. + */ + abstract public Object getValueAsPigType(); + + @Override + public int hashCode() { + // For now, always give a null a hash code of 0. It isn't clear this + // is what we'll always want. If nulls make a significant but + // not overwhelming amount of the data we may want them to get their + // own partition. If they make up a big enough percentage of the + // data we may want to split them across partitions (though that + // would obviously limit how they could be dealt with afterwards). + if (mNull) return 0; + else return mValue.hashCode(); + } + + @Override + public String toString() { + return new String("Null: " + mNull + " index: " + mIndex + " " + + mValue.toString()); + } +} Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestAlgebraicEval.java Fri Sep 19 13:39:31 2008 @@ -126,8 +126,6 @@ } - /* - @Test public void testSimpleCount() throws Exception { File tmpFile = File.createTempFile("test", "txt"); @@ -281,7 +279,6 @@ } } } -*/ private int generateInput(PrintStream ps, boolean withNulls ) { int numNulls = 0; Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestEvalPipeline.java Fri Sep 19 13:39:31 2008 @@ -345,10 +345,11 @@ if (eliminateDuplicates){ pigServer.registerQuery("B = DISTINCT (FOREACH A GENERATE $0) PARALLEL 10;"); }else{ - if(!useUDF) + if(!useUDF) { pigServer.registerQuery("B = ORDER A BY $0 PARALLEL 10;"); - else + } else { pigServer.registerQuery("B = ORDER A BY $0 using " + TupComp.class.getName() + ";"); + } } pigServer.store("B", tmpOutputFile); @@ -359,12 +360,13 @@ if(!iter.hasNext()) fail("No Results obtained"); while (iter.hasNext()){ Tuple t = iter.next(); - System.out.println(t.get(0).toString()); + //System.out.println(t.get(0).toString()); if (eliminateDuplicates){ Integer act = Integer.parseInt(t.get(0).toString()); assertFalse(seen.contains(act)); seen.add(act); }else{ +System.out.println(last + " " + t.get(0)); assertTrue(last.compareTo(t.get(0).toString())<=0); assertEquals(t.size(), 2); last = t.get(0).toString(); Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestForEach.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestForEach.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestForEach.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestForEach.java Fri Sep 19 13:39:31 2008 @@ -29,7 +29,6 @@ import org.apache.pig.data.DataType; import org.apache.pig.data.DefaultBagFactory; import org.apache.pig.data.DefaultTuple; -import org.apache.pig.data.IndexedTuple; import org.apache.pig.data.Tuple; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestLocalRearrange.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestLocalRearrange.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestLocalRearrange.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestLocalRearrange.java Fri Sep 19 13:39:31 2008 @@ -25,7 +25,6 @@ import org.apache.pig.data.DataBag; import org.apache.pig.data.DataType; import org.apache.pig.data.DefaultTuple; -import org.apache.pig.data.IndexedTuple; import org.apache.pig.data.Tuple; import org.apache.pig.data.TupleFactory; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; @@ -34,6 +33,7 @@ import org.apache.pig.backend.hadoop.executionengine.physicalLayer.expressionOperators.POProject; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.plans.PhysicalPlan; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POLocalRearrange; +import org.apache.pig.impl.io.NullableTuple; import org.apache.pig.impl.plan.PlanException; import org.apache.pig.test.utils.GenPhyOp; import org.apache.pig.test.utils.GenRandomData; @@ -84,21 +84,21 @@ int size=0; for(Result res=lr.getNext(t);res.returnStatus!=POStatus.STATUS_EOP;res=lr.getNext(t)){ Tuple t = (Tuple)res.result; - IndexedTuple it = (IndexedTuple)t.get(1); + Tuple val = (Tuple)t.get(2); //Check if the index is same as input index - assertEquals((float)0, (float)it.index, 0.01f); + assertEquals((byte)0, (byte)(Byte)t.get(0)); - //Check if the input baf contains the value tuple - assertEquals(true, TestHelper.bagContains(db, it.toTuple())); + //Check if the input bag contains the value tuple + assertTrue(TestHelper.bagContains(db, val)); //Check if the input key and the output key are same - String inpKey = (String)it.toTuple().get(0); - assertEquals(true, inpKey.compareTo((String)t.get(0))==0); + String inpKey = (String)val.get(0); + assertEquals(0, inpKey.compareTo((String)t.get(1))); ++size; } //check if all the tuples in the input are generated - assertEquals(true, size==db.size()); + assertEquals(db.size(), size); } private void setUp2() throws PlanException, ExecException{ @@ -127,23 +127,23 @@ int size=0; for(Result res=lr.getNext(t);res.returnStatus!=POStatus.STATUS_EOP;res=lr.getNext(t)){ Tuple t = (Tuple)res.result; - IndexedTuple it = (IndexedTuple)t.get(1); + Tuple val = (Tuple)t.get(2); //Check if the index is same as input index - assertEquals((float)0, (float)it.index, 0.01f); + assertEquals((byte)0, (byte)(Byte)t.get(0)); //Check if the input baf contains the value tuple - assertEquals(true, TestHelper.bagContains(db, it.toTuple())); + assertTrue(TestHelper.bagContains(db, val)); //Check if the input key and the output key are same Tuple inpKey = TupleFactory.getInstance().newTuple(2); - inpKey.set(0,it.toTuple().get(0)); - inpKey.set(1,it.toTuple().get(1)); - assertEquals(true, inpKey.compareTo((Tuple)t.get(0))==0); + inpKey.set(0, val.get(0)); + inpKey.set(1, val.get(1)); + assertEquals(0, inpKey.compareTo((Tuple)t.get(1))); ++size; } //check if all the tuples in the input are generated - assertEquals(true, size==db.size()); + assertEquals(db.size(), size); } } Modified: incubator/pig/branches/types/test/org/apache/pig/test/TestPackage.java URL: http://svn.apache.org/viewvc/incubator/pig/branches/types/test/org/apache/pig/test/TestPackage.java?rev=697229&r1=697228&r2=697229&view=diff ============================================================================== --- incubator/pig/branches/types/test/org/apache/pig/test/TestPackage.java (original) +++ incubator/pig/branches/types/test/org/apache/pig/test/TestPackage.java Fri Sep 19 13:39:31 2008 @@ -20,19 +20,24 @@ import static org.junit.Assert.assertEquals; import java.io.IOException; +import java.util.ArrayList; import java.util.Iterator; +import java.util.List; +import java.util.Map; import java.util.Random; import org.apache.pig.backend.executionengine.ExecException; +import org.apache.pig.backend.hadoop.HDataType; +import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; +import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result; +import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage; import org.apache.pig.data.DataBag; import org.apache.pig.data.DataType; import org.apache.pig.data.DefaultBagFactory; -import org.apache.pig.data.IndexedTuple; import org.apache.pig.data.Tuple; +import org.apache.pig.impl.io.NullableTuple; +import org.apache.pig.impl.io.PigNullableWritable; import org.apache.pig.impl.plan.OperatorKey; -import org.apache.pig.backend.hadoop.executionengine.physicalLayer.POStatus; -import org.apache.pig.backend.hadoop.executionengine.physicalLayer.Result; -import org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POPackage; import org.apache.pig.test.utils.GenRandomData; import org.apache.pig.test.utils.TestHelper; import org.junit.After; @@ -49,124 +54,104 @@ public void tearDown() throws Exception { } - static class ITIterator implements Iterator<IndexedTuple>, - Iterable<IndexedTuple> { - private Iterator<Tuple> it; - - public ITIterator(Iterator<Tuple> it) { - this.it = it; - } - - public boolean hasNext() { - return it.hasNext(); - } - - public IndexedTuple next() { - return (IndexedTuple) it.next(); - } - - public void remove() { - // TODO Auto-generated method stub - - } - - public Iterator<IndexedTuple> iterator() { - return this; - } - - } - - public static boolean test(Object key, boolean inner[]) throws ExecException, IOException { - boolean ret = false; + private void runTest(Object key, boolean inner[]) throws ExecException, IOException { Random r = new Random(); DataBag db1 = GenRandomData.genRandSmallTupDataBag(r, 10, 100); DataBag db2 = GenRandomData.genRandSmallTupDataBag(r, 10, 100); - DataBag db = DefaultBagFactory.getInstance().newDefaultBag(); + List<NullableTuple> db = new ArrayList<NullableTuple>(200); Iterator db1Iter = db1.iterator(); if(!inner[0]){ while (db1Iter.hasNext()) { - IndexedTuple it = new IndexedTuple((Tuple) db1Iter.next(), 0); + NullableTuple it = new NullableTuple((Tuple)db1Iter.next()); + it.setIndex((byte)0); db.add(it); } } Iterator db2Iter = db2.iterator(); while (db2Iter.hasNext()) { - IndexedTuple it = new IndexedTuple((Tuple) db2Iter.next(), 1); + NullableTuple it = new NullableTuple((Tuple) db2Iter.next()); + it.setIndex((byte)1); db.add(it); } - ITIterator iti = new TestPackage.ITIterator(db.iterator()); + //ITIterator iti = new TestPackage.ITIterator(db.iterator()); POPackage pop = new POPackage(new OperatorKey("", r.nextLong())); pop.setNumInps(2); pop.setInner(inner); - pop.attachInput(key, iti); + PigNullableWritable k = HDataType.getWritableComparableTypes(key, (byte)0); + pop.attachInput(k, db.iterator()); Tuple t = null; Result res = null; res = (Result) pop.getNext(t); - if(res.returnStatus==POStatus.STATUS_NULL && inner[0]) - return true; - if (res.returnStatus != POStatus.STATUS_OK) - return false; + if(res.returnStatus==POStatus.STATUS_NULL && inner[0]) return; + assertEquals(POStatus.STATUS_OK, res.returnStatus); t = (Tuple) res.result; Object outKey = t.get(0); DataBag outDb1 = (DataBag) t.get(1); DataBag outDb2 = (DataBag) t.get(2); - if (outKey == key && TestHelper.compareBags(db1, outDb1) - && TestHelper.compareBags(db2, outDb2)) - return true; - return ret; + assertEquals(key, outKey); + assertTrue(TestHelper.compareBags(db1, outDb1)); + assertTrue(TestHelper.compareBags(db2, outDb2)); } /** * To show that it does not have any type specific * code */ - private static boolean test(byte t, boolean[] inner) throws ExecException, IOException { + private void pickTest(byte t, boolean[] inner) throws ExecException, IOException { Random r = new Random(); switch (t) { case DataType.BAG: - return test(GenRandomData.genRandSmallTupDataBag(r, 10, 100),inner); + runTest(GenRandomData.genRandSmallTupDataBag(r, 10, 100),inner); + break; case DataType.BOOLEAN: - return test(r.nextBoolean(),inner); + runTest(r.nextBoolean(),inner); + break; case DataType.BYTEARRAY: - return test(GenRandomData.genRandDBA(r),inner); + runTest(GenRandomData.genRandDBA(r),inner); + break; case DataType.CHARARRAY: - return test(GenRandomData.genRandString(r),inner); + runTest(GenRandomData.genRandString(r),inner); + break; case DataType.DOUBLE: - return test(r.nextDouble(),inner); + runTest(r.nextDouble(),inner); + break; case DataType.FLOAT: - return test(r.nextFloat(),inner); + runTest(r.nextFloat(),inner); + break; case DataType.INTEGER: - return test(r.nextLong(),inner); + runTest(r.nextLong(),inner); + break; case DataType.LONG: - return test(r.nextLong(),inner); + runTest(r.nextLong(),inner); + break; case DataType.MAP: - return test(GenRandomData.genRandMap(r, 10),inner); + case DataType.BYTE: + return; // map not key type case DataType.TUPLE: - return test(GenRandomData.genRandSmallBagTuple(r, 10, 100),inner); + runTest(GenRandomData.genRandSmallBagTuple(r, 10, 100),inner); + break; + + default: + fail("No test case for type " + DataType.findTypeName(t)); } - return false; } @Test public void testOperator() throws ExecException, IOException{ byte[] types = DataType.genAllTypes(); -// Map<Byte, String> map = operatorHelper.genTypeToNameMap(); -// System.out.println("Testing Package:"); for (byte b : types) { -// System.out.print("\t With " + map.get(b) + ": "); + System.out.println("Type " + DataType.findTypeName(b)); boolean succ = true; int NUM_TRIALS = 10; boolean[] inner1 = { false , false }; for (int i = 0; i < NUM_TRIALS; i++) - succ &= test(b, inner1); - assertEquals(true, succ); + pickTest(b, inner1); boolean[] inner2 = { true , false }; for (int i = 0; i < NUM_TRIALS; i++) - succ &= test(b, inner2); - assertEquals(true, succ); + pickTest(b, inner2); /*if (succ) System.out.println("Success!!"); else