Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleNextup.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleNextup.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleNextup.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleNextup.java Thu Nov 27 12:49:54 2014 @@ -61,7 +61,7 @@ public class DoubleNextup extends EvalFu * @param output returns a single numeric value, nextup value of the argument */ public Double exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; Double d; try{
Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleRound.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleRound.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleRound.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleRound.java Thu Nov 27 12:49:54 2014 @@ -68,7 +68,7 @@ public class DoubleRound extends EvalFun */ @Override public Long exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleSignum.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleSignum.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleSignum.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleSignum.java Thu Nov 27 12:49:54 2014 @@ -62,7 +62,7 @@ public class DoubleSignum extends EvalFu */ @Override public Double exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleUlp.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleUlp.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleUlp.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/DoubleUlp.java Thu Nov 27 12:49:54 2014 @@ -66,7 +66,7 @@ public class DoubleUlp extends EvalFunc< * the size of an ulp of the argument. */ public Double exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatAbs.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatAbs.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatAbs.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatAbs.java Thu Nov 27 12:49:54 2014 @@ -67,7 +67,7 @@ public class FloatAbs extends EvalFunc<F * @param output returns a single numeric value, absolute value of the argument */ public Float exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; Float d; Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatCopySign.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatCopySign.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatCopySign.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatCopySign.java Thu Nov 27 12:49:54 2014 @@ -67,6 +67,9 @@ public class FloatCopySign extends EvalF public Float exec(Tuple input) throws IOException { if (input == null || input.size() < 2) return null; + if (input.get(0) == null || input.get(1) == null) { + return null; + } try{ float first = (Float)input.get(0); float second = (Float)input.get(1); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatGetExponent.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatGetExponent.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatGetExponent.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatGetExponent.java Thu Nov 27 12:49:54 2014 @@ -62,7 +62,7 @@ public class FloatGetExponent extends Ev * exponent used in the representation of a double */ public Integer exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try { Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatMax.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatMax.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatMax.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatMax.java Thu Nov 27 12:49:54 2014 @@ -60,9 +60,8 @@ public class FloatMax extends EvalFunc<F * @param output returns a single numeric value, which is the smaller of two inputs */ public Float exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() < 2) return null; - try{ Float first = (Float)input.get(0); Float second = (Float)input.get(1); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatMin.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatMin.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatMin.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatMin.java Thu Nov 27 12:49:54 2014 @@ -60,9 +60,8 @@ public class FloatMin extends EvalFunc<F * @param output returns a single numeric value, which is the smaller of two inputs */ public Float exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() < 2) return null; - try{ Float first = (Float)input.get(0); Float second = (Float)input.get(1); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatNextAfter.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatNextAfter.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatNextAfter.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatNextAfter.java Thu Nov 27 12:49:54 2014 @@ -70,6 +70,11 @@ public class FloatNextAfter extends Eval */ @Override public Float exec(Tuple input) throws IOException { + if (input == null || input.size() < 2) + return null; + if (input.get(0) == null || input.get(1) == null) { + return null; + } try{ Float first = (Float)input.get(0); Double second = (Double)input.get(0); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatNextup.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatNextup.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatNextup.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatNextup.java Thu Nov 27 12:49:54 2014 @@ -61,7 +61,7 @@ public class FloatNextup extends EvalFun * @param output returns a single numeric value, nextup value of the argument */ public Float exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; Float d; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatRound.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatRound.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatRound.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatRound.java Thu Nov 27 12:49:54 2014 @@ -68,7 +68,7 @@ public class FloatRound extends EvalFunc */ @Override public Integer exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatSignum.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatSignum.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatSignum.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatSignum.java Thu Nov 27 12:49:54 2014 @@ -62,7 +62,7 @@ public class FloatSignum extends EvalFun */ @Override public Float exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatUlp.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatUlp.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatUlp.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/FloatUlp.java Thu Nov 27 12:49:54 2014 @@ -66,7 +66,7 @@ public class FloatUlp extends EvalFunc<F * the size of an ulp of the argument. */ public Float exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntAbs.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntAbs.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntAbs.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntAbs.java Thu Nov 27 12:49:54 2014 @@ -66,7 +66,7 @@ public class IntAbs extends EvalFunc<Int * @param output returns a single numeric value, absolute value of the argument */ public Integer exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; Integer d; Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntMax.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntMax.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntMax.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntMax.java Thu Nov 27 12:49:54 2014 @@ -60,7 +60,7 @@ public class IntMax extends EvalFunc<Int * @param output returns a single numeric value, which is the smaller of two inputs */ public Integer exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() < 2) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntMin.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntMin.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntMin.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/IntMin.java Thu Nov 27 12:49:54 2014 @@ -60,7 +60,7 @@ public class IntMin extends EvalFunc<Int * @param output returns a single numeric value, which is the smaller of two inputs */ public Integer exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() < 2) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongAbs.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongAbs.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongAbs.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongAbs.java Thu Nov 27 12:49:54 2014 @@ -67,7 +67,7 @@ public class LongAbs extends EvalFunc<Lo * @param output returns a single numeric value, absolute value of the argument */ public Long exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; Long d; Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongMax.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongMax.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongMax.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongMax.java Thu Nov 27 12:49:54 2014 @@ -60,7 +60,7 @@ public class LongMax extends EvalFunc<Lo * @param output returns a single numeric value, which is the smaller of two inputs */ public Long exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() < 2) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongMin.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongMin.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongMin.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/LongMin.java Thu Nov 27 12:49:54 2014 @@ -60,7 +60,7 @@ public class LongMin extends EvalFunc<Lo * @param output returns a single numeric value, which is the smaller of two inputs */ public Long exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() < 2) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/NEXTUP.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/NEXTUP.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/NEXTUP.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/NEXTUP.java Thu Nov 27 12:49:54 2014 @@ -65,7 +65,7 @@ public class NEXTUP extends EvalFunc<Dou * @param output returns a single numeric value, nextup value of the argument */ public Double exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; Double d; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/ROUND.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/ROUND.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/ROUND.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/ROUND.java Thu Nov 27 12:49:54 2014 @@ -72,7 +72,7 @@ public class ROUND extends EvalFunc<Long */ @Override public Long exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/SCALB.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/SCALB.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/SCALB.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/SCALB.java Thu Nov 27 12:49:54 2014 @@ -72,7 +72,9 @@ public class SCALB extends EvalFunc<Doub public Double exec(Tuple input) throws IOException { if (input == null || input.size() < 2) return null; - + if (input.get(0) == null || input.get(1) == null) { + return null; + } try{ Double first = DataType.toDouble(input.get(0)); Integer second = DataType.toInteger(input.get(1)); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/SIGNUM.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/SIGNUM.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/SIGNUM.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/SIGNUM.java Thu Nov 27 12:49:54 2014 @@ -66,7 +66,7 @@ public class SIGNUM extends EvalFunc<Dou */ @Override public Double exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/ULP.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/ULP.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/ULP.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/ULP.java Thu Nov 27 12:49:54 2014 @@ -66,7 +66,7 @@ public class ULP extends EvalFunc<Double * the size of an ulp of the argument. */ public Double exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/copySign.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/copySign.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/copySign.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/copySign.java Thu Nov 27 12:49:54 2014 @@ -70,6 +70,9 @@ public class copySign extends EvalFunc<D public Double exec(Tuple input) throws IOException { if (input == null || input.size() < 2) return null; + if (input.get(0) == null || input.get(1) == null) { + return null; + } try{ double first = DataType.toDouble(input.get(0)); double second = DataType.toDouble(input.get(1)); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/getExponent.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/getExponent.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/getExponent.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/getExponent.java Thu Nov 27 12:49:54 2014 @@ -66,7 +66,7 @@ public class getExponent extends EvalFun * exponent used in the representation of a double */ public Integer exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; try { Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/nextAfter.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/nextAfter.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/nextAfter.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/math/nextAfter.java Thu Nov 27 12:49:54 2014 @@ -70,6 +70,12 @@ public class nextAfter extends EvalFunc< */ @Override public Double exec(Tuple input) throws IOException { + if (input == null || input.size() < 2) { + return null; + } + if (input.get(0) == null || input.get(1) == null) { + return null; + } try{ double first = DataType.toDouble(input.get(0)); double second = DataType.toDouble(input.get(1)); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/HashFNV2.java Thu Nov 27 12:49:54 2014 @@ -34,7 +34,7 @@ public class HashFNV2 extends HashFNV { String msg = "HashFNV : Only 2 parameters are allowed."; throw new IOException(msg); } - if (input.get(0)==null) + if (input.get(0)==null || input.get(1)==null) return null; try { mMod = (Integer)input.get(1); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexExtract.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexExtract.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexExtract.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexExtract.java Thu Nov 27 12:49:54 2014 @@ -66,7 +66,7 @@ public class RegexExtract extends EvalFu String msg = "RegexExtract : Only 3 parameters are allowed."; throw new IOException(msg); } - if (input.get(0)==null) + if (input.get(0)==null || input.get(1)==null || input.get(2)==null) return null; try { if (!input.get(1).equals(mExpression)) Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexExtractAll.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexExtractAll.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexExtractAll.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexExtractAll.java Thu Nov 27 12:49:54 2014 @@ -59,7 +59,7 @@ public class RegexExtractAll extends Eva throw new IOException(msg); } - if (input.get(0)==null) + if (input.get(0)==null||input.get(1)==null) return null; try { if (!input.get(1).equals(mExpression)) { Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexMatch.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexMatch.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexMatch.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/string/RegexMatch.java Thu Nov 27 12:49:54 2014 @@ -56,7 +56,7 @@ public class RegexMatch extends EvalFunc String msg = "RegexMatch : Only 2 parameters are allowed."; throw new IOException(msg); } - if (input.get(0)==null) + if (input.get(0)==null || input.get(1)==null) return null; try { if (!input.get(1).equals(mExpression)) Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/DateExtractor.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/DateExtractor.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/DateExtractor.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/DateExtractor.java Thu Nov 27 12:49:54 2014 @@ -104,7 +104,7 @@ public class DateExtractor extends EvalF @Override public String exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; String str=""; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/HostExtractor.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/HostExtractor.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/HostExtractor.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/HostExtractor.java Thu Nov 27 12:49:54 2014 @@ -44,7 +44,7 @@ import org.apache.pig.impl.logicalLayer. public class HostExtractor extends EvalFunc<String> { @Override public String exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; String str=""; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/SearchEngineExtractor.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/SearchEngineExtractor.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/SearchEngineExtractor.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/util/apachelogparser/SearchEngineExtractor.java Thu Nov 27 12:49:54 2014 @@ -389,7 +389,7 @@ public class SearchEngineExtractor exten @Override public String exec(Tuple input) throws IOException { - if (input == null || input.size() == 0) + if (input == null || input.size() == 0 || input.get(0) == null) return null; String referer=""; try{ Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/XPath.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/XPath.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/XPath.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/evaluation/xml/XPath.java Thu Nov 27 12:49:54 2014 @@ -81,6 +81,9 @@ public class XPath extends EvalFunc<Stri try { final String xml = (String) input.get(0); + if (xml == null) { + return null; + } if(input.size() > 2) cache = (Boolean) input.get(2); Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/DBStorage.java Thu Nov 27 12:49:54 2014 @@ -19,10 +19,10 @@ package org.apache.pig.piggybank.storage import java.io.IOException; import java.sql.Connection; -import java.sql.Date; import java.sql.DriverManager; import java.sql.PreparedStatement; import java.sql.SQLException; +import java.sql.Timestamp; import java.util.Properties; import org.apache.commons.logging.Log; @@ -136,7 +136,7 @@ public class DBStorage extends StoreFunc break; case DataType.DATETIME: - ps.setDate(sqlPos, new Date(((DateTime) field).getMillis())); + ps.setTimestamp(sqlPos, new Timestamp(((DateTime) field).getMillis())); sqlPos++; break; Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/MultiStorage.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/MultiStorage.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/MultiStorage.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/MultiStorage.java Thu Nov 27 12:49:54 2014 @@ -99,7 +99,9 @@ public class MultiStorage extends StoreF * Constructor * * @param parentPathStr - * Parent output dir path + * Parent output dir path (this will be specified in store statement, + * so MultiStorage don't use this parameter in reality. However, we don't + * want to change the construct to break backward compatibility) * @param splitFieldIndex * key field index * @param compression Modified: pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/XMLLoader.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/XMLLoader.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/XMLLoader.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/main/java/org/apache/pig/piggybank/storage/XMLLoader.java Thu Nov 27 12:49:54 2014 @@ -150,11 +150,12 @@ public class XMLLoader extends LoadFunc // In case of an tag matched with an open tag and a closed tag, this buffer // is used to accumulate matched element if it is spans multiple lines. StringBuffer currentMatch = new StringBuffer(); - // The start offset of first matched open tag. This marks the first byte - // in the range to be copied to output. - int offsetOfFirstMatchedOpenTag = 0; try { while (true) { + // The start offset of first matched open tag. This marks the first byte + // in the range to be copied to output. + int offsetOfFirstMatchedOpenTag = 0; + while (buffer == null || buffer.length() == 0) { if (!wrapped.nextKeyValue()) return false; // End of split Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/datetime/convert/TestConvertDateTime.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/datetime/convert/TestConvertDateTime.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/datetime/convert/TestConvertDateTime.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/datetime/convert/TestConvertDateTime.java Thu Nov 27 12:49:54 2014 @@ -23,10 +23,30 @@ import org.apache.pig.data.TupleFactory; import org.apache.pig.piggybank.evaluation.datetime.convert.CustomFormatToISO; import org.apache.pig.piggybank.evaluation.datetime.convert.ISOToUnix; import org.apache.pig.piggybank.evaluation.datetime.convert.UnixToISO; +import org.joda.time.DateTimeZone; + +import org.junit.After; +import org.junit.Before; import org.junit.Test; public class TestConvertDateTime { + private DateTimeZone currentDTZ, defaultDTZ; + + @Before + public void setUp() throws Exception { + currentDTZ = DateTimeZone.getDefault(); + + // set the timezone to somethere other than UTC + defaultDTZ = DateTimeZone.forID("+08:00"); + DateTimeZone.setDefault(defaultDTZ); + } + + @After + public void tearDown() throws Exception { + DateTimeZone.setDefault(currentDTZ); + } + @Test public void testBadFormat() throws Exception { Tuple t1 = TupleFactory.getInstance().newTuple(2); @@ -35,7 +55,7 @@ public class TestConvertDateTime { CustomFormatToISO convert = new CustomFormatToISO(); assertNull("Input that doesn't match format should result in null", convert.exec(t1)); t1.set(0, "July, 2012"); - assertEquals("Matching format should work correctly", "2012-07-01T00:00:00.000Z", convert.exec(t1)); + assertEquals("Matching format should work correctly", "2012-07-01T00:00:00.000+08:00", convert.exec(t1)); } @Test @@ -48,7 +68,8 @@ public class TestConvertDateTime { UnixToISO func = new UnixToISO(); String iso = func.exec(t1); - assertTrue(iso.equals("2009-01-07T01:07:01.000Z")); + assertEquals("2009-01-07T09:07:01.000+08:00", iso); + assertEquals("Should not change the default timezone", defaultDTZ, DateTimeZone.getDefault()); } @Test @@ -61,7 +82,7 @@ public class TestConvertDateTime { Long unix = func2.exec(t2); assertTrue(unix == 1231290421000L); - + assertEquals("Should not change the default timezone", defaultDTZ, DateTimeZone.getDefault()); } @Test @@ -73,6 +94,20 @@ public class TestConvertDateTime { CustomFormatToISO func = new CustomFormatToISO(); String iso = func.exec(t); - assertTrue(iso.equals("2010-10-10T00:00:00.000Z")); + assertEquals("2010-10-10T00:00:00.000+08:00", iso); } + + @Test + public void testCustomFormatToISOWithTimezone() throws Exception { + + Tuple t = TupleFactory.getInstance().newTuple(2); + t.set(0, "10/10/2010 Z"); + t.set(1, "dd/MM/yyyy Z"); + CustomFormatToISO func = new CustomFormatToISO(); + String iso = func.exec(t); + + assertEquals("2010-10-10T00:00:00.000Z", iso); + assertEquals("Should not change the default timezone", defaultDTZ, DateTimeZone.getDefault()); + } + } Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/datetime/truncate/TestTruncateDateTime.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/datetime/truncate/TestTruncateDateTime.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/datetime/truncate/TestTruncateDateTime.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/evaluation/datetime/truncate/TestTruncateDateTime.java Thu Nov 27 12:49:54 2014 @@ -42,10 +42,8 @@ public class TestTruncateDateTime extend Tuple t1 = TupleFactory.getInstance().newTuple(1); t1.set(0, "2010-04-15T08:11:33.020"); - // ISOHelper's internal default timezone is preferred over previous default DateTimeZone. - assertEquals(ISOHelper.parseDateTime(t1), new DateTime(2010, 4, 15, 8, 11, 33, 20, ISOHelper.DEFAULT_DATE_TIME_ZONE)); + assertEquals(new DateTime(2010, 4, 15, 8, 11, 33, 20, testDefaultDateTimeZone), ISOHelper.parseDateTime(t1)); - // Calling parseDate restores DateTimeZone.default before it returns. assertTrue(testDefaultDateTimeZone.equals(DateTimeZone.getDefault())); // Restore pre-test default time zone. @@ -59,7 +57,7 @@ public class TestTruncateDateTime extend t1.set(0, "2010-04-15T08:11:33.020Z"); // Time zone is preserved. - assertEquals(ISOHelper.parseDateTime(t1), new DateTime(2010, 4, 15, 8, 11, 33, 20, DateTimeZone.UTC)); + assertEquals(new DateTime(2010, 4, 15, 8, 11, 33, 20, DateTimeZone.UTC), ISOHelper.parseDateTime(t1)); // Time zone is strictly preserved. Parsed date is not equal to "simultaneous" datetime in different time zone. assertFalse(ISOHelper.parseDateTime(t1).equals(new DateTime(2010, 4, 15, 0, 11, 33, 20, DateTimeZone.forOffsetHours(-8)))); @@ -72,14 +70,14 @@ public class TestTruncateDateTime extend t1.set(0, "2010-04-15T08:11:33.020-08:00"); // Time zone is preserved. - assertEquals(ISOHelper.parseDateTime(t1), new DateTime(2010, 4, 15, 8, 11, 33, 20, DateTimeZone.forOffsetHours(-8))); + assertEquals(new DateTime(2010, 4, 15, 8, 11, 33, 20, DateTimeZone.forOffsetHours(-8)), ISOHelper.parseDateTime(t1)); // Time zone is strictly preserved. Parsed date is not equal to "simultaneous" datetime in different time zone. assertFalse(ISOHelper.parseDateTime(t1).equals(new DateTime(2010, 4, 15, 16, 11, 33, 20, DateTimeZone.UTC))); } /** - * When no time zone is specified at all, UTC is presumed. + * When no time zone is specified at all, we use the default. * @throws Exception */ @Test @@ -89,7 +87,7 @@ public class TestTruncateDateTime extend t1.set(0, "2010-04-15T08:11:33.020"); // Time zone is preserved. - assertEquals(ISOHelper.parseDateTime(t1), new DateTime(2010, 4, 15, 8, 11, 33, 20, DateTimeZone.UTC)); + assertEquals(new DateTime(2010, 4, 15, 8, 11, 33, 20, DateTimeZone.getDefault()), ISOHelper.parseDateTime(t1)); } /** @@ -112,7 +110,7 @@ public class TestTruncateDateTime extend /** * Parsing ISO date with no time and no time zone works. - * Time defaults to midnight in UTC. + * Time defaults to midnight in default timezone. * @throws Exception */ @Test @@ -122,7 +120,7 @@ public class TestTruncateDateTime extend t1.set(0, "2010-04-15"); // Time zone is preserved. - assertEquals(ISOHelper.parseDateTime(t1), new DateTime(2010, 4, 15, 0, 0, 0, 0, DateTimeZone.UTC)); + assertEquals(new DateTime(2010, 4, 15, 0, 0, 0, 0, DateTimeZone.getDefault()), ISOHelper.parseDateTime(t1)); } /** @@ -137,7 +135,7 @@ public class TestTruncateDateTime extend t1.set(0, "T08:11:33.020Z"); // Time zone is preserved. - assertEquals(ISOHelper.parseDateTime(t1), new DateTime(1970, 1, 1, 8, 11, 33, 20, DateTimeZone.UTC)); + assertEquals(new DateTime(1970, 1, 1, 8, 11, 33, 20, DateTimeZone.UTC), ISOHelper.parseDateTime(t1)); } /** @@ -152,7 +150,7 @@ public class TestTruncateDateTime extend t1.set(0, "T08:11:33.020-0800"); // Time zone is preserved. - assertEquals(ISOHelper.parseDateTime(t1), new DateTime(1970, 1, 1, 8, 11, 33, 20, DateTimeZone.forOffsetHours(-8))); + assertEquals(new DateTime(1970, 1, 1, 8, 11, 33, 20, DateTimeZone.forOffsetHours(-8)), ISOHelper.parseDateTime(t1)); } /** @@ -167,7 +165,7 @@ public class TestTruncateDateTime extend t1.set(0, "T08:11:33.020Z"); // Time zone is preserved. - assertEquals(ISOHelper.parseDateTime(t1), new DateTime(1970, 1, 1, 8, 11, 33, 20, DateTimeZone.UTC)); + assertEquals(new DateTime(1970, 1, 1, 8, 11, 33, 20, DateTimeZone.UTC), ISOHelper.parseDateTime(t1)); } @Test @@ -179,7 +177,7 @@ public class TestTruncateDateTime extend ISOToYear func = new ISOToYear(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-01-01T00:00:00.000Z"); + assertEquals("2010-01-01T00:00:00.000Z", truncated); } @Test @@ -191,7 +189,7 @@ public class TestTruncateDateTime extend ISOToMonth func = new ISOToMonth(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-01T00:00:00.000Z"); + assertEquals("2010-04-01T00:00:00.000Z", truncated); } @Test @@ -203,7 +201,7 @@ public class TestTruncateDateTime extend ISOToWeek func = new ISOToWeek(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-12T00:00:00.000Z"); + assertEquals("2010-04-12T00:00:00.000Z", truncated); } @Test @@ -215,7 +213,7 @@ public class TestTruncateDateTime extend ISOToDay func = new ISOToDay(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-15T00:00:00.000Z"); + assertEquals("2010-04-15T00:00:00.000Z", truncated); } @Test @@ -227,7 +225,7 @@ public class TestTruncateDateTime extend ISOToHour func = new ISOToHour(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-15T08:00:00.000Z"); + assertEquals("2010-04-15T08:00:00.000Z", truncated); } @Test @@ -239,7 +237,7 @@ public class TestTruncateDateTime extend ISOToMinute func = new ISOToMinute(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-15T08:11:00.000Z"); + assertEquals("2010-04-15T08:11:00.000Z", truncated); } @Test @@ -251,7 +249,7 @@ public class TestTruncateDateTime extend ISOToSecond func = new ISOToSecond(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-15T08:11:33.000Z"); + assertEquals("2010-04-15T08:11:33.000Z", truncated); } @@ -264,7 +262,7 @@ public class TestTruncateDateTime extend ISOToYear func = new ISOToYear(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-01-01T00:00:00.000-08:00"); + assertEquals("2010-01-01T00:00:00.000-08:00", truncated); } @Test @@ -276,7 +274,7 @@ public class TestTruncateDateTime extend ISOToMonth func = new ISOToMonth(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-01T00:00:00.000-08:00"); + assertEquals("2010-04-01T00:00:00.000-08:00", truncated); } @Test @@ -288,7 +286,7 @@ public class TestTruncateDateTime extend ISOToWeek func = new ISOToWeek(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-12T00:00:00.000-08:00"); + assertEquals("2010-04-12T00:00:00.000-08:00", truncated); } @Test @@ -300,7 +298,7 @@ public class TestTruncateDateTime extend ISOToDay func = new ISOToDay(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-15T00:00:00.000-08:00"); + assertEquals("2010-04-15T00:00:00.000-08:00", truncated); } @Test @@ -312,7 +310,7 @@ public class TestTruncateDateTime extend ISOToHour func = new ISOToHour(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-15T08:00:00.000-08:00"); + assertEquals("2010-04-15T08:00:00.000-08:00", truncated); } @Test @@ -324,7 +322,7 @@ public class TestTruncateDateTime extend ISOToMinute func = new ISOToMinute(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-15T08:11:00.000-08:00"); + assertEquals("2010-04-15T08:11:00.000-08:00", truncated); } @Test @@ -336,7 +334,7 @@ public class TestTruncateDateTime extend ISOToSecond func = new ISOToSecond(); String truncated = func.exec(t1); - assertEquals(truncated, "2010-04-15T08:11:33.000-08:00"); + assertEquals("2010-04-15T08:11:33.000-08:00", truncated); } -} \ No newline at end of file +} Modified: pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java URL: http://svn.apache.org/viewvc/pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java (original) +++ pig/branches/spark/contrib/piggybank/java/src/test/java/org/apache/pig/piggybank/test/storage/TestXMLLoader.java Thu Nov 27 12:49:54 2014 @@ -99,6 +99,12 @@ public class TestXMLLoader extends TestC inlineClosedTags.add(new String[] { "</events>"}); } + public static ArrayList<String[]> indentedXmlWithMultilineLineContent = new ArrayList<String[]>(); + static { + indentedXmlWithMultilineLineContent.add(new String[] { " <page>You have " }); + indentedXmlWithMultilineLineContent.add(new String[] { "not missed it</page>" }); + } + public void testShouldReturn0TupleCountIfSearchTagIsNotFound () throws Exception { String filename = TestHelper.createTempFile(data, ""); PigServer pig = new PigServer(LOCAL); @@ -354,6 +360,27 @@ public class TestXMLLoader extends TestC assertEquals(4, tupleCount); } + public void testXMLLoaderShouldWorkWithIndentedXmlWithMultilineContent() throws Exception { + String filename = TestHelper.createTempFile(indentedXmlWithMultilineLineContent, ""); + PigServer pig = new PigServer(LOCAL); + filename = filename.replace("\\", "\\\\"); + String query = "A = LOAD '" + filename + "' USING org.apache.pig.piggybank.storage.XMLLoader('page') as (doc:chararray);"; + pig.registerQuery(query); + Iterator<?> it = pig.openIterator("A"); + int tupleCount = 0; + while (it.hasNext()) { + Tuple tuple = (Tuple) it.next(); + if (tuple == null) + break; + else { + System.out.println(((String) tuple.get(0))); + assertTrue(((String) tuple.get(0)).equals("<page>You have not missed it</page>")); + tupleCount++; + } + } + assertEquals(1, tupleCount); + } + public void testXMLLoaderShouldReturnValidXML() throws Exception { String filename = TestHelper.createTempFile(inlineClosedTags, ""); PigServer pig = new PigServer(LOCAL); Modified: pig/branches/spark/ivy.xml URL: http://svn.apache.org/viewvc/pig/branches/spark/ivy.xml?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/ivy.xml (original) +++ pig/branches/spark/ivy.xml Thu Nov 27 12:49:54 2014 @@ -406,25 +406,25 @@ <!-- for piggybank --> <dependency org="hsqldb" name="hsqldb" rev="${hsqldb.version}" conf="test->default" /> - <dependency org="org.apache.hive" name="hive-exec" rev="${hive.version}" conf="compile->master"> + <dependency org="org.apache.hive" name="hive-exec" rev="${hive.version}" conf="compile->master" changing="true"> <artifact name="hive-exec" m:classifier="core" /> </dependency> - <dependency org="org.apache.hive" name="hive-serde" rev="${hive.version}" + <dependency org="org.apache.hive" name="hive-serde" rev="${hive.version}" changing="true" conf="compile->master" /> - <dependency org="org.apache.hive" name="hive-common" rev="${hive.version}" + <dependency org="org.apache.hive" name="hive-common" rev="${hive.version}" changing="true" conf="compile->master" /> - <dependency org="org.apache.hive.shims" name="hive-shims-common" rev="${hive.version}" - conf="test->master" /> - <dependency org="org.apache.hive.shims" name="hive-shims-common-secure" rev="${hive.version}" - conf="test->master" /> - <dependency org="org.apache.hive.shims" name="hive-shims-0.23" rev="${hive.version}" + <dependency org="org.apache.hive.shims" name="hive-shims-common" rev="${hive.version}" changing="true" + conf="compile->master" /> + <dependency org="org.apache.hive.shims" name="hive-shims-common-secure" rev="${hive.version}" changing="true" + conf="compile->master" /> + <dependency org="org.apache.hive.shims" name="hive-shims-0.23" rev="${hive.version}" changing="true" conf="hadoop23->master" /> - <dependency org="org.apache.hive.shims" name="hive-shims-0.20S" rev="${hive.version}" + <dependency org="org.apache.hive.shims" name="hive-shims-0.20S" rev="${hive.version}" changing="true" conf="hadoop20->master" /> <dependency org="org.iq80.snappy" name="snappy" rev="${snappy.version}" conf="test->master" /> <dependency org="com.esotericsoftware.kryo" name="kryo" rev="${kryo.version}" - conf="test->master" /> + conf="compile->master" /> <dependency org="org.vafer" name="jdeb" rev="${jdeb.version}" conf="compile->master"> Modified: pig/branches/spark/ivy/libraries.properties URL: http://svn.apache.org/viewvc/pig/branches/spark/ivy/libraries.properties?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/ivy/libraries.properties (original) +++ pig/branches/spark/ivy/libraries.properties Thu Nov 27 12:49:54 2014 @@ -48,7 +48,7 @@ hadoop-mapreduce.version=2.4.0 hbase94.version=0.94.1 hbase95.version=0.96.0-${hbase.hadoop.version} hsqldb.version=1.8.0.10 -hive.version=0.14.0-SNAPSHOT +hive.version=0.14.0 httpcomponents.version=4.1 jackson.version=1.8.8 jackson-pig-3039-test.version=1.9.9 @@ -62,7 +62,7 @@ jettison.version=1.3.4 jetty.version=6.1.26 jetty-util.version=6.1.26 jline.version=1.0 -joda-time.version=2.1 +joda-time.version=2.5 jopt.version=4.1 json-simple.version=1.1 junit.version=4.11 @@ -95,6 +95,6 @@ mockito.version=1.8.4 jansi.version=1.9 asm.version=3.3.1 snappy.version=1.1.0.1 -tez.version=0.5.0 +tez.version=0.5.2 parquet-pig-bundle.version=1.2.3 snappy.version=0.2 Modified: pig/branches/spark/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java URL: http://svn.apache.org/viewvc/pig/branches/spark/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original) +++ pig/branches/spark/shims/src/hadoop20/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Thu Nov 27 12:49:54 2014 @@ -18,6 +18,8 @@ package org.apache.pig.backend.hadoop.executionengine.shims; import java.io.IOException; +import java.util.Arrays; +import java.util.Iterator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -184,15 +186,19 @@ public class HadoopShims { runningJob.killJob(); } - public static TaskReport[] getTaskReports(Job job, TaskType type) throws IOException { + public static Iterator<TaskReport> getTaskReports(Job job, TaskType type) throws IOException { if (job.getJobConf().getBoolean(PigConfiguration.PIG_NO_TASK_REPORT, false)) { LOG.info("TaskReports are disabled for job: " + job.getAssignedJobID()); return null; } JobClient jobClient = job.getJobClient(); - return (type == TaskType.MAP) - ? jobClient.getMapTaskReports(job.getAssignedJobID()) - : jobClient.getReduceTaskReports(job.getAssignedJobID()); + TaskReport[] reports = null; + if (type == TaskType.MAP) { + reports = jobClient.getMapTaskReports(job.getAssignedJobID()); + } else { + reports = jobClient.getReduceTaskReports(job.getAssignedJobID()); + } + return reports == null ? null : Arrays.asList(reports).iterator(); } public static boolean isHadoopYARN() { Modified: pig/branches/spark/shims/src/hadoop23/org/apache/hadoop/mapred/DowngradeHelper.java URL: http://svn.apache.org/viewvc/pig/branches/spark/shims/src/hadoop23/org/apache/hadoop/mapred/DowngradeHelper.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/shims/src/hadoop23/org/apache/hadoop/mapred/DowngradeHelper.java (original) +++ pig/branches/spark/shims/src/hadoop23/org/apache/hadoop/mapred/DowngradeHelper.java Thu Nov 27 12:49:54 2014 @@ -17,11 +17,39 @@ */ package org.apache.hadoop.mapred; +import java.util.Iterator; + public class DowngradeHelper { - // This is required since hadoop 2 TaskReport allows + // This is required since hadoop 2 TaskReport allows // only package level access to this api - public static TaskReport[] downgradeTaskReports( + public static Iterator<TaskReport> downgradeTaskReports( org.apache.hadoop.mapreduce.TaskReport[] reports) { - return TaskReport.downgradeArray(reports); + return reports == null ? null : new TaskReportIterator(reports); + } + + private static class TaskReportIterator implements Iterator<TaskReport> { + + private org.apache.hadoop.mapreduce.TaskReport[] reports; + private int curIndex = 0; + + public TaskReportIterator(org.apache.hadoop.mapreduce.TaskReport[] reports) { + this.reports = reports; + } + + @Override + public boolean hasNext() { + return curIndex < this.reports.length ; + } + + @Override + public TaskReport next() { + return TaskReport.downgrade(reports[curIndex++]); + } + + @Override + public void remove() { + throw new UnsupportedOperationException(); + } + } } Modified: pig/branches/spark/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java URL: http://svn.apache.org/viewvc/pig/branches/spark/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java (original) +++ pig/branches/spark/shims/src/hadoop23/org/apache/pig/backend/hadoop/executionengine/shims/HadoopShims.java Thu Nov 27 12:49:54 2014 @@ -19,6 +19,7 @@ package org.apache.pig.backend.hadoop.ex import java.io.IOException; import java.lang.reflect.Method; +import java.util.Iterator; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; @@ -32,6 +33,7 @@ import org.apache.hadoop.mapred.TIPStatu import org.apache.hadoop.mapred.TaskReport; import org.apache.hadoop.mapred.jobcontrol.Job; import org.apache.hadoop.mapred.jobcontrol.JobControl; +import org.apache.hadoop.mapreduce.Cluster; import org.apache.hadoop.mapreduce.ContextFactory; import org.apache.hadoop.mapreduce.JobContext; import org.apache.hadoop.mapreduce.JobID; @@ -120,7 +122,12 @@ public class HadoopShims { public static Counters getCounters(Job job) throws IOException { try { - return new Counters(job.getJob().getCounters()); + Cluster cluster = new Cluster(job.getJobConf()); + org.apache.hadoop.mapreduce.Job mrJob = cluster.getJob(job.getAssignedJobID()); + if (mrJob == null) { // In local mode, mrJob will be null + mrJob = job.getJob(); + } + return new Counters(mrJob.getCounters()); } catch (Exception ir) { throw new IOException(ir); } @@ -214,20 +221,24 @@ public class HadoopShims { } } - public static TaskReport[] getTaskReports(Job job, TaskType type) throws IOException { + public static Iterator<TaskReport> getTaskReports(Job job, TaskType type) throws IOException { if (job.getJobConf().getBoolean(PigConfiguration.PIG_NO_TASK_REPORT, false)) { LOG.info("TaskReports are disabled for job: " + job.getAssignedJobID()); return null; } - org.apache.hadoop.mapreduce.Job mrJob = job.getJob(); + Cluster cluster = new Cluster(job.getJobConf()); try { + org.apache.hadoop.mapreduce.Job mrJob = cluster.getJob(job.getAssignedJobID()); + if (mrJob == null) { // In local mode, mrJob will be null + mrJob = job.getJob(); + } org.apache.hadoop.mapreduce.TaskReport[] reports = mrJob.getTaskReports(type); return DowngradeHelper.downgradeTaskReports(reports); } catch (InterruptedException ir) { throw new IOException(ir); } } - + public static boolean isHadoopYARN() { return true; } Modified: pig/branches/spark/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java URL: http://svn.apache.org/viewvc/pig/branches/spark/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java (original) +++ pig/branches/spark/shims/test/hadoop23/org/apache/pig/test/TezMiniCluster.java Thu Nov 27 12:49:54 2014 @@ -65,9 +65,7 @@ public class TezMiniCluster extends Mini CONF_DIR.mkdirs(); // Build mini DFS cluster - Configuration hdfsConf = new Configuration(false); - hdfsConf.addResource("core-default.xml"); - hdfsConf.addResource("hdfs-default.xml"); + Configuration hdfsConf = new Configuration(); m_dfs = new MiniDFSCluster.Builder(hdfsConf) .numDataNodes(2) .format(true) @@ -98,8 +96,8 @@ public class TezMiniCluster extends Mini m_mr_conf = m_mr.getConfig(); m_mr_conf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, System.getProperty("java.class.path")); - m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx2048m"); - m_mr_conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx2048m"); + m_mr_conf.set(MRJobConfig.MAP_JAVA_OPTS, "-Xmx512m"); + m_mr_conf.set(MRJobConfig.REDUCE_JAVA_OPTS, "-Xmx512m"); Configuration mapred_site = new Configuration(false); Configuration yarn_site = new Configuration(false); @@ -146,7 +144,7 @@ public class TezMiniCluster extends Mini m_conf = m_mr_conf; // Turn FetchOptimizer off so that we can actually test Tez - m_conf.set(PigConfiguration.OPT_FETCH, System.getProperty("test.opt.fetch", "false")); + m_conf.set(PigConfiguration.PIG_OPT_FETCH, System.getProperty("test.opt.fetch", "false")); System.setProperty("junit.hadoop.conf", CONF_DIR.getPath()); System.setProperty("hadoop.log.dir", "build/test/logs"); Modified: pig/branches/spark/src/docs/src/documentation/content/xdocs/basic.xml URL: http://svn.apache.org/viewvc/pig/branches/spark/src/docs/src/documentation/content/xdocs/basic.xml?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/src/docs/src/documentation/content/xdocs/basic.xml (original) +++ pig/branches/spark/src/docs/src/documentation/content/xdocs/basic.xml Thu Nov 27 12:49:54 2014 @@ -6495,12 +6495,14 @@ E = GROUP D BY A::x; <title>Example: Nested Block</title> <p>In this example a CROSS is performed within the nested block.</p> <source> - user = load 'user' as (uid, age, gender, region); +user = load 'user' as (uid, age, gender, region); session = load 'session' as (uid, region); C = cogroup user by uid, session by uid; D = foreach C { crossed = cross user, session; - generate crossed; + generate crossed; +} +dump D; </source> <p>In this example FOREACH is nested to the second level.</p> <source> @@ -8958,7 +8960,7 @@ B = FOREACH A GENERATE myFunc($0); <p id="register-glob">Additionally, JAR files stored in local file systems can be specified as a glob pattern using â*â. Pig will search for matching jars in the local file system, either the relative path (relative to your working directory) or the absolute path. Pig will pick up all JARs that match the glob.</p> <p><strong>Command Line</strong></p> - <p>You can register additional files (to use with your Pig script) via the command line using the -Dpig.additional.jars option. + <p>You can register additional files (to use with your Pig script) via PIG_OPTS environment variable using the -Dpig.additional.jars.uris option. For more information see <a href="udf.html">User Defined Functions</a>.</p> </section> @@ -8973,16 +8975,18 @@ A = LOAD 'students'; B = FOREACH A GENERATE myfunc.MyEvalFunc($0); </source> -<p>In this example additional JAR files are registered via the command line.</p> +<p>In this example additional JAR files are registered via PIG_OPTS environment variable.</p> <source> -pig -Dpig.additional.jars=my.jar:your.jar script.pig +export PIG_OPTS="-Dpig.additional.jars.uris=my.jar,your.jar" </source> -<p>In this example a JAR file stored in HDFS is registered.</p> +<p>In this example a JAR file stored in HDFS and a local JAR file are registered.</p> <source> -pig -Dpig.additional.jars=hdfs://nn.mydomain.com:9020/myjars/my.jar script.pig +export PIG_OPTS="-Dpig.additional.jars.uris=hdfs://nn.mydomain.com:9020/myjars/my.jar,file:///home/root/pig/your.jar" </source> +<p>Note, the legacy property pig.additional.jars which use colon as separator is still supported. But we recommend to use pig.additional.jars.uris since colon is also used in URL scheme, and thus we cannot use full scheme in the list. We will deprecate pig.additional.jar in future releases.</p> + <p>This example shows how to specify a glob pattern using either a relative path or an absolute path.</p> <source> register /homes/user/pig/myfunc*.jar Modified: pig/branches/spark/src/docs/src/documentation/content/xdocs/cont.xml URL: http://svn.apache.org/viewvc/pig/branches/spark/src/docs/src/documentation/content/xdocs/cont.xml?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/src/docs/src/documentation/content/xdocs/cont.xml (original) +++ pig/branches/spark/src/docs/src/documentation/content/xdocs/cont.xml Thu Nov 27 12:49:54 2014 @@ -39,8 +39,6 @@ <p><strong>Python</strong></p> <source> $ pig myembedded.py -OR -$ java -cp <jython jars>:<pig jars>; [--embedded python] /tmp/myembedded.py </source> <p></p> <p>Pig will look for the <code>#!/usr/bin/python</code> line in the script.</p> @@ -69,8 +67,6 @@ else : <p><strong>JavaScript</strong></p> <source> $ pig myembedded.js -OR -$ java -cp <rhino jars>:<pig jars>; [--embedded javascript] /tmp/myembedded.js </source> <p></p> <p>Pig will look for the *.js extension in the script.</p> @@ -98,8 +94,6 @@ function main() { <p><strong>Groovy</strong></p> <source> $ pig myembedded.groovy -OR -$ java -cp <groovy-all jar>:<pig jars>; [--embedded groovy] /tmp/myembedded.groovy </source> <p></p> <p>Pig will look for the *.groovy extension in the script.</p> @@ -836,8 +830,8 @@ $ javac -cp pig.jar idlocal.java <p> </p> <p>From your current working directory, run the program. To view the results, check the output file, id.out.</p> <source> -Unix: $ java -cp pig.jar:. idlocal -Cygwin: $ java âcp '.;pig.jar' idlocal +Unix: $ java -cp pig.jar:. idlocal +Windows: $ java âcp .;pig.jar idlocal </source> <p>idlocal.java - The sample code is based on Pig Latin statements that extract all user IDs from the /etc/passwd file. Modified: pig/branches/spark/src/docs/src/documentation/content/xdocs/func.xml URL: http://svn.apache.org/viewvc/pig/branches/spark/src/docs/src/documentation/content/xdocs/func.xml?rev=1642132&r1=1642131&r2=1642132&view=diff ============================================================================== --- pig/branches/spark/src/docs/src/documentation/content/xdocs/func.xml (original) +++ pig/branches/spark/src/docs/src/documentation/content/xdocs/func.xml Thu Nov 27 12:49:54 2014 @@ -2408,9 +2408,90 @@ STORE A INTO 'accumulo://flights?instanc because the first element in the Tuple is used as the row in Accumulo. </p> </section> - </section> -</section> + </section> + <section id="OrcStorage"> + <title>OrcStorage</title> + <p>Loads from or stores data to Orc file.</p> + <section> + <title>Syntax</title> + <table> + <tr> + <td> + <p>OrcStorage(['options'])</p> + </td> + </tr> + </table> + </section> + <section> + <title>Options</title> + <table> + <tr> + <td> + <p>A string that contains space-separated options (‘-optionA valueA -optionB valueB -optionC ’). Current options are only applicable with STORE operation and not for LOAD.</p> + <p>Currently supported options are:</p> + <ul> + <li>--stripeSize or -s Set the stripe size for the file. Default is 268435456(256 MB).</li> + <li>--rowIndexStride or -r Set the distance between entries in the row index. Default is 10000.</li> + <li>--bufferSize or -b Set the size of the memory buffers used for compressing and storing the stripe in memory. Default is 262144 (256K).</li> + <li>--blockPadding or -p Sets whether the HDFS blocks are padded to prevent stripes from straddling blocks. Default is true.</li> + <li>--compress or -c Sets the generic compression that is used to compress the data. Valid codecs are: NONE, ZLIB, SNAPPY, LZO. Default is ZLIB.</li> + <li>--version or -v Sets the version of the file that will be written</li> + </ul> + </td> + </tr> + </table> + </section> + <section> + <title>Example</title> + <p>OrcStorage as a StoreFunc.</p> +<source> +A = LOAD 'student.txt' as (name:chararray, age:int, gpa:double); +store A into 'student.orc' using OrcStorage('-c SNAPPY'); -- store student.txt into data.orc with SNAPPY compression +</source> + <p>OrcStorage as a LoadFunc.</p> +<source> +A = LOAD 'student.orc' USING OrcStorage(); +describe A; -- See the schema of student.orc +B = filter A by age > 25 and gpa < 3; -- filter condition will be pushed up to loader +dump B; -- dump the content of student.orc +</source> + </section> + <section> + <title>Data types</title> + <p>Most Orc data type has one to one mapping to Pig data type. Several exceptions are:</p> + <p>Loader side:</p> + <ul> + <li>Orc STRING/CHAR/VARCHAR all map to Pig varchar</li> + <li>Orc BYTE/BINARY all map to Pig bytearray</li> + <li>Orc TIMESTAMP/DATE all maps to Pig datetime</li> + <li>Orc DECIMAL maps to Pig bigdecimal</li> + </ul> + <p>Storer side:</p> + <ul> + <li>Pig chararray maps to Orc STRING</li> + <li>Pig datetime maps to Orc TIMESTAMP</li> + <li>Pig bigdecimal/biginteger all map to Orc DECIMAL</li> + <li>Pig bytearray maps to Orc BINARY</li> + </ul> + </section> + <section> + <title>Predicate pushdown</title> + <p>If there is a filter statement right after OrcStorage, Pig will push the filter condition to the loader. + OrcStorage will prune file/stripe/row group which does not satisfy the condition entirely. For the file/stripe/row group contains + data that satisfies the filter condition, OrcStorage will load the file/stripe/row group and Pig will evaluate the filter condition + again to remove additional data which does not satisfy the filter condition.</p> + <p>OrcStorage predicate pushdown currently support all primitive data types but none of the complex data types. For example, map condition + cannot push into OrcStorage:</p> +<source> +A = LOAD 'student.orc' USING OrcStorage(); +B = filter A by info#'age' > 25; -- map condition cannot push to OrcStorage +dump B; +</source> + <p>Currently, the following expressions in filter condition are supported in OrcStorage predicate pushdown: >, >=, <, <=, ==, !=, between, in, and, or, not. The missing expressions are: is null, is not null, matches.</p> + </section> + </section> +</section> <!-- ======================================================== --> <!-- ======================================================== --> @@ -4358,7 +4439,70 @@ For example, given the string (open:sour </section> </section> -<!-- ======================================================== --> +<!-- ======================================================== --> + <section id="strsplittobag"> + <title>STRSPLITTOBAG</title> + <p>Splits a string around matches of a given regular expression and returns a databag</p> + <section> + <title>Syntax</title> + <table> + <tr> + <td> + <p>STRSPLITTOBAG(string, regex, limit)</p> + </td> + </tr> + </table> + </section> + <section> + <title>Terms</title> + <table> + <tr> + <td> + <p>string</p> + </td> + <td> + <p>The string to be split.</p> + </td> + </tr> + <tr> + <td> + <p>regex</p> + </td> + <td> + <p>The regular expression.</p> + </td> + </tr> + <tr> + <td> + <p>limit</p> + </td> + <td> + <p>If the value is positive, the pattern (the compiled representation of the regular expression) + is applied at most limit-1 times, therefore the value of the argument means the maximum size + of the result bag. The last tuple of the result bag will contain all input after the last + match. + </p> + <p>If the value is negative, no limit is applied to the size of the result bag.</p> + <p>If the value is zero, no limit is applied to the size of the result bag too, and trailing + empty strings (if any) will be removed. + </p> + </td> + </tr> + </table> + </section> + <section> + <title>Usage</title> + <p> + Use the STRSPLITTOBAG function to split a string around matches of a given regular expression. + </p> + <p> + For example, given the string (open:source:software), STRSPLITTOBAG (string, ':',2) will return + {(open),(source:software)} and STRSPLITTOBAG (string, ':',3) will return {(open),(source),(software)}. + </p> + </section> + </section> + + <!-- ======================================================== --> <section id="substring"> <title>SUBSTRING</title> <p>Returns a substring from a given string. </p> @@ -4530,6 +4674,16 @@ Use the UPPER function to convert all ch </section> </section> + <section id="uniqueid"> + <title>UniqueID</title> + <p>Returns a unique id string for each record in the alias. </p> + <section> + <title>Usage</title> + <p> + UniqueID generates a unique id for each records. The id takes form "taskindex-sequence" + </p> + </section> + </section> </section> <!-- End String Functions -->
