http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java new file mode 100644 index 0000000..8e752eb --- /dev/null +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/util/DecimalUtility.java @@ -0,0 +1,692 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.util; + +import io.netty.buffer.DrillBuf; + +import java.math.BigDecimal; +import java.math.BigInteger; +import java.util.Arrays; + +import org.apache.drill.common.types.TypeProtos; +import org.apache.drill.common.util.CoreDecimalUtility; + +public class DecimalUtility extends CoreDecimalUtility{ + + public final static int MAX_DIGITS = 9; + public final static int DIGITS_BASE = 1000000000; + public final static int DIGITS_MAX = 999999999; + public final static int integerSize = (Integer.SIZE/8); + + public final static String[] decimalToString = {"", + "0", + "00", + "000", + "0000", + "00000", + "000000", + "0000000", + "00000000", + "000000000"}; + + public final static long[] scale_long_constants = { + 1, + 10, + 100, + 1000, + 10000, + 100000, + 1000000, + 10000000, + 100000000, + 1000000000, + 10000000000l, + 100000000000l, + 1000000000000l, + 10000000000000l, + 100000000000000l, + 1000000000000000l, + 10000000000000000l, + 100000000000000000l, + 1000000000000000000l}; + + /* + * Simple function that returns the static precomputed + * power of ten, instead of using Math.pow + */ + public static long getPowerOfTen(int power) { + assert power >= 0 && power < scale_long_constants.length; + return scale_long_constants[(power)]; + } + + /* + * Math.pow returns a double and while multiplying with large digits + * in the decimal data type we encounter noise. So instead of multiplying + * with Math.pow we use the static constants to perform the multiplication + */ + public static long adjustScaleMultiply(long input, int factor) { + int index = Math.abs(factor); + assert index >= 0 && index < scale_long_constants.length; + if (factor >= 0) { + return input * scale_long_constants[index]; + } else { + return input / scale_long_constants[index]; + } + } + + public static long adjustScaleDivide(long input, int factor) { + int index = Math.abs(factor); + assert index >= 0 && index < scale_long_constants.length; + if (factor >= 0) { + return input / scale_long_constants[index]; + } else { + return input * scale_long_constants[index]; + } + } + + /* Given the number of actual digits this function returns the + * number of indexes it will occupy in the array of integers + * which are stored in base 1 billion + */ + public static int roundUp(int ndigits) { + return (ndigits + MAX_DIGITS - 1)/MAX_DIGITS; + } + + /* Returns a string representation of the given integer + * If the length of the given integer is less than the + * passed length, this function will prepend zeroes to the string + */ + public static StringBuilder toStringWithZeroes(int number, int desiredLength) { + String value = ((Integer) number).toString(); + int length = value.length(); + + StringBuilder str = new StringBuilder(); + str.append(decimalToString[desiredLength - length]); + str.append(value); + + return str; + } + + public static StringBuilder toStringWithZeroes(long number, int desiredLength) { + String value = ((Long) number).toString(); + int length = value.length(); + + StringBuilder str = new StringBuilder(); + + // Desired length can be > MAX_DIGITS + int zeroesLength = desiredLength - length; + while (zeroesLength > MAX_DIGITS) { + str.append(decimalToString[MAX_DIGITS]); + zeroesLength -= MAX_DIGITS; + } + str.append(decimalToString[zeroesLength]); + str.append(value); + + return str; + } + + public static BigDecimal getBigDecimalFromIntermediate(DrillBuf data, int startIndex, int nDecimalDigits, int scale) { + + // In the intermediate representation we don't pad the scale with zeroes, so set truncate = false + return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, false); + } + + public static BigDecimal getBigDecimalFromSparse(DrillBuf data, int startIndex, int nDecimalDigits, int scale) { + + // In the sparse representation we pad the scale with zeroes for ease of arithmetic, need to truncate + return getBigDecimalFromDrillBuf(data, startIndex, nDecimalDigits, scale, true); + } + + public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int start, int length, int scale) { + byte[] value = new byte[length]; + bytebuf.getBytes(start, value, 0, length); + BigInteger unscaledValue = new BigInteger(value); + return new BigDecimal(unscaledValue, scale); + } + + /* Create a BigDecimal object using the data in the DrillBuf. + * This function assumes that data is provided in a non-dense format + * It works on both sparse and intermediate representations. + */ + public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf data, int startIndex, int nDecimalDigits, int scale, boolean truncateScale) { + + // For sparse decimal type we have padded zeroes at the end, strip them while converting to BigDecimal. + int actualDigits; + + // Initialize the BigDecimal, first digit in the DrillBuf has the sign so mask it out + BigInteger decimalDigits = BigInteger.valueOf((data.getInt(startIndex)) & 0x7FFFFFFF); + + BigInteger base = BigInteger.valueOf(DIGITS_BASE); + + for (int i = 1; i < nDecimalDigits; i++) { + + BigInteger temp = BigInteger.valueOf(data.getInt(startIndex + (i * integerSize))); + decimalDigits = decimalDigits.multiply(base); + decimalDigits = decimalDigits.add(temp); + } + + // Truncate any additional padding we might have added + if (truncateScale == true && scale > 0 && (actualDigits = scale % MAX_DIGITS) != 0) { + BigInteger truncate = BigInteger.valueOf((int)Math.pow(10, (MAX_DIGITS - actualDigits))); + decimalDigits = decimalDigits.divide(truncate); + } + + // set the sign + if ((data.getInt(startIndex) & 0x80000000) != 0) { + decimalDigits = decimalDigits.negate(); + } + + BigDecimal decimal = new BigDecimal(decimalDigits, scale); + + return decimal; + } + + /* This function returns a BigDecimal object from the dense decimal representation. + * First step is to convert the dense representation into an intermediate representation + * and then invoke getBigDecimalFromDrillBuf() to get the BigDecimal object + */ + public static BigDecimal getBigDecimalFromDense(DrillBuf data, int startIndex, int nDecimalDigits, int scale, int maxPrecision, int width) { + + /* This method converts the dense representation to + * an intermediate representation. The intermediate + * representation has one more integer than the dense + * representation. + */ + byte[] intermediateBytes = new byte[((nDecimalDigits + 1) * integerSize)]; + + // Start storing from the least significant byte of the first integer + int intermediateIndex = 3; + + int[] mask = {0x03, 0x0F, 0x3F, 0xFF}; + int[] reverseMask = {0xFC, 0xF0, 0xC0, 0x00}; + + int maskIndex; + int shiftOrder; + byte shiftBits; + + // TODO: Some of the logic here is common with casting from Dense to Sparse types, factor out common code + if (maxPrecision == 38) { + maskIndex = 0; + shiftOrder = 6; + shiftBits = 0x00; + intermediateBytes[intermediateIndex++] = (byte) (data.getByte(startIndex) & 0x7F); + } else if (maxPrecision == 28) { + maskIndex = 1; + shiftOrder = 4; + shiftBits = (byte) ((data.getByte(startIndex) & 0x03) << shiftOrder); + intermediateBytes[intermediateIndex++] = (byte) (((data.getByte(startIndex) & 0x3C) & 0xFF) >>> 2); + } else { + throw new UnsupportedOperationException("Dense types with max precision 38 and 28 are only supported"); + } + + int inputIndex = 1; + boolean sign = false; + + if ((data.getByte(startIndex) & 0x80) != 0) { + sign = true; + } + + while (inputIndex < width) { + + intermediateBytes[intermediateIndex] = (byte) ((shiftBits) | (((data.getByte(startIndex + inputIndex) & reverseMask[maskIndex]) & 0xFF) >>> (8 - shiftOrder))); + + shiftBits = (byte) ((data.getByte(startIndex + inputIndex) & mask[maskIndex]) << shiftOrder); + + inputIndex++; + intermediateIndex++; + + if (((inputIndex - 1) % integerSize) == 0) { + shiftBits = (byte) ((shiftBits & 0xFF) >>> 2); + maskIndex++; + shiftOrder -= 2; + } + + } + /* copy the last byte */ + intermediateBytes[intermediateIndex] = shiftBits; + + if (sign == true) { + intermediateBytes[0] = (byte) (intermediateBytes[0] | 0x80); + } + DrillBuf intermediate = data.getAllocator().buffer(intermediateBytes.length); + intermediate.setBytes(0, intermediateBytes); + + BigDecimal ret = getBigDecimalFromIntermediate(intermediate, 0, nDecimalDigits + 1, scale); + intermediate.release(); + return ret; + } + + /* + * Function converts the BigDecimal and stores it in out internal sparse representation + */ + public static void getSparseFromBigDecimal(BigDecimal input, DrillBuf data, int startIndex, int scale, int precision, int nDecimalDigits) { + + // Initialize the buffer + for (int i = 0; i < nDecimalDigits; i++) { + data.setInt(startIndex + (i * integerSize), 0); + } + + boolean sign = false; + + if (input.signum() == -1) { + // negative input + sign = true; + input = input.abs(); + } + + // Truncate the input as per the scale provided + input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); + + // Separate out the integer part + BigDecimal integerPart = input.setScale(0, BigDecimal.ROUND_DOWN); + + int destIndex = nDecimalDigits - roundUp(scale) - 1; + + // we use base 1 billion integer digits for out integernal representation + BigDecimal base = new BigDecimal(DIGITS_BASE); + + while (integerPart.compareTo(BigDecimal.ZERO) == 1) { + // store the modulo as the integer value + data.setInt(startIndex + (destIndex * integerSize), (integerPart.remainder(base)).intValue()); + destIndex--; + // Divide by base 1 billion + integerPart = (integerPart.divide(base)).setScale(0, BigDecimal.ROUND_DOWN); + } + + /* Sparse representation contains padding of additional zeroes + * so each digit contains MAX_DIGITS for ease of arithmetic + */ + int actualDigits; + if ((actualDigits = (scale % MAX_DIGITS)) != 0) { + // Pad additional zeroes + scale = scale + (MAX_DIGITS - actualDigits); + input = input.setScale(scale, BigDecimal.ROUND_DOWN); + } + + //separate out the fractional part + BigDecimal fractionalPart = input.remainder(BigDecimal.ONE).movePointRight(scale); + + destIndex = nDecimalDigits - 1; + + while (scale > 0) { + // Get next set of MAX_DIGITS (9) store it in the DrillBuf + fractionalPart = fractionalPart.movePointLeft(MAX_DIGITS); + BigDecimal temp = fractionalPart.remainder(BigDecimal.ONE); + + data.setInt(startIndex + (destIndex * integerSize), (temp.unscaledValue().intValue())); + destIndex--; + + fractionalPart = fractionalPart.setScale(0, BigDecimal.ROUND_DOWN); + scale -= MAX_DIGITS; + } + + // Set the negative sign + if (sign == true) { + data.setInt(startIndex, data.getInt(startIndex) | 0x80000000); + } + + } + + + public static long getDecimal18FromBigDecimal(BigDecimal input, int scale, int precision) { + // Truncate or pad to set the input to the correct scale + input = input.setScale(scale, BigDecimal.ROUND_HALF_UP); + + return (input.unscaledValue().longValue()); + } + + public static BigDecimal getBigDecimalFromPrimitiveTypes(int input, int scale, int precision) { + return BigDecimal.valueOf(input, scale); + } + + public static BigDecimal getBigDecimalFromPrimitiveTypes(long input, int scale, int precision) { + return BigDecimal.valueOf(input, scale); + } + + + public static int compareDenseBytes(DrillBuf left, int leftStart, boolean leftSign, DrillBuf right, int rightStart, boolean rightSign, int width) { + + int invert = 1; + + /* If signs are different then simply look at the + * sign of the two inputs and determine which is greater + */ + if (leftSign != rightSign) { + + return((leftSign == true) ? -1 : 1); + } else if(leftSign == true) { + /* Both inputs are negative, at the end we will + * have to invert the comparison + */ + invert = -1; + } + + int cmp = 0; + + for (int i = 0; i < width; i++) { + byte leftByte = left.getByte(leftStart + i); + byte rightByte = right.getByte(rightStart + i); + // Unsigned byte comparison + if ((leftByte & 0xFF) > (rightByte & 0xFF)) { + cmp = 1; + break; + } else if ((leftByte & 0xFF) < (rightByte & 0xFF)) { + cmp = -1; + break; + } + } + cmp *= invert; // invert the comparison if both were negative values + + return cmp; + } + + public static int getIntegerFromSparseBuffer(DrillBuf buffer, int start, int index) { + int value = buffer.getInt(start + (index * 4)); + + if (index == 0) { + /* the first byte contains sign bit, return value without it */ + value = (value & 0x7FFFFFFF); + } + return value; + } + + public static void setInteger(DrillBuf buffer, int start, int index, int value) { + buffer.setInt(start + (index * 4), value); + } + + public static int compareSparseBytes(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits, boolean absCompare) { + + int invert = 1; + + if (absCompare == false) { + if (leftSign != rightSign) { + return (leftSign == true) ? -1 : 1; + } + + // Both values are negative invert the outcome of the comparison + if (leftSign == true) { + invert = -1; + } + } + + int cmp = compareSparseBytesInner(left, leftStart, leftSign, leftScale, leftPrecision, right, rightStart, rightSign, rightPrecision, rightScale, width, nDecimalDigits); + return cmp * invert; + } + public static int compareSparseBytesInner(DrillBuf left, int leftStart, boolean leftSign, int leftScale, int leftPrecision, DrillBuf right, int rightStart, boolean rightSign, int rightPrecision, int rightScale, int width, int nDecimalDigits) { + /* compute the number of integer digits in each decimal */ + int leftInt = leftPrecision - leftScale; + int rightInt = rightPrecision - rightScale; + + /* compute the number of indexes required for storing integer digits */ + int leftIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftInt); + int rightIntRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightInt); + + /* compute number of indexes required for storing scale */ + int leftScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(leftScale); + int rightScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(rightScale); + + /* compute index of the most significant integer digits */ + int leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp; + int rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp; + + int leftStopIndex = nDecimalDigits - leftScaleRoundedUp; + int rightStopIndex = nDecimalDigits - rightScaleRoundedUp; + + /* Discard the zeroes in the integer part */ + while (leftIndex1 < leftStopIndex) { + if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) { + break; + } + + /* Digit in this location is zero, decrement the actual number + * of integer digits + */ + leftIntRoundedUp--; + leftIndex1++; + } + + /* If we reached the stop index then the number of integers is zero */ + if (leftIndex1 == leftStopIndex) { + leftIntRoundedUp = 0; + } + + while (rightIndex1 < rightStopIndex) { + if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) { + break; + } + + /* Digit in this location is zero, decrement the actual number + * of integer digits + */ + rightIntRoundedUp--; + rightIndex1++; + } + + if (rightIndex1 == rightStopIndex) { + rightIntRoundedUp = 0; + } + + /* We have the accurate number of non-zero integer digits, + * if the number of integer digits are different then we can determine + * which decimal is larger and needn't go down to comparing individual values + */ + if (leftIntRoundedUp > rightIntRoundedUp) { + return 1; + } + else if (rightIntRoundedUp > leftIntRoundedUp) { + return -1; + } + + /* The number of integer digits are the same, set the each index + * to the first non-zero integer and compare each digit + */ + leftIndex1 = nDecimalDigits - leftScaleRoundedUp - leftIntRoundedUp; + rightIndex1 = nDecimalDigits - rightScaleRoundedUp - rightIntRoundedUp; + + while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) { + if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) { + return 1; + } + else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) { + return -1; + } + + leftIndex1++; + rightIndex1++; + } + + /* The integer part of both the decimal's are equal, now compare + * each individual fractional part. Set the index to be at the + * beginning of the fractional part + */ + leftIndex1 = leftStopIndex; + rightIndex1 = rightStopIndex; + + /* Stop indexes will be the end of the array */ + leftStopIndex = nDecimalDigits; + rightStopIndex = nDecimalDigits; + + /* compare the two fractional parts of the decimal */ + while (leftIndex1 < leftStopIndex && rightIndex1 < rightStopIndex) { + if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) > getIntegerFromSparseBuffer(right, rightStart, rightIndex1)) { + return 1; + } + else if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) > getIntegerFromSparseBuffer(left, leftStart, leftIndex1)) { + return -1; + } + + leftIndex1++; + rightIndex1++; + } + + /* Till now the fractional part of the decimals are equal, check + * if one of the decimal has fractional part that is remaining + * and is non-zero + */ + while (leftIndex1 < leftStopIndex) { + if (getIntegerFromSparseBuffer(left, leftStart, leftIndex1) != 0) { + return 1; + } + leftIndex1++; + } + + while(rightIndex1 < rightStopIndex) { + if (getIntegerFromSparseBuffer(right, rightStart, rightIndex1) != 0) { + return -1; + } + rightIndex1++; + } + + /* Both decimal values are equal */ + return 0; + } + + public static BigDecimal getBigDecimalFromByteArray(byte[] bytes, int start, int length, int scale) { + byte[] value = Arrays.copyOfRange(bytes, start, start + length); + BigInteger unscaledValue = new BigInteger(value); + return new BigDecimal(unscaledValue, scale); + } + + public static void roundDecimal(DrillBuf result, int start, int nDecimalDigits, int desiredScale, int currentScale) { + int newScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(desiredScale); + int origScaleRoundedUp = org.apache.drill.exec.util.DecimalUtility.roundUp(currentScale); + + if (desiredScale < currentScale) { + + boolean roundUp = false; + + //Extract the first digit to be truncated to check if we need to round up + int truncatedScaleIndex = desiredScale + 1; + if (truncatedScaleIndex <= currentScale) { + int extractDigitIndex = nDecimalDigits - origScaleRoundedUp -1; + extractDigitIndex += org.apache.drill.exec.util.DecimalUtility.roundUp(truncatedScaleIndex); + int extractDigit = getIntegerFromSparseBuffer(result, start, extractDigitIndex); + int temp = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (truncatedScaleIndex % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS); + if (temp != 0) { + extractDigit = extractDigit / (int) (Math.pow(10, temp)); + } + if ((extractDigit % 10) > 4) { + roundUp = true; + } + } + + // Get the source index beyond which we will truncate + int srcIntIndex = nDecimalDigits - origScaleRoundedUp - 1; + int srcIndex = srcIntIndex + newScaleRoundedUp; + + // Truncate the remaining fractional part, move the integer part + int destIndex = nDecimalDigits - 1; + if (srcIndex != destIndex) { + while (srcIndex >= 0) { + setInteger(result, start, destIndex--, getIntegerFromSparseBuffer(result, start, srcIndex--)); + } + + // Set the remaining portion of the decimal to be zeroes + while (destIndex >= 0) { + setInteger(result, start, destIndex--, 0); + } + srcIndex = nDecimalDigits - 1; + } + + // We truncated the decimal digit. Now we need to truncate within the base 1 billion fractional digit + int truncateFactor = org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS - (desiredScale % org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS); + if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) { + truncateFactor = (int) Math.pow(10, truncateFactor); + int fractionalDigits = getIntegerFromSparseBuffer(result, start, nDecimalDigits - 1); + fractionalDigits /= truncateFactor; + setInteger(result, start, nDecimalDigits - 1, fractionalDigits * truncateFactor); + } + + // Finally round up the digit if needed + if (roundUp == true) { + srcIndex = nDecimalDigits - 1; + int carry; + if (truncateFactor != org.apache.drill.exec.util.DecimalUtility.MAX_DIGITS) { + carry = truncateFactor; + } else { + carry = 1; + } + + while (srcIndex >= 0) { + int value = getIntegerFromSparseBuffer(result, start, srcIndex); + value += carry; + + if (value >= org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE) { + setInteger(result, start, srcIndex--, value % org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE); + carry = value / org.apache.drill.exec.util.DecimalUtility.DIGITS_BASE; + } else { + setInteger(result, start, srcIndex--, value); + carry = 0; + break; + } + } + } + } else if (desiredScale > currentScale) { + // Add fractional digits to the decimal + + // Check if we need to shift the decimal digits to the left + if (newScaleRoundedUp > origScaleRoundedUp) { + int srcIndex = 0; + int destIndex = newScaleRoundedUp - origScaleRoundedUp; + + // Check while extending scale, we are not overwriting integer part + while (srcIndex < destIndex) { + if (getIntegerFromSparseBuffer(result, start, srcIndex++) != 0) { + throw new org.apache.drill.common.exceptions.DrillRuntimeException("Truncate resulting in loss of integer part, reduce scale specified"); + } + } + + srcIndex = 0; + while (destIndex < nDecimalDigits) { + setInteger(result, start, srcIndex++, getIntegerFromSparseBuffer(result, start, destIndex++)); + } + + // Clear the remaining part + while (srcIndex < nDecimalDigits) { + setInteger(result, start, srcIndex++, 0); + } + } + } + } + + public static int getFirstFractionalDigit(int decimal, int scale) { + if (scale == 0) { + return 0; + } + int temp = (int) adjustScaleDivide(decimal, scale - 1); + return Math.abs(temp % 10); + } + + public static int getFirstFractionalDigit(long decimal, int scale) { + if (scale == 0) { + return 0; + } + long temp = adjustScaleDivide(decimal, scale - 1); + return (int) (Math.abs(temp % 10)); + } + + public static int getFirstFractionalDigit(DrillBuf data, int scale, int start, int nDecimalDigits) { + if (scale == 0) { + return 0; + } + + int index = nDecimalDigits - roundUp(scale); + return (int) (adjustScaleDivide(data.getInt(start + (index * integerSize)), MAX_DIGITS - 1)); + } +} +
http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java index 7d0fbc7..b711c66 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseDataValueVector.java @@ -17,9 +17,10 @@ */ package org.apache.drill.exec.vector; -import java.util.Iterator; +import io.netty.buffer.DrillBuf; +import io.netty.buffer.DrillBuf; -import io.netty.buffer.ByteBuf; +import java.util.Iterator; import org.apache.drill.exec.memory.BufferAllocator; import org.apache.drill.exec.proto.UserBitShared.SerializedField; @@ -30,7 +31,7 @@ import com.google.common.collect.Iterators; public abstract class BaseDataValueVector extends BaseValueVector{ - protected ByteBuf data = DeadBuf.DEAD_BUFFER; + protected DrillBuf data; protected int valueCount; protected int currentValueCount; @@ -40,13 +41,13 @@ public abstract class BaseDataValueVector extends BaseValueVector{ } /** - * Release the underlying ByteBuf and reset the ValueVector + * Release the underlying DrillBuf and reset the ValueVector */ @Override public void clear() { - if (data != DeadBuf.DEAD_BUFFER) { + if (data != null) { data.release(); - data = DeadBuf.DEAD_BUFFER; + data = data.getAllocator().getEmpty(); valueCount = 0; } } @@ -61,12 +62,12 @@ public abstract class BaseDataValueVector extends BaseValueVector{ @Override - public ByteBuf[] getBuffers(){ - ByteBuf[] out; + public DrillBuf[] getBuffers(){ + DrillBuf[] out; if(valueCount == 0){ - out = new ByteBuf[0]; + out = new DrillBuf[0]; }else{ - out = new ByteBuf[]{data}; + out = new DrillBuf[]{data}; data.readerIndex(0); data.retain(); } @@ -82,10 +83,14 @@ public abstract class BaseDataValueVector extends BaseValueVector{ @Override public abstract SerializedField getMetadata(); - public ByteBuf getData(){ + public DrillBuf getData(){ return data; } + public long getDataAddr(){ + return data.memoryAddress(); + } + @Override public Iterator<ValueVector> iterator() { return Iterators.emptyIterator(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java index e310b81..cf62d54 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BaseValueVector.java @@ -63,7 +63,7 @@ public abstract class BaseValueVector implements ValueVector{ abstract public ByteBuf getData(); - abstract class BaseAccessor implements ValueVector.Accessor{ + abstract static class BaseAccessor implements ValueVector.Accessor{ public abstract int getValueCount(); public void reset(){} } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java index 01fcfe0..0076a1d 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/BitVector.java @@ -17,8 +17,7 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.AccountingByteBuf; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import org.apache.drill.common.expression.FieldReference; import org.apache.drill.exec.expr.holders.BitHolder; @@ -113,11 +112,11 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe } @Override - public int load(int valueCount, ByteBuf buf) { + public int load(int valueCount, DrillBuf buf) { clear(); this.valueCount = valueCount; int len = getSizeFromCount(valueCount); - data = buf.slice(0, len); + data = (DrillBuf) buf.slice(0, len); data.retain(); return len; } @@ -136,7 +135,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe } @Override - public void load(SerializedField metadata, ByteBuf buffer) { + public void load(SerializedField metadata, DrillBuf buffer) { assert this.field.matches(metadata); int loaded = load(metadata.getValueCount(), buffer); assert metadata.getBufferLength() == loaded; @@ -181,7 +180,7 @@ public final class BitVector extends BaseDataValueVector implements FixedWidthVe int offset = startIndex % 8; if (offset == 0) { // slice - target.data = this.data.slice(firstByte, byteSize); + target.data = (DrillBuf) this.data.slice(firstByte, byteSize); target.data.retain(); } else { // Copy data http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java index c5e1ae7..b546b9e 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/FixedWidthVector.java @@ -17,26 +17,26 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; public interface FixedWidthVector extends ValueVector{ - + /** * Allocate a new memory space for this vector. Must be called prior to using the ValueVector. * * @param valueCount Number of values in the vector. */ public void allocateNew(int valueCount); - + /** * Load the records in the provided buffer based on the given number of values. * @param valueCount Number of values the buffer contains. * @param buf Incoming buffer. * @return The number of bytes of the buffer that were consumed. */ - public int load(int valueCount, ByteBuf buf); - - + public int load(int valueCount, DrillBuf buf); + + public abstract Mutator getMutator(); /** http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java index 650f5ae..2c215ff 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ObjectVector.java @@ -18,6 +18,8 @@ package org.apache.drill.exec.vector; import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; + import org.apache.drill.common.expression.FieldReference; import org.apache.drill.exec.expr.holders.ObjectHolder; import org.apache.drill.exec.memory.BufferAllocator; @@ -164,12 +166,12 @@ public class ObjectVector extends BaseValueVector{ } @Override - public ByteBuf[] getBuffers() { + public DrillBuf[] getBuffers() { throw new UnsupportedOperationException("ObjectVector does not support this"); } @Override - public void load(UserBitShared.SerializedField metadata, ByteBuf buffer) { + public void load(UserBitShared.SerializedField metadata, DrillBuf buffer) { throw new UnsupportedOperationException("ObjectVector does not support this"); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java index 6d03541..a28caf1 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedFixedWidthVector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; public interface RepeatedFixedWidthVector extends ValueVector{ /** @@ -27,7 +27,7 @@ public interface RepeatedFixedWidthVector extends ValueVector{ * @param childValueCount Number of supported values in the vector. */ public void allocateNew(int parentValueCount, int childValueCount); - + /** * Load the records in the provided buffer based on the given number of values. * @param parentValueCount Number of separate repeating groupings. @@ -35,7 +35,7 @@ public interface RepeatedFixedWidthVector extends ValueVector{ * @param buf Incoming buffer. * @return The number of bytes of the buffer that were consumed. */ - public int load(int parentValueCount, int childValueCount, ByteBuf buf); + public int load(int parentValueCount, int childValueCount, DrillBuf buf); public abstract RepeatedAccessor getAccessor(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java index a2c884e..2e7689c 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/RepeatedVariableWidthVector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; public interface RepeatedVariableWidthVector extends ValueVector{ /** @@ -45,5 +45,5 @@ public interface RepeatedVariableWidthVector extends ValueVector{ * @param buf Incoming buffer. * @return The number of bytes of the buffer that were consumed. */ - public int load(int dataBytes, int parentValueCount, int childValueCount, ByteBuf buf); + public int load(int dataBytes, int parentValueCount, int childValueCount, DrillBuf buf); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java index 6968c21..31afb6f 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueHolderHelper.java @@ -17,20 +17,18 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.SwappedByteBuf; -import io.netty.buffer.Unpooled; -import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.buffer.DrillBuf; import java.math.BigDecimal; -import java.nio.ByteOrder; -import org.apache.drill.common.util.DecimalUtility; import org.apache.drill.exec.expr.holders.Decimal18Holder; import org.apache.drill.exec.expr.holders.Decimal28SparseHolder; import org.apache.drill.exec.expr.holders.Decimal38SparseHolder; import org.apache.drill.exec.expr.holders.Decimal9Holder; import org.apache.drill.exec.expr.holders.IntervalDayHolder; import org.apache.drill.exec.expr.holders.VarCharHolder; +import org.apache.drill.exec.memory.BufferAllocator; +import org.apache.drill.exec.util.DecimalUtility; import com.google.common.base.Charsets; @@ -38,13 +36,24 @@ import com.google.common.base.Charsets; public class ValueHolderHelper { static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ValueHolderHelper.class); - public static VarCharHolder getVarCharHolder(String s){ + public static VarCharHolder getVarCharHolder(DrillBuf buf, String s){ VarCharHolder vch = new VarCharHolder(); byte[] b = s.getBytes(Charsets.UTF_8); vch.start = 0; vch.end = b.length; - vch.buffer = UnpooledByteBufAllocator.DEFAULT.buffer(b.length).order(ByteOrder.LITTLE_ENDIAN); // use the length of input string to allocate buffer. + vch.buffer = buf.reallocIfNeeded(b.length); + vch.buffer.setBytes(0, b); + return vch; + } + + public static VarCharHolder getVarCharHolder(BufferAllocator a, String s){ + VarCharHolder vch = new VarCharHolder(); + + byte[] b = s.getBytes(Charsets.UTF_8); + vch.start = 0; + vch.end = b.length; + vch.buffer = a.buffer(b.length); // vch.buffer.setBytes(0, b); return vch; } @@ -53,7 +62,7 @@ public class ValueHolderHelper { IntervalDayHolder dch = new IntervalDayHolder(); dch.days = days; - dch.milliSeconds = millis; + dch.milliseconds = millis; return dch; } @@ -77,7 +86,7 @@ public class ValueHolderHelper { return dch; } - public static Decimal28SparseHolder getDecimal28Holder(String decimal) { + public static Decimal28SparseHolder getDecimal28Holder(DrillBuf buf, String decimal) { Decimal28SparseHolder dch = new Decimal28SparseHolder(); @@ -85,18 +94,15 @@ public class ValueHolderHelper { dch.scale = bigDecimal.scale(); dch.precision = bigDecimal.precision(); - dch.setSign(bigDecimal.signum() == -1); + Decimal28SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer); dch.start = 0; - - dch.buffer = Unpooled.wrappedBuffer(new byte[5 * DecimalUtility.integerSize]); - dch.buffer = new SwappedByteBuf(dch.buffer); + dch.buffer = buf.reallocIfNeeded(5 * DecimalUtility.integerSize); DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits); return dch; } - public static Decimal38SparseHolder getDecimal38Holder(String decimal) { - + public static Decimal38SparseHolder getDecimal38Holder(DrillBuf buf, String decimal) { Decimal38SparseHolder dch = new Decimal38SparseHolder(); @@ -104,12 +110,9 @@ public class ValueHolderHelper { dch.scale = bigDecimal.scale(); dch.precision = bigDecimal.precision(); - dch.setSign(bigDecimal.signum() == -1); + Decimal38SparseHolder.setSign(bigDecimal.signum() == -1, dch.start, dch.buffer); dch.start = 0; - - - dch.buffer = Unpooled.wrappedBuffer(new byte[dch.maxPrecision * DecimalUtility.integerSize]); - dch.buffer = new SwappedByteBuf(dch.buffer); + dch.buffer = buf.reallocIfNeeded(dch.maxPrecision * DecimalUtility.integerSize); DecimalUtility.getSparseFromBigDecimal(bigDecimal, dch.buffer, dch.start, dch.scale, dch.precision, dch.nDecimalDigits); return dch; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java index 8b871fc..f7f010a 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/ValueVector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import java.io.Closeable; @@ -56,7 +56,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { public void close(); /** - * Release the underlying ByteBuf and reset the ValueVector to empty. + * Release the underlying DrillBuf and reset the ValueVector to empty. */ public void clear(); @@ -99,9 +99,9 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { * this buffer so it only should be used for in-context access. Also note that this buffer changes regularly thus * external classes shouldn't hold a reference to it (unless they change it). * - * @return The underlying ByteBuf. + * @return The underlying DrillBuf. */ - public abstract ByteBuf[] getBuffers(); + public abstract DrillBuf[] getBuffers(); /** * Load the data provided in the buffer. Typically used when deserializing from the wire. @@ -111,7 +111,7 @@ public interface ValueVector extends Closeable, Iterable<ValueVector> { * @param buffer * The buffer that contains the ValueVector. */ - public void load(SerializedField metadata, ByteBuf buffer); + public void load(SerializedField metadata, DrillBuf buffer); /** * Get the metadata for this field. Used in serialization http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java index 5c1d3ab..ff3ee63 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VariableWidthVector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; public interface VariableWidthVector extends ValueVector{ @@ -28,15 +28,13 @@ public interface VariableWidthVector extends ValueVector{ * @param valueCount Number of values in the vector. */ public void allocateNew(int totalBytes, int valueCount); - + /** * Provide the maximum amount of variable width bytes that can be stored int his vector. * @return */ public int getByteCapacity(); - public int getCurrentSizeInBytes(); - /** * Load the records in the provided buffer based on the given number of values. * @param dataBytes The number of bytes associated with the data array. @@ -44,8 +42,8 @@ public interface VariableWidthVector extends ValueVector{ * @param buf Incoming buffer. * @return The number of bytes of the buffer that were consumed. */ - public int load(int dataBytes, int valueCount, ByteBuf buf); - + public int load(int dataBytes, int valueCount, DrillBuf buf); + public abstract VariableWidthMutator getMutator(); public abstract VariableWidthAccessor getAccessor(); @@ -54,6 +52,8 @@ public interface VariableWidthVector extends ValueVector{ public int getValueLength(int index); } + public int getCurrentSizeInBytes(); + public interface VariableWidthMutator extends Mutator { public boolean setValueLengthSafe(int index, int length); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java index e946afe..619dd95 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/VectorTrimmer.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.vector; -import io.netty.buffer.AccountingByteBuf; +import io.netty.buffer.DrillBuf; import io.netty.buffer.ByteBuf; public class VectorTrimmer { @@ -25,7 +25,7 @@ public class VectorTrimmer { public static void trim(ByteBuf data, int idx) { data.writerIndex(idx); - if (data instanceof AccountingByteBuf) { + if (data instanceof DrillBuf) { // data.capacity(idx); data.writerIndex(idx); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java index 480b863..c91c397 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/MapVector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.vector.complex; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import java.util.HashMap; import java.util.Iterator; @@ -276,18 +276,18 @@ public class MapVector extends AbstractContainerVector { } @Override - public ByteBuf[] getBuffers() { - List<ByteBuf> bufs = Lists.newArrayList(); + public DrillBuf[] getBuffers() { + List<DrillBuf> bufs = Lists.newArrayList(); for(ValueVector v : vectors.values()){ - for(ByteBuf b : v.getBuffers()){ + for(DrillBuf b : v.getBuffers()){ bufs.add(b); } } - return bufs.toArray(new ByteBuf[bufs.size()]); + return bufs.toArray(new DrillBuf[bufs.size()]); } @Override - public void load(SerializedField metadata, ByteBuf buf) { + public void load(SerializedField metadata, DrillBuf buf) { List<SerializedField> fields = metadata.getChildList(); valueCount = metadata.getValueCount(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java index 57c47d4..00716cf 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedListVector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.vector.complex; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import java.util.Collections; import java.util.Iterator; @@ -354,7 +354,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea } @Override - public ByteBuf[] getBuffers() { + public DrillBuf[] getBuffers() { return ArrayUtils.addAll(offsets.getBuffers(), vector.getBuffers()); } @@ -364,7 +364,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea } @Override - public void load(SerializedField metadata, ByteBuf buf) { + public void load(SerializedField metadata, DrillBuf buf) { SerializedField childField = metadata.getChildList().get(0); int bufOffset = offsets.load(metadata.getValueCount()+1, buf); @@ -429,7 +429,7 @@ public class RepeatedListVector extends AbstractContainerVector implements Repea } @Override - public int load(int parentValueCount, int childValueCount, ByteBuf buf) { + public int load(int parentValueCount, int childValueCount, DrillBuf buf) { throw new UnsupportedOperationException(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java index c67c047..f7baaa9 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/RepeatedMapVector.java @@ -17,7 +17,7 @@ */ package org.apache.drill.exec.vector.complex; -import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import java.util.Iterator; import java.util.List; @@ -307,20 +307,20 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat } @Override - public ByteBuf[] getBuffers() { - List<ByteBuf> bufs = Lists.newArrayList(offsets.getBuffers()); + public DrillBuf[] getBuffers() { + List<DrillBuf> bufs = Lists.newArrayList(offsets.getBuffers()); for(ValueVector v : vectors.values()){ - for(ByteBuf b : v.getBuffers()){ + for(DrillBuf b : v.getBuffers()){ bufs.add(b); } } - return bufs.toArray(new ByteBuf[bufs.size()]); + return bufs.toArray(new DrillBuf[bufs.size()]); } @Override - public void load(SerializedField metadata, ByteBuf buf) { + public void load(SerializedField metadata, DrillBuf buf) { List<SerializedField> fields = metadata.getChildList(); int bufOffset = offsets.load(metadata.getValueCount()+1, buf); @@ -513,7 +513,7 @@ public class RepeatedMapVector extends AbstractContainerVector implements Repeat } @Override - public int load(int parentValueCount, int childValueCount, ByteBuf buf) { + public int load(int parentValueCount, int childValueCount, DrillBuf buf) { throw new UnsupportedOperationException(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java index d365c6f..59035cb 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReader.java @@ -17,9 +17,7 @@ */ package org.apache.drill.exec.vector.complex.fn; -import io.netty.buffer.ByteBuf; -import io.netty.buffer.ByteBufInputStream; -import io.netty.buffer.UnpooledByteBufAllocator; +import io.netty.buffer.DrillBuf; import java.io.IOException; import java.io.Reader; @@ -33,11 +31,10 @@ import org.apache.drill.exec.vector.complex.writer.BaseWriter.ListWriter; import org.apache.drill.exec.vector.complex.writer.BaseWriter.MapWriter; import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonGenerator; import com.fasterxml.jackson.core.JsonParseException; import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.core.JsonToken; import com.fasterxml.jackson.core.JsonParser.Feature; +import com.fasterxml.jackson.core.JsonToken; import com.google.common.base.Charsets; public class JsonReader { @@ -48,10 +45,12 @@ public class JsonReader { private final JsonFactory factory = new JsonFactory(); private JsonParser parser; + private DrillBuf workBuf; - public JsonReader() throws JsonParseException, IOException { + public JsonReader(DrillBuf managedBuf) throws JsonParseException, IOException { factory.configure(Feature.ALLOW_UNQUOTED_FIELD_NAMES, true); factory.configure(Feature.ALLOW_COMMENTS, true); + this.workBuf = managedBuf; } public boolean write(Reader reader, ComplexWriter writer) throws JsonParseException, IOException { @@ -134,9 +133,9 @@ public class JsonReader { VarCharHolder vh = new VarCharHolder(); String value = parser.getText(); byte[] b = value.getBytes(Charsets.UTF_8); - ByteBuf d = UnpooledByteBufAllocator.DEFAULT.buffer(b.length); - d.setBytes(0, b); - vh.buffer = d; + ensure(b.length); + workBuf.setBytes(0, b); + vh.buffer = workBuf; vh.start = 0; vh.end = b.length; map.varChar(fieldName).write(vh); @@ -152,6 +151,10 @@ public class JsonReader { } + private void ensure(int length){ + workBuf = workBuf.reallocIfNeeded(length); + } + private void writeData(ListWriter list) throws JsonParseException, IOException { list.start(); outside: while(true){ @@ -197,9 +200,9 @@ public class JsonReader { VarCharHolder vh = new VarCharHolder(); String value = parser.getText(); byte[] b = value.getBytes(Charsets.UTF_8); - ByteBuf d = UnpooledByteBufAllocator.DEFAULT.buffer(b.length); - d.setBytes(0, b); - vh.buffer = d; + ensure(b.length); + workBuf.setBytes(0, b); + vh.buffer = workBuf; vh.start = 0; vh.end = b.length; list.varChar().write(vh); @@ -209,7 +212,7 @@ public class JsonReader { } } list.end(); - - + + } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java index cf885a4..54e063b 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/vector/complex/fn/JsonReaderWithState.java @@ -18,6 +18,8 @@ package org.apache.drill.exec.vector.complex.fn; +import io.netty.buffer.DrillBuf; + import java.io.IOException; import java.io.Reader; @@ -35,10 +37,10 @@ public class JsonReaderWithState { private JsonRecordSplitter splitter; private JsonReader jsonReader; - public JsonReaderWithState(JsonRecordSplitter splitter) throws IOException{ + public JsonReaderWithState(JsonRecordSplitter splitter, DrillBuf workspace) throws IOException{ this.splitter = splitter; reader = splitter.getNextReader(); - jsonReader = new JsonReader(); + jsonReader = new JsonReader(workspace); } public WriteState write(ComplexWriter writer) throws JsonParseException, IOException { @@ -50,7 +52,7 @@ public class JsonReaderWithState { } jsonReader.write(reader, writer); - + if (!writer.ok()) { reader.reset(); return WriteState.WRITE_FAILED; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java index 76d642c..c7527d5 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/batch/SpoolingRawBatchBuffer.java @@ -18,6 +18,7 @@ package org.apache.drill.exec.work.batch; import io.netty.buffer.ByteBuf; +import io.netty.buffer.DrillBuf; import java.io.IOException; import java.util.List; @@ -36,10 +37,6 @@ import org.apache.drill.exec.proto.BitData; import org.apache.drill.exec.proto.ExecProtos; import org.apache.drill.exec.proto.helper.QueryIdHelper; import org.apache.drill.exec.record.RawFragmentBatch; -import org.apache.drill.exec.rpc.RemoteConnection; -import org.apache.drill.exec.rpc.ResponseSender; -import org.apache.drill.exec.rpc.data.BitServerConnection; -import org.apache.drill.exec.rpc.data.DataRpcConfig; import org.apache.drill.exec.store.LocalSyncableFileSystem; import org.apache.drill.exec.work.fragment.FragmentManager; import org.apache.hadoop.conf.Configuration; @@ -272,7 +269,7 @@ public class SpoolingRawBatchBuffer implements RawBatchBuffer { Stopwatch watch = new Stopwatch(); watch.start(); BitData.FragmentRecordBatch header = BitData.FragmentRecordBatch.parseDelimitedFrom(stream); - ByteBuf buf = allocator.buffer(bodyLength); + DrillBuf buf = allocator.buffer(bodyLength); buf.writeBytes(stream, bodyLength); batch = new RawFragmentBatch(null, header, buf, null); buf.release(); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java b/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java new file mode 100644 index 0000000..4258e60 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/RunRootExec.java @@ -0,0 +1,74 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec; + + +import java.io.File; +import java.util.concurrent.TimeUnit; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; +import org.apache.drill.exec.ops.FragmentContext; +import org.apache.drill.exec.physical.PhysicalPlan; +import org.apache.drill.exec.physical.base.FragmentRoot; +import org.apache.drill.exec.physical.impl.ImplCreator; +import org.apache.drill.exec.physical.impl.SimpleRootExec; +import org.apache.drill.exec.planner.PhysicalPlanReader; +import org.apache.drill.exec.proto.BitControl.PlanFragment; +import org.apache.drill.exec.server.Drillbit; +import org.apache.drill.exec.server.DrillbitContext; +import org.apache.drill.exec.server.RemoteServiceSet; +import org.apache.drill.exec.vector.ValueVector; + +import com.google.common.base.Charsets; +import com.google.common.base.Stopwatch; +import com.google.common.io.Files; + +public class RunRootExec { + public static DrillConfig c = DrillConfig.create(); + + public static void main(String args[]) throws Exception { + String path = args[0]; + int iterations = Integer.parseInt(args[1]); + Drillbit bit = new Drillbit(c, RemoteServiceSet.getLocalServiceSet()); + bit.run(); + DrillbitContext bitContext = bit.getContext(); + PhysicalPlanReader reader = bitContext.getPlanReader(); + PhysicalPlan plan = reader.readPhysicalPlan(Files.toString(new File(path), Charsets.UTF_8)); + FunctionImplementationRegistry registry = bitContext.getFunctionImplementationRegistry(); + FragmentContext context = new FragmentContext(bitContext, PlanFragment.getDefaultInstance(), null, registry); + SimpleRootExec exec; + for (int i = 0; i < iterations; i ++) { + Stopwatch w= new Stopwatch().start(); + System.out.println("STARTITER:" + i); + exec = new SimpleRootExec(ImplCreator.getExec(context, (FragmentRoot) plan.getSortedOperators(false).iterator().next())); + + while (exec.next()) { + for (ValueVector v : exec) { + v.clear(); + } + } + System.out.println("ENDITER: " + i); + System.out.println("TIME: " + w.elapsed(TimeUnit.MILLISECONDS) + "ms"); + exec.stop(); + } + context.close(); + bit.close(); + } + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java index 9090685..e678c29 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/client/DumpCatTest.java @@ -28,6 +28,7 @@ import org.apache.drill.common.config.DrillConfig; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; import org.apache.drill.exec.ExecConstants; +import org.apache.drill.exec.compile.CodeCompiler; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.ops.FragmentContext; @@ -70,6 +71,7 @@ public class DumpCatTest extends ExecTest{ bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getConfig(); result = c; + bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); }}; @@ -87,6 +89,8 @@ public class DumpCatTest extends ExecTest{ } assertTrue(!context.isFailed()); + exec.stop(); + FragmentHandle handle = context.getHandle(); /* Form the file name to which the trace output will dump the record batches */ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java index e1fefdf..896cf4d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestClassTransformation.java @@ -20,6 +20,7 @@ package org.apache.drill.exec.compile; import java.io.IOException; import org.apache.drill.BaseTestQuery; +import org.apache.drill.exec.cache.local.LocalCache; import org.apache.drill.exec.compile.ClassTransformer.ClassSet; import org.apache.drill.exec.compile.sig.GeneratorMapping; import org.apache.drill.exec.compile.sig.MappingSet; @@ -72,7 +73,7 @@ public class TestClassTransformation extends BaseTestQuery { public void testCompilationNoDebug() throws CompileException, ClassNotFoundException, ClassTransformationException, IOException { CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class); ClassSet classSet = new ClassSet(null, cg.getDefinition().getTemplateClassName(), cg.getMaterializedClassName()); - String sourceCode = cg.generate(); + String sourceCode = cg.generateAndGet(); sessionOptions.setOption(OptionValue.createString(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_OPTION, QueryClassLoader.CompilerPolicy.JDK.name())); sessionOptions.setOption(OptionValue.createBoolean(OptionType.SESSION, QueryClassLoader.JAVA_COMPILER_DEBUG_OPTION, false)); @@ -104,9 +105,9 @@ public class TestClassTransformation extends BaseTestQuery { private void compilationInnerClass(QueryClassLoader loader) throws Exception{ CodeGenerator<ExampleInner> cg = newCodeGenerator(ExampleInner.class, ExampleTemplateWithInner.class); - ClassTransformer ct = new ClassTransformer(); - ExampleInner t = ct.getImplementationClass(loader, cg.getDefinition(), cg.generate(), cg.getMaterializedClassName()); - + ClassTransformer ct = new ClassTransformer(new LocalCache()); + Class<? extends ExampleInner> c = (Class<? extends ExampleInner>) ct.getImplementationClass(loader, cg.getDefinition(), cg.generateAndGet(), cg.getMaterializedClassName()); + ExampleInner t = (ExampleInner) c.newInstance(); t.doOutside(); t.doInsideOutside(); } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java index 4d0cbcd..2265347 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/TestEvaluationVisitor.java @@ -69,7 +69,7 @@ public class TestEvaluationVisitor { ValueVectorWriteExpression e2 = new ValueVectorWriteExpression(outId, e, true); v.addExpr(e2, g.getRoot()); - System.out.println(g.generate()); + System.out.println(g.generateAndGet()); } private LogicalExpression getExpr(String expr) throws Exception{ http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java new file mode 100644 index 0000000..c7467a0 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/compile/bytecode/ReplaceMethodInvoke.java @@ -0,0 +1,105 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.compile.bytecode; + +import java.io.File; +import java.io.PrintWriter; +import java.io.StringWriter; +import java.net.URL; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.compile.QueryClassLoader; +import org.apache.drill.exec.server.options.SystemOptionManager; +import org.apache.drill.exec.store.sys.local.LocalPStoreProvider; +import org.objectweb.asm.ClassReader; +import org.objectweb.asm.ClassVisitor; +import org.objectweb.asm.ClassWriter; +import org.objectweb.asm.util.ASMifier; +import org.objectweb.asm.util.CheckClassAdapter; +import org.objectweb.asm.util.Textifier; +import org.objectweb.asm.util.TraceClassVisitor; + +import com.google.common.io.Files; +import com.google.common.io.Resources; + +public class ReplaceMethodInvoke { + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(ReplaceMethodInvoke.class); + + @SuppressWarnings("unchecked") + public static void main(String[] args) throws Exception{ + String e = "org/apache/drill/ExampleReplaceable.class"; + String h = "org/apache/drill/exec/physical/impl/aggregate/HashAggTemplate.class"; + String k = "org/apache/drill/Pickle$OutgoingBatch.class"; + String k2 = "org/apache/drill/Pickle.class"; + String r = "org/apache/drill/exec/test/generated/FiltererGen0.class"; + String p = "org/apache/drill/exec/test/generated/PartitionerGen4.class"; + URL url = Resources.getResource(k2); + byte[] clazz = Resources.toByteArray(url); + ClassReader cr = new ClassReader(clazz); + + ClassWriter cw = writer(); + TraceClassVisitor visitor = new TraceClassVisitor(cw, new Textifier(), new PrintWriter(System.out)); + //getTracer(false) + ValueHolderReplacementVisitor v2 = new ValueHolderReplacementVisitor(visitor); + cr.accept(v2, ClassReader.EXPAND_FRAMES );//| ClassReader.SKIP_DEBUG); + + byte[] output = cw.toByteArray(); + Files.write(output, new File("/src/scratch/bytes/S.class")); + check(output); + + + DrillConfig c = DrillConfig.createClient(); + SystemOptionManager m = new SystemOptionManager(c, new LocalPStoreProvider(c)); + m.init(); + QueryClassLoader ql = new QueryClassLoader(DrillConfig.create(), m); + ql.injectByteCode("org.apache.drill.Pickle$OutgoingBatch", output); + Class<?> clz = ql.loadClass("org.apache.drill.Pickle$OutgoingBatch"); + clz.getMethod("x").invoke(null); + + } + + + private static final void check(byte[] b) { + ClassReader cr = new ClassReader(b); + ClassWriter cw = writer(); + ClassVisitor cv = new CheckClassAdapter(cw); + cr.accept(cv, 0); + + StringWriter sw = new StringWriter(); + PrintWriter pw = new PrintWriter(sw); + CheckClassAdapter.verify(new ClassReader(cw.toByteArray()), false, pw); + + assert sw.toString().length() == 0 : sw.toString(); + } + + private static ClassWriter writer() { + ClassWriter cw = new ClassWriter(ClassWriter.COMPUTE_FRAMES); + return cw; + } + + private static ClassVisitor getTracer(boolean asm) { + if (asm) { + return new TraceClassVisitor(null, new ASMifier(), new PrintWriter(System.out)); + } else { + return new TraceClassVisitor(null, new Textifier(), new PrintWriter(System.out)); + } + } + + + +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java index da82998..392af18 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/expr/ExpressionTest.java @@ -126,7 +126,7 @@ public class ExpressionTest extends ExecTest { ClassGenerator<Projector> cg = CodeGenerator.get(Projector.TEMPLATE_DEFINITION, new FunctionImplementationRegistry(DrillConfig.create())).getRoot(); cg.addExpr(new ValueVectorWriteExpression(new TypedFieldId(materializedExpr.getMajorType(), -1), materializedExpr)); - return cg.getCodeGenerator().generate(); + return cg.getCodeGenerator().generateAndGet(); } } http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java new file mode 100644 index 0000000..632f5a4 --- /dev/null +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestByteComparisonFunctions.java @@ -0,0 +1,129 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.drill.exec.fn.impl; + +import static org.junit.Assert.*; + +import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.ExecTest; +import org.apache.drill.exec.expr.fn.impl.ByteFunctionHelpers; +import org.apache.drill.exec.expr.holders.VarCharHolder; +import org.apache.drill.exec.memory.TopLevelAllocator; +import org.apache.drill.exec.vector.ValueHolderHelper; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +public class TestByteComparisonFunctions extends ExecTest{ + static final org.slf4j.Logger logger = org.slf4j.LoggerFactory.getLogger(TestByteComparisonFunctions.class); + + static TopLevelAllocator allocator; + static VarCharHolder hello; + static VarCharHolder goodbye; + static VarCharHolder helloLong; + static VarCharHolder goodbyeLong; + + @BeforeClass + public static void setup(){ + DrillConfig c= DrillConfig.create(); + allocator = new TopLevelAllocator(c); + hello = ValueHolderHelper.getVarCharHolder(allocator, "hello"); + goodbye = ValueHolderHelper.getVarCharHolder(allocator, "goodbye"); + helloLong = ValueHolderHelper.getVarCharHolder(allocator, "hellomyfriend"); + goodbyeLong = ValueHolderHelper.getVarCharHolder(allocator, "goodbyemyenemy"); + } + + @AfterClass + public static void teardown(){ + hello.buffer.release(); + helloLong.buffer.release(); + goodbye.buffer.release(); + goodbyeLong.buffer.release(); + allocator.close(); + } + + @Test + public void testAfter(){ + VarCharHolder left = hello; + VarCharHolder right = goodbye; + assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1); + } + + @Test + public void testBefore(){ + VarCharHolder left = goodbye; + VarCharHolder right = hello; + assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == -1); + } + + @Test + public void testEqualCompare(){ + VarCharHolder left = hello; + VarCharHolder right = hello; + assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0); + } + + @Test + public void testEqual(){ + VarCharHolder left = hello; + VarCharHolder right = hello; + assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1); + } + + @Test + public void testNotEqual(){ + VarCharHolder left = hello; + VarCharHolder right = goodbye; + assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0); + } + + @Test + public void testAfterLong(){ + VarCharHolder left = helloLong; + VarCharHolder right = goodbyeLong; + assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1); + } + + @Test + public void testBeforeLong(){ + VarCharHolder left = goodbyeLong; + VarCharHolder right = helloLong; + assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == -1); + } + + @Test + public void testEqualCompareLong(){ + VarCharHolder left = helloLong; + VarCharHolder right = helloLong; + assertTrue(ByteFunctionHelpers.compare(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0); + } + + @Test + public void testEqualLong(){ + VarCharHolder left = helloLong; + VarCharHolder right = helloLong; + assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 1); + } + + @Test + public void testNotEqualLong(){ + VarCharHolder left = helloLong; + VarCharHolder right = goodbyeLong; + assertTrue(ByteFunctionHelpers.equal(left.buffer.memoryAddress(), left.start, left.end, right.buffer.memoryAddress(), right.start, right.end) == 0); + } +} http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java index 104923b..4f06a9d 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestMathFunctions.java @@ -28,6 +28,7 @@ import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; +import org.apache.drill.exec.compile.CodeCompiler; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.ops.FragmentContext; @@ -64,6 +65,7 @@ public class TestMathFunctions extends ExecTest { bitContext.getMetrics(); result = new MetricRegistry(); bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getConfig(); result = c; + bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); }}; http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java index 62a224e..3d41f0b 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestNewMathFunctions.java @@ -27,6 +27,7 @@ import mockit.Injectable; import mockit.NonStrictExpectations; import org.apache.drill.common.config.DrillConfig; +import org.apache.drill.exec.compile.CodeCompiler; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.ops.FragmentContext; @@ -83,6 +84,7 @@ public class TestNewMathFunctions { bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); bitContext.getConfig(); result = c; + bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); }}; String planString = Resources.toString(Resources.getResource(planPath), Charsets.UTF_8); http://git-wip-us.apache.org/repos/asf/incubator-drill/blob/5c5cef06/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java index 4345560..fd71dd9 100644 --- a/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java +++ b/exec/java-exec/src/test/java/org/apache/drill/exec/fn/impl/TestRepeatedFunction.java @@ -27,6 +27,7 @@ import org.apache.drill.common.expression.ExpressionPosition; import org.apache.drill.common.expression.SchemaPath; import org.apache.drill.common.util.FileUtils; import org.apache.drill.exec.ExecTest; +import org.apache.drill.exec.compile.CodeCompiler; import org.apache.drill.exec.expr.fn.FunctionImplementationRegistry; import org.apache.drill.exec.memory.TopLevelAllocator; import org.apache.drill.exec.ops.FragmentContext; @@ -64,6 +65,7 @@ public class TestRepeatedFunction extends ExecTest{ bitContext.getAllocator(); result = new TopLevelAllocator(); bitContext.getOperatorCreatorRegistry(); result = new OperatorCreatorRegistry(c); bitContext.getConfig(); result = c; + bitContext.getCompiler(); result = CodeCompiler.getTestCompiler(c); }};
