This is an automated email from the ASF dual-hosted git repository.
cgivre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/drill.git
The following commit(s) were added to refs/heads/master by this push:
new 546fb15 DRILL-7947: Unable to aggregate or filter nullable decimals
in large record sets (#2255)
546fb15 is described below
commit 546fb15ee8b0444d6da91bcc47116084fd6e27b5
Author: Volodymyr Vysotskyi <[email protected]>
AuthorDate: Fri Jun 11 00:49:08 2021 +0300
DRILL-7947: Unable to aggregate or filter nullable decimals in large record
sets (#2255)
---
.../drill/exec/store/avro/AvroColumnConverterFactory.java | 7 ++++---
.../exec/store/parquet/ParquetTableMetadataUtils.java | 15 ++++++++++-----
.../java/org/apache/drill/exec/util/DecimalUtility.java | 6 +++---
3 files changed, 17 insertions(+), 11 deletions(-)
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/AvroColumnConverterFactory.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/AvroColumnConverterFactory.java
index cfd7d50..152d4bd 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/AvroColumnConverterFactory.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/avro/AvroColumnConverterFactory.java
@@ -91,14 +91,15 @@ public class AvroColumnConverterFactory extends
ColumnConverterFactory {
});
case VARDECIMAL:
return new ColumnConverter.ScalarColumnConverter(value -> {
- BigInteger bigInteger;
+ byte[] bytes;
if (value instanceof ByteBuffer) {
ByteBuffer decBuf = (ByteBuffer) value;
- bigInteger = new BigInteger(decBuf.array());
+ bytes = decBuf.array();
} else {
GenericFixed genericFixed = (GenericFixed) value;
- bigInteger = new BigInteger(genericFixed.bytes());
+ bytes = genericFixed.bytes();
}
+ BigInteger bigInteger = bytes.length == 0 ? BigInteger.ZERO : new
BigInteger(bytes);
BigDecimal decimalValue = new BigDecimal(bigInteger,
readerSchema.scale());
writer.setDecimal(decimalValue);
});
diff --git
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java
index 074709d..67a1232 100644
---
a/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java
+++
b/exec/java-exec/src/main/java/org/apache/drill/exec/store/parquet/ParquetTableMetadataUtils.java
@@ -360,7 +360,8 @@ public class ParquetTableMetadataUtils {
case BINARY:
case FIXED_LEN_BYTE_ARRAY:
if (originalType == OriginalType.DECIMAL) {
- return new BigInteger(getBytes(value));
+ byte[] bytes = getBytes(value);
+ return bytes.length == 0 ? BigInteger.ZERO : new BigInteger(bytes);
} else if (originalType == OriginalType.INTERVAL) {
return getBytes(value);
} else {
@@ -402,9 +403,11 @@ public class ParquetTableMetadataUtils {
} else if (value instanceof String) {
return Integer.parseInt(value.toString());
} else if (value instanceof byte[]) {
- return new BigInteger((byte[]) value).intValue();
+ byte[] bytes = (byte[]) value;
+ return bytes.length == 0 ? 0 : new BigInteger(bytes).intValue();
} else if (value instanceof Binary) {
- return new BigInteger(((Binary) value).getBytes()).intValue();
+ byte[] bytes = ((Binary) value).getBytes();
+ return bytes.length == 0 ? 0 : new BigInteger(bytes).intValue();
}
throw new UnsupportedOperationException(String.format("Cannot obtain
Integer using value %s", value));
}
@@ -415,9 +418,11 @@ public class ParquetTableMetadataUtils {
} else if (value instanceof String) {
return Long.parseLong(value.toString());
} else if (value instanceof byte[]) {
- return new BigInteger((byte[]) value).longValue();
+ byte[] bytes = (byte[]) value;
+ return bytes.length == 0 ? 0L : new BigInteger(bytes).longValue();
} else if (value instanceof Binary) {
- return new BigInteger(((Binary) value).getBytes()).longValue();
+ byte[] bytes = ((Binary) value).getBytes();
+ return bytes.length == 0 ? 0L : new BigInteger(bytes).longValue();
}
throw new UnsupportedOperationException(String.format("Cannot obtain
Integer using value %s", value));
}
diff --git
a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
index ce12bf1..de388cb 100644
--- a/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
+++ b/exec/vector/src/main/java/org/apache/drill/exec/util/DecimalUtility.java
@@ -67,7 +67,7 @@ public class DecimalUtility {
public static BigDecimal getBigDecimalFromDrillBuf(DrillBuf bytebuf, int
start, int length, int scale) {
byte[] value = new byte[length];
bytebuf.getBytes(start, value, 0, length);
- BigInteger unscaledValue = new BigInteger(value);
+ BigInteger unscaledValue = length == 0 ? BigInteger.ZERO : new
BigInteger(value);
return new BigDecimal(unscaledValue, scale);
}
@@ -337,9 +337,9 @@ public class DecimalUtility {
* @return 1 if left > right, 0 if left = right, -1 if left < right. two
values that are numerically equal, but with different
* scales (e.g., 2.00 and 2), are considered equal.
*/
- public static int compareVarLenBytes(DrillBuf left, int leftStart, int
leftEnd, int leftScale, byte right[], int rightScale, boolean absCompare) {
+ public static int compareVarLenBytes(DrillBuf left, int leftStart, int
leftEnd, int leftScale, byte[] right, int rightScale, boolean absCompare) {
java.math.BigDecimal bdLeft = getBigDecimalFromDrillBuf(left, leftStart,
leftEnd - leftStart, leftScale);
- java.math.BigDecimal bdRight = new BigDecimal(new BigInteger(right),
rightScale);
+ java.math.BigDecimal bdRight = new BigDecimal(right.length == 0 ?
BigInteger.ZERO : new BigInteger(right), rightScale);
if (absCompare) {
bdLeft = bdLeft.abs();
bdRight = bdRight.abs();
