This is an automated email from the ASF dual-hosted git repository.
agrove pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new fbe86d0c fix: skip negative scale checks for creating decimals (#723)
fbe86d0c is described below
commit fbe86d0c02fcc3c4af9e41cc6348f95f2efa9a4a
Author: KAZUYUKI TANIMURA <[email protected]>
AuthorDate: Thu Aug 1 08:04:18 2024 -0700
fix: skip negative scale checks for creating decimals (#723)
---
.../org/apache/comet/vector/CometDictionary.java | 2 +-
.../java/org/apache/comet/vector/CometVector.java | 51 ++++++++++++++--------
pom.xml | 1 -
3 files changed, 35 insertions(+), 19 deletions(-)
diff --git a/common/src/main/java/org/apache/comet/vector/CometDictionary.java
b/common/src/main/java/org/apache/comet/vector/CometDictionary.java
index fd5c2410..b610d5f3 100644
--- a/common/src/main/java/org/apache/comet/vector/CometDictionary.java
+++ b/common/src/main/java/org/apache/comet/vector/CometDictionary.java
@@ -30,7 +30,7 @@ public class CometDictionary implements AutoCloseable {
private final int numValues;
/** Decoded dictionary values. We only need to copy values for decimal type.
*/
- private ByteArrayWrapper[] binaries;
+ private volatile ByteArrayWrapper[] binaries;
public CometDictionary(CometPlainVector values) {
this.values = values;
diff --git a/common/src/main/java/org/apache/comet/vector/CometVector.java
b/common/src/main/java/org/apache/comet/vector/CometVector.java
index 27f40bac..8b32b39a 100644
--- a/common/src/main/java/org/apache/comet/vector/CometVector.java
+++ b/common/src/main/java/org/apache/comet/vector/CometVector.java
@@ -45,6 +45,19 @@ public abstract class CometVector extends ColumnVector {
private final byte[] DECIMAL_BYTES = new byte[DECIMAL_BYTE_WIDTH];
protected final boolean useDecimal128;
+ private static final long decimalValOffset;
+
+ static {
+ try {
+ java.lang.reflect.Field unsafeField =
sun.misc.Unsafe.class.getDeclaredField("theUnsafe");
+ unsafeField.setAccessible(true);
+ final sun.misc.Unsafe unsafe = (sun.misc.Unsafe) unsafeField.get(null);
+ decimalValOffset =
unsafe.objectFieldOffset(Decimal.class.getDeclaredField("decimalVal"));
+ } catch (Throwable e) {
+ throw new RuntimeException(e);
+ }
+ }
+
protected CometVector(DataType type, boolean useDecimal128) {
super(type);
this.useDecimal128 = useDecimal128;
@@ -73,31 +86,35 @@ public abstract class CometVector extends ColumnVector {
@Override
public Decimal getDecimal(int i, int precision, int scale) {
if (!useDecimal128 && precision <= Decimal.MAX_INT_DIGITS() && type
instanceof IntegerType) {
- return Decimal.createUnsafe(getInt(i), precision, scale);
+ return createDecimal(getInt(i), precision, scale);
} else if (!useDecimal128 && precision <= Decimal.MAX_LONG_DIGITS()) {
- return Decimal.createUnsafe(getLong(i), precision, scale);
+ return createDecimal(getLong(i), precision, scale);
} else {
byte[] bytes = getBinaryDecimal(i);
BigInteger bigInteger = new BigInteger(bytes);
BigDecimal javaDecimal = new BigDecimal(bigInteger, scale);
- try {
- return Decimal.apply(javaDecimal, precision, scale);
- } catch (ArithmeticException e) {
- throw new ArithmeticException(
- "Cannot convert "
- + javaDecimal
- + " (bytes: "
- + bytes
- + ", integer: "
- + bigInteger
- + ") to decimal with precision: "
- + precision
- + " and scale: "
- + scale);
- }
+ return createDecimal(javaDecimal, precision, scale);
}
}
+ /** This method skips the negative scale check, otherwise the same as
Decimal.createUnsafe(). */
+ private Decimal createDecimal(long unscaled, int precision, int scale) {
+ Decimal dec = new Decimal();
+ dec.org$apache$spark$sql$types$Decimal$$longVal_$eq(unscaled);
+ dec.org$apache$spark$sql$types$Decimal$$_precision_$eq(precision);
+ dec.org$apache$spark$sql$types$Decimal$$_scale_$eq(scale);
+ return dec;
+ }
+
+ /** This method skips a few checks, otherwise the same as Decimal.apply(). */
+ private Decimal createDecimal(BigDecimal value, int precision, int scale) {
+ Decimal dec = new Decimal();
+ Platform.putObjectVolatile(dec, decimalValOffset, new
scala.math.BigDecimal(value));
+ dec.org$apache$spark$sql$types$Decimal$$_precision_$eq(precision);
+ dec.org$apache$spark$sql$types$Decimal$$_scale_$eq(scale);
+ return dec;
+ }
+
/**
* Reads a 16-byte byte array which are encoded big-endian for decimal128
into internal byte
* array.
diff --git a/pom.xml b/pom.xml
index 2b766d39..311437cc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -711,7 +711,6 @@ under the License.
</execution>
</executions>
<configuration>
- -->
<scalaVersion>${scala.version}</scalaVersion>
<checkMultipleScalaVersions>true</checkMultipleScalaVersions>
<failOnMultipleScalaVersions>true</failOnMultipleScalaVersions>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
