This is an automated email from the ASF dual-hosted git repository.
jackie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 6ffb9718e4 bugfix: Add missing BIG_DECIMAL support for GenericRow
serde (#9661)
6ffb9718e4 is described below
commit 6ffb9718e40dcc65a50b1c8854904a04a0f241b8
Author: Tim Santos <[email protected]>
AuthorDate: Wed Oct 26 15:35:57 2022 -0700
bugfix: Add missing BIG_DECIMAL support for GenericRow serde (#9661)
---
.../genericrow/GenericRowDeserializer.java | 28 ++++++++++++++++++++++
.../genericrow/GenericRowSerializer.java | 26 +++++++++++++-------
.../processing/genericrow/GenericRowSerDeTest.java | 6 ++++-
3 files changed, 50 insertions(+), 10 deletions(-)
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowDeserializer.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowDeserializer.java
index 338f888b12..8a81c013c6 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowDeserializer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowDeserializer.java
@@ -23,6 +23,7 @@ import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.ByteArray;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -81,6 +82,15 @@ public class GenericRowDeserializer {
buffer.putValue(fieldName, _dataBuffer.getDouble(offset));
offset += Double.BYTES;
break;
+ case BIG_DECIMAL: {
+ int numBytes = _dataBuffer.getInt(offset);
+ offset += Integer.BYTES;
+ byte[] bigDecimalBytes = new byte[numBytes];
+ _dataBuffer.copyTo(offset, bigDecimalBytes);
+ offset += numBytes;
+ buffer.putValue(fieldName,
BigDecimalUtils.deserialize(bigDecimalBytes));
+ break;
+ }
case STRING: {
int numBytes = _dataBuffer.getInt(offset);
offset += Integer.BYTES;
@@ -238,6 +248,24 @@ public class GenericRowDeserializer {
offset2 += numBytes2;
break;
}
+ case BIG_DECIMAL: {
+ int numBytes1 = _dataBuffer.getInt(offset1);
+ offset1 += Integer.BYTES;
+ byte[] bigDecimalBytes1 = new byte[numBytes1];
+ _dataBuffer.copyTo(offset1, bigDecimalBytes1);
+ int numBytes2 = _dataBuffer.getInt(offset2);
+ offset2 += Integer.BYTES;
+ byte[] bigDecimalBytes2 = new byte[numBytes2];
+ _dataBuffer.copyTo(offset2, bigDecimalBytes2);
+ int result =
+
BigDecimalUtils.deserialize(bigDecimalBytes1).compareTo(BigDecimalUtils.deserialize(bigDecimalBytes2));
+ if (result != 0) {
+ return result;
+ }
+ offset1 += numBytes1;
+ offset2 += numBytes2;
+ break;
+ }
default:
throw new IllegalStateException("Unsupported SV stored type: " +
_storedTypes[i]);
}
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerializer.java
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerializer.java
index 527a6660ae..422386018d 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerializer.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerializer.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.segment.processing.genericrow;
+import java.math.BigDecimal;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.HashMap;
@@ -28,6 +29,7 @@ import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.data.readers.GenericRow;
+import org.apache.pinot.spi.utils.BigDecimalUtils;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -42,8 +44,8 @@ public class GenericRowSerializer {
private final String[] _fieldNames;
private final boolean[] _isSingleValueFields;
private final DataType[] _storedTypes;
- // Cache the encoded string bytes
- private final Object[] _stringBytes;
+ // Cache the encoded objects as bytes
+ private final Object[] _objectBytes;
// Store index for null fields
private final Map<String, Integer> _fieldIndexMap;
private final int[] _nullFieldIndexes;
@@ -53,7 +55,7 @@ public class GenericRowSerializer {
_fieldNames = new String[_numFields];
_isSingleValueFields = new boolean[_numFields];
_storedTypes = new DataType[_numFields];
- _stringBytes = new Object[_numFields];
+ _objectBytes = new Object[_numFields];
for (int i = 0; i < _numFields; i++) {
FieldSpec fieldSpec = fieldSpecs.get(i);
_fieldNames[i] = fieldSpec.getName();
@@ -96,10 +98,15 @@ public class GenericRowSerializer {
case DOUBLE:
numBytes += Double.BYTES;
break;
+ case BIG_DECIMAL:
+ byte[] bigDecimalBytes = BigDecimalUtils.serialize((BigDecimal)
value);
+ numBytes += Integer.BYTES + bigDecimalBytes.length;
+ _objectBytes[i] = bigDecimalBytes;
+ break;
case STRING:
byte[] stringBytes = ((String) value).getBytes(UTF_8);
numBytes += Integer.BYTES + stringBytes.length;
- _stringBytes[i] = stringBytes;
+ _objectBytes[i] = stringBytes;
break;
case BYTES:
numBytes += Integer.BYTES + ((byte[]) value).length;
@@ -133,7 +140,7 @@ public class GenericRowSerializer {
numBytes += stringBytes.length;
stringBytesArray[j] = stringBytes;
}
- _stringBytes[i] = stringBytesArray;
+ _objectBytes[i] = stringBytesArray;
break;
default:
throw new IllegalStateException("Unsupported MV stored type: " +
_storedTypes[i]);
@@ -176,10 +183,11 @@ public class GenericRowSerializer {
case DOUBLE:
byteBuffer.putDouble((double) value);
break;
+ case BIG_DECIMAL:
case STRING:
- byte[] stringBytes = (byte[]) _stringBytes[i];
- byteBuffer.putInt(stringBytes.length);
- byteBuffer.put(stringBytes);
+ byte[] objectBytes = (byte[]) _objectBytes[i];
+ byteBuffer.putInt(objectBytes.length);
+ byteBuffer.put(objectBytes);
break;
case BYTES:
byte[] bytes = (byte[]) value;
@@ -215,7 +223,7 @@ public class GenericRowSerializer {
}
break;
case STRING:
- byte[][] stringBytesArray = (byte[][]) _stringBytes[i];
+ byte[][] stringBytesArray = (byte[][]) _objectBytes[i];
for (byte[] stringBytes : stringBytesArray) {
byteBuffer.putInt(stringBytes.length);
byteBuffer.put(stringBytes);
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerDeTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerDeTest.java
index b17c7c7e8e..ffc991413d 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerDeTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/segment/processing/genericrow/GenericRowSerDeTest.java
@@ -18,6 +18,7 @@
*/
package org.apache.pinot.core.segment.processing.genericrow;
+import java.math.BigDecimal;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@@ -27,6 +28,7 @@ import org.apache.pinot.segment.spi.memory.PinotDataBuffer;
import org.apache.pinot.spi.data.DimensionFieldSpec;
import org.apache.pinot.spi.data.FieldSpec;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.data.MetricFieldSpec;
import org.apache.pinot.spi.data.readers.GenericRow;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
@@ -45,7 +47,8 @@ public class GenericRowSerDeTest {
new DimensionFieldSpec("longSV", DataType.LONG, true), new
DimensionFieldSpec("floatSV", DataType.FLOAT, true),
new DimensionFieldSpec("doubleSV", DataType.DOUBLE, true),
new DimensionFieldSpec("stringSV", DataType.STRING, true),
- new DimensionFieldSpec("bytesSV", DataType.BYTES, true), new
DimensionFieldSpec("nullSV", DataType.INT, true),
+ new DimensionFieldSpec("bytesSV", DataType.BYTES, true),
+ new MetricFieldSpec("bigDecimalSV", DataType.BIG_DECIMAL), new
DimensionFieldSpec("nullSV", DataType.INT, true),
new DimensionFieldSpec("intMV", DataType.INT, false), new
DimensionFieldSpec("longMV", DataType.LONG, false),
new DimensionFieldSpec("floatMV", DataType.FLOAT, false),
new DimensionFieldSpec("doubleMV", DataType.DOUBLE, false),
@@ -59,6 +62,7 @@ public class GenericRowSerDeTest {
_row.putValue("doubleSV", 123.0);
_row.putValue("stringSV", "123");
_row.putValue("bytesSV", new byte[]{1, 2, 3});
+ _row.putValue("bigDecimalSV", new BigDecimal("122333"));
_row.putDefaultNullValue("nullSV", Integer.MAX_VALUE);
_row.putValue("intMV", new Object[]{123, 456});
_row.putValue("longMV", new Object[]{123L, 456L});
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
