This is an automated email from the ASF dual-hosted git repository.
xiangfu0 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/pinot.git
The following commit(s) were added to refs/heads/master by this push:
new 0513b741343 Support BYTES array literals end to end (#19247)
0513b741343 is described below
commit 0513b7413439f5d5a5c4318a6cbec93e0b9b398e
Author: Xiang Fu <[email protected]>
AuthorDate: Tue Aug 25 16:58:44 2026 -0700
Support BYTES array literals end to end (#19247)
* Support BYTES array literals end to end
Pinot can ingest and project multi-value BYTES columns, but neither query
engine could reliably construct an equivalent SQL literal. Single-stage parsing
lacked a native MV BYTES representation, while multi-stage execution returned
external arrays where DataBlock expects ByteArray[].
Add the Thrift literal arm and the conversions needed by both engines. Keep
ordinary single-stage broker-to-server requests encoded as
arrayValueConstructor with scalar binary literals so older Literal readers can
still decode them. Servers still need the new execution support before using
the feature, but mixed-version decoding remains compatible.
Cache constant multi-stage byte-array operands in both internal and
external form to avoid per-row allocation. Exercise Avro array<bytes>
ingestion, dictionary and raw segment storage, literal projection, and
arraysOverlap through both query engines.
* Test BYTES array runtime conversion
Directly exercise TypeUtils conversion from external byte[][] values to the
internal ByteArray[] representation required for multi-stage block
serialization. Cover empty, single-byte, multi-byte, and unsigned byte content.
* Optimize dynamic BYTES array construction
Use a BYTES_ARRAY-specific transform operand for mixed literal and dynamic
values. Reuse existing ByteArray wrappers and backing bytes while allocating
only the required outer result array per row.
* Use generic dispatch for BYTES array construction
FunctionOperand already converts stored values to the external UDF
representation. Let ArrayFunctions build byte[][] through that shared path,
then normalize the result back to internal ByteArray[] in TypeUtils instead of
maintaining a one-off operand and factory branch.
* Address review feedback for BYTES array literals
Make BYTES array conversion explicit across Rex and scalar paths, reject
unsupported elements consistently, and cover broker, wire, and end-to-end
review cases.
* Use native Thrift literal for BYTES arrays
* Follow array literal conventions
* Address BYTES array review feedback
---
.../LiteralOnlyBrokerRequestTest.java | 31 +++
.../common/function/scalar/ArrayFunctions.java | 7 +
.../org/apache/pinot/common/request/Literal.java | 257 ++++++++++++++-------
.../common/request/context/LiteralContext.java | 27 ++-
.../org/apache/pinot/common/utils/DataSchema.java | 11 +
.../pinot/common/utils/request/RequestUtils.java | 39 ++++
.../common/function/scalar/ArrayFunctionsTest.java | 34 +++
.../pinot/common/request/LiteralSerDeTest.java | 168 ++++++++++++++
.../common/request/context/LiteralContextTest.java | 44 ++++
.../apache/pinot/common/utils/DataSchemaTest.java | 17 ++
.../common/utils/request/RequestUtilsTest.java | 22 ++
pinot-common/src/thrift/query.thrift | 1 +
.../function/ArrayLiteralTransformFunction.java | 46 +++-
.../ArrayLiteralTransformFunctionTest.java | 25 ++
.../integration/tests/custom/BytesMvTypeTest.java | 61 +++++
.../query/parser/CalciteRexExpressionParser.java | 11 +-
.../parser/CalciteRexExpressionParserTest.java | 13 ++
.../planner/serde/RexExpressionSerDeTest.java | 9 +-
.../runtime/operator/TransformOperatorTest.java | 52 +++++
19 files changed, 784 insertions(+), 91 deletions(-)
diff --git
a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java
b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java
index b3154c5bb90..5db658610dc 100644
---
a/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java
+++
b/pinot-broker/src/test/java/org/apache/pinot/broker/requesthandler/LiteralOnlyBrokerRequestTest.java
@@ -80,6 +80,37 @@ public class LiteralOnlyBrokerRequestTest {
assertTrue(isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery("SELECT
1, '2', 3 FROM myTable")));
}
+ @Test
+ public void testArrayLiteralBrokerRequestFromSQL()
+ throws Exception {
+ SingleConnectionBrokerRequestHandler requestHandler =
+ new SingleConnectionBrokerRequestHandler(new PinotConfiguration(),
"testBrokerId",
+ new BrokerRequestIdGenerator(), null, ACCESS_CONTROL_FACTORY,
null, null, null, null,
+ mock(ServerRoutingStatsManager.class), mock(FailureDetector.class),
+ ThreadAccountantUtils.getNoOpAccountant(), null, null);
+
+ BrokerResponse brokerResponse = requestHandler.handleRequest(
+ "SELECT ARRAY[1, 2] AS ints, ARRAY['one', 'two'] AS strings");
+ ResultTable resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getDataSchema().getColumnDataType(0),
DataSchema.ColumnDataType.INT_ARRAY);
+ assertEquals(resultTable.getDataSchema().getColumnDataType(1),
DataSchema.ColumnDataType.STRING_ARRAY);
+ assertEquals((int[]) resultTable.getRows().get(0)[0], new int[]{1, 2});
+ assertEquals((String[]) resultTable.getRows().get(0)[1], new
String[]{"one", "two"});
+
+ brokerResponse = requestHandler.handleRequest("SELECT ARRAY[X'00',
X'0102'] AS bytes");
+ resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getDataSchema().getColumnName(0), "bytes");
+ assertEquals(resultTable.getDataSchema().getColumnDataType(0),
DataSchema.ColumnDataType.BYTES_ARRAY);
+ assertEquals(resultTable.getRows().size(), 1);
+ assertEquals(resultTable.getRows().get(0), new Object[]{new String[]{"00",
"0102"}});
+
+ brokerResponse = requestHandler.handleRequest(
+ "SELECT ARRAYS_OVERLAP(ARRAY[X'00', X'0102'], ARRAY[X'03', X'0102'])
AS overlaps");
+ resultTable = brokerResponse.getResultTable();
+ assertEquals(resultTable.getDataSchema().getColumnDataType(0),
DataSchema.ColumnDataType.BOOLEAN);
+ assertEquals(resultTable.getRows().get(0)[0], true);
+ }
+
@Test
public void testLiteralOnlyTransformBrokerRequestFromSQL() {
assertTrue(isLiteralOnlyQuery(CalciteSqlParser.compileToPinotQuery("SELECT
now()")));
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ArrayFunctions.java
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ArrayFunctions.java
index 2aa1a7c076a..a6db5b7906f 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ArrayFunctions.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/function/scalar/ArrayFunctions.java
@@ -352,6 +352,13 @@ public class ArrayFunctions {
}
return strArr;
}
+ if (clazz == byte[].class) {
+ byte[][] bytesArr = new byte[arr.length][];
+ for (int i = 0; i < arr.length; i++) {
+ bytesArr[i] = (byte[]) arr[i];
+ }
+ return bytesArr;
+ }
return arr;
}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/request/Literal.java
b/pinot-common/src/main/java/org/apache/pinot/common/request/Literal.java
index ca15bb5f87d..6f084d7f1a5 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/request/Literal.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/request/Literal.java
@@ -25,7 +25,7 @@
package org.apache.pinot.common.request;
@SuppressWarnings({"cast", "rawtypes", "serial", "unchecked", "unused"})
[email protected](value = "Autogenerated by Thrift Compiler
(0.21.0)", date = "2025-04-16")
[email protected](value = "Autogenerated by Thrift Compiler
(0.21.0)", date = "2026-08-13")
public class Literal extends org.apache.thrift.TUnion<Literal,
Literal._Fields> {
private static final org.apache.thrift.protocol.TStruct STRUCT_DESC = new
org.apache.thrift.protocol.TStruct("Literal");
private static final org.apache.thrift.protocol.TField BOOL_VALUE_FIELD_DESC
= new org.apache.thrift.protocol.TField("boolValue",
org.apache.thrift.protocol.TType.BOOL, (short)1);
@@ -44,6 +44,7 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
private static final org.apache.thrift.protocol.TField
FLOAT_ARRAY_VALUE_FIELD_DESC = new
org.apache.thrift.protocol.TField("floatArrayValue",
org.apache.thrift.protocol.TType.LIST, (short)14);
private static final org.apache.thrift.protocol.TField
DOUBLE_ARRAY_VALUE_FIELD_DESC = new
org.apache.thrift.protocol.TField("doubleArrayValue",
org.apache.thrift.protocol.TType.LIST, (short)15);
private static final org.apache.thrift.protocol.TField
STRING_ARRAY_VALUE_FIELD_DESC = new
org.apache.thrift.protocol.TField("stringArrayValue",
org.apache.thrift.protocol.TType.LIST, (short)16);
+ private static final org.apache.thrift.protocol.TField
BYTES_ARRAY_VALUE_FIELD_DESC = new
org.apache.thrift.protocol.TField("bytesArrayValue",
org.apache.thrift.protocol.TType.LIST, (short)17);
/** The set of fields this struct contains, along with convenience methods
for finding and manipulating them. */
public enum _Fields implements org.apache.thrift.TFieldIdEnum {
@@ -62,7 +63,8 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
LONG_ARRAY_VALUE((short)13, "longArrayValue"),
FLOAT_ARRAY_VALUE((short)14, "floatArrayValue"),
DOUBLE_ARRAY_VALUE((short)15, "doubleArrayValue"),
- STRING_ARRAY_VALUE((short)16, "stringArrayValue");
+ STRING_ARRAY_VALUE((short)16, "stringArrayValue"),
+ BYTES_ARRAY_VALUE((short)17, "bytesArrayValue");
private static final java.util.Map<java.lang.String, _Fields> byName = new
java.util.HashMap<java.lang.String, _Fields>();
@@ -110,6 +112,8 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
return DOUBLE_ARRAY_VALUE;
case 16: // STRING_ARRAY_VALUE
return STRING_ARRAY_VALUE;
+ case 17: // BYTES_ARRAY_VALUE
+ return BYTES_ARRAY_VALUE;
default:
return null;
}
@@ -192,6 +196,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
tmpMap.put(_Fields.STRING_ARRAY_VALUE, new
org.apache.thrift.meta_data.FieldMetaData("stringArrayValue",
org.apache.thrift.TFieldRequirementType.OPTIONAL,
new
org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING))));
+ tmpMap.put(_Fields.BYTES_ARRAY_VALUE, new
org.apache.thrift.meta_data.FieldMetaData("bytesArrayValue",
org.apache.thrift.TFieldRequirementType.OPTIONAL,
+ new
org.apache.thrift.meta_data.ListMetaData(org.apache.thrift.protocol.TType.LIST,
+ new
org.apache.thrift.meta_data.FieldValueMetaData(org.apache.thrift.protocol.TType.STRING
, true))));
metaDataMap = java.util.Collections.unmodifiableMap(tmpMap);
org.apache.thrift.meta_data.FieldMetaData.addStructMetaDataMap(Literal.class,
metaDataMap);
}
@@ -320,6 +327,12 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
return x;
}
+ public static Literal bytesArrayValue(java.util.List<java.nio.ByteBuffer>
value) {
+ Literal x = new Literal();
+ x.setBytesArrayValue(value);
+ return x;
+ }
+
@Override
protected void checkType(_Fields setField, java.lang.Object value) throws
java.lang.ClassCastException {
@@ -404,6 +417,11 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
break;
}
throw new java.lang.ClassCastException("Was expecting value of type
java.util.List<java.lang.String> for field 'stringArrayValue', but got " +
value.getClass().getSimpleName());
+ case BYTES_ARRAY_VALUE:
+ if (value instanceof java.util.List) {
+ break;
+ }
+ throw new java.lang.ClassCastException("Was expecting value of type
java.util.List<java.nio.ByteBuffer> for field 'bytesArrayValue', but got " +
value.getClass().getSimpleName());
default:
throw new java.lang.IllegalArgumentException("Unknown field id " +
setField);
}
@@ -517,13 +535,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
if (field.type == INT_ARRAY_VALUE_FIELD_DESC.type) {
java.util.List<java.lang.Integer> intArrayValue;
{
- org.apache.thrift.protocol.TList _list44 = iprot.readListBegin();
- intArrayValue = new
java.util.ArrayList<java.lang.Integer>(_list44.size);
- int _elem45;
- for (int _i46 = 0; _i46 < _list44.size; ++_i46)
+ org.apache.thrift.protocol.TList _list60 = iprot.readListBegin();
+ intArrayValue = new
java.util.ArrayList<java.lang.Integer>(_list60.size);
+ int _elem61;
+ for (int _i62 = 0; _i62 < _list60.size; ++_i62)
{
- _elem45 = iprot.readI32();
- intArrayValue.add(_elem45);
+ _elem61 = iprot.readI32();
+ intArrayValue.add(_elem61);
}
iprot.readListEnd();
}
@@ -536,13 +554,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
if (field.type == LONG_ARRAY_VALUE_FIELD_DESC.type) {
java.util.List<java.lang.Long> longArrayValue;
{
- org.apache.thrift.protocol.TList _list47 = iprot.readListBegin();
- longArrayValue = new
java.util.ArrayList<java.lang.Long>(_list47.size);
- long _elem48;
- for (int _i49 = 0; _i49 < _list47.size; ++_i49)
+ org.apache.thrift.protocol.TList _list63 = iprot.readListBegin();
+ longArrayValue = new
java.util.ArrayList<java.lang.Long>(_list63.size);
+ long _elem64;
+ for (int _i65 = 0; _i65 < _list63.size; ++_i65)
{
- _elem48 = iprot.readI64();
- longArrayValue.add(_elem48);
+ _elem64 = iprot.readI64();
+ longArrayValue.add(_elem64);
}
iprot.readListEnd();
}
@@ -555,13 +573,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
if (field.type == FLOAT_ARRAY_VALUE_FIELD_DESC.type) {
java.util.List<java.lang.Integer> floatArrayValue;
{
- org.apache.thrift.protocol.TList _list50 = iprot.readListBegin();
- floatArrayValue = new
java.util.ArrayList<java.lang.Integer>(_list50.size);
- int _elem51;
- for (int _i52 = 0; _i52 < _list50.size; ++_i52)
+ org.apache.thrift.protocol.TList _list66 = iprot.readListBegin();
+ floatArrayValue = new
java.util.ArrayList<java.lang.Integer>(_list66.size);
+ int _elem67;
+ for (int _i68 = 0; _i68 < _list66.size; ++_i68)
{
- _elem51 = iprot.readI32();
- floatArrayValue.add(_elem51);
+ _elem67 = iprot.readI32();
+ floatArrayValue.add(_elem67);
}
iprot.readListEnd();
}
@@ -574,13 +592,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
if (field.type == DOUBLE_ARRAY_VALUE_FIELD_DESC.type) {
java.util.List<java.lang.Double> doubleArrayValue;
{
- org.apache.thrift.protocol.TList _list53 = iprot.readListBegin();
- doubleArrayValue = new
java.util.ArrayList<java.lang.Double>(_list53.size);
- double _elem54;
- for (int _i55 = 0; _i55 < _list53.size; ++_i55)
+ org.apache.thrift.protocol.TList _list69 = iprot.readListBegin();
+ doubleArrayValue = new
java.util.ArrayList<java.lang.Double>(_list69.size);
+ double _elem70;
+ for (int _i71 = 0; _i71 < _list69.size; ++_i71)
{
- _elem54 = iprot.readDouble();
- doubleArrayValue.add(_elem54);
+ _elem70 = iprot.readDouble();
+ doubleArrayValue.add(_elem70);
}
iprot.readListEnd();
}
@@ -593,13 +611,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
if (field.type == STRING_ARRAY_VALUE_FIELD_DESC.type) {
java.util.List<java.lang.String> stringArrayValue;
{
- org.apache.thrift.protocol.TList _list56 = iprot.readListBegin();
- stringArrayValue = new
java.util.ArrayList<java.lang.String>(_list56.size);
- @org.apache.thrift.annotation.Nullable java.lang.String _elem57;
- for (int _i58 = 0; _i58 < _list56.size; ++_i58)
+ org.apache.thrift.protocol.TList _list72 = iprot.readListBegin();
+ stringArrayValue = new
java.util.ArrayList<java.lang.String>(_list72.size);
+ @org.apache.thrift.annotation.Nullable java.lang.String _elem73;
+ for (int _i74 = 0; _i74 < _list72.size; ++_i74)
{
- _elem57 = iprot.readString();
- stringArrayValue.add(_elem57);
+ _elem73 = iprot.readString();
+ stringArrayValue.add(_elem73);
}
iprot.readListEnd();
}
@@ -608,6 +626,25 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
return null;
}
+ case BYTES_ARRAY_VALUE:
+ if (field.type == BYTES_ARRAY_VALUE_FIELD_DESC.type) {
+ java.util.List<java.nio.ByteBuffer> bytesArrayValue;
+ {
+ org.apache.thrift.protocol.TList _list75 = iprot.readListBegin();
+ bytesArrayValue = new
java.util.ArrayList<java.nio.ByteBuffer>(_list75.size);
+ @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer
_elem76;
+ for (int _i77 = 0; _i77 < _list75.size; ++_i77)
+ {
+ _elem76 = iprot.readBinary();
+ bytesArrayValue.add(_elem76);
+ }
+ iprot.readListEnd();
+ }
+ return bytesArrayValue;
+ } else {
+ org.apache.thrift.protocol.TProtocolUtil.skip(iprot, field.type);
+ return null;
+ }
default:
throw new java.lang.IllegalStateException("setField wasn't null, but
didn't match any of the case statements!");
}
@@ -668,9 +705,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.Integer> intArrayValue =
(java.util.List<java.lang.Integer>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32,
intArrayValue.size()));
- for (int _iter59 : intArrayValue)
+ for (int _iter78 : intArrayValue)
{
- oprot.writeI32(_iter59);
+ oprot.writeI32(_iter78);
}
oprot.writeListEnd();
}
@@ -679,9 +716,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.Long> longArrayValue =
(java.util.List<java.lang.Long>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64,
longArrayValue.size()));
- for (long _iter60 : longArrayValue)
+ for (long _iter79 : longArrayValue)
{
- oprot.writeI64(_iter60);
+ oprot.writeI64(_iter79);
}
oprot.writeListEnd();
}
@@ -690,9 +727,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.Integer> floatArrayValue =
(java.util.List<java.lang.Integer>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32,
floatArrayValue.size()));
- for (int _iter61 : floatArrayValue)
+ for (int _iter80 : floatArrayValue)
{
- oprot.writeI32(_iter61);
+ oprot.writeI32(_iter80);
}
oprot.writeListEnd();
}
@@ -701,9 +738,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.Double> doubleArrayValue =
(java.util.List<java.lang.Double>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.DOUBLE,
doubleArrayValue.size()));
- for (double _iter62 : doubleArrayValue)
+ for (double _iter81 : doubleArrayValue)
{
- oprot.writeDouble(_iter62);
+ oprot.writeDouble(_iter81);
}
oprot.writeListEnd();
}
@@ -712,9 +749,20 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.String> stringArrayValue =
(java.util.List<java.lang.String>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
stringArrayValue.size()));
- for (java.lang.String _iter63 : stringArrayValue)
+ for (java.lang.String _iter82 : stringArrayValue)
+ {
+ oprot.writeString(_iter82);
+ }
+ oprot.writeListEnd();
+ }
+ return;
+ case BYTES_ARRAY_VALUE:
+ java.util.List<java.nio.ByteBuffer> bytesArrayValue =
(java.util.List<java.nio.ByteBuffer>)value_;
+ {
+ oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
bytesArrayValue.size()));
+ for (java.nio.ByteBuffer _iter83 : bytesArrayValue)
{
- oprot.writeString(_iter63);
+ oprot.writeBinary(_iter83);
}
oprot.writeListEnd();
}
@@ -776,13 +824,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
case INT_ARRAY_VALUE:
java.util.List<java.lang.Integer> intArrayValue;
{
- org.apache.thrift.protocol.TList _list64 = iprot.readListBegin();
- intArrayValue = new
java.util.ArrayList<java.lang.Integer>(_list64.size);
- int _elem65;
- for (int _i66 = 0; _i66 < _list64.size; ++_i66)
+ org.apache.thrift.protocol.TList _list84 = iprot.readListBegin();
+ intArrayValue = new
java.util.ArrayList<java.lang.Integer>(_list84.size);
+ int _elem85;
+ for (int _i86 = 0; _i86 < _list84.size; ++_i86)
{
- _elem65 = iprot.readI32();
- intArrayValue.add(_elem65);
+ _elem85 = iprot.readI32();
+ intArrayValue.add(_elem85);
}
iprot.readListEnd();
}
@@ -790,13 +838,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
case LONG_ARRAY_VALUE:
java.util.List<java.lang.Long> longArrayValue;
{
- org.apache.thrift.protocol.TList _list67 = iprot.readListBegin();
- longArrayValue = new
java.util.ArrayList<java.lang.Long>(_list67.size);
- long _elem68;
- for (int _i69 = 0; _i69 < _list67.size; ++_i69)
+ org.apache.thrift.protocol.TList _list87 = iprot.readListBegin();
+ longArrayValue = new
java.util.ArrayList<java.lang.Long>(_list87.size);
+ long _elem88;
+ for (int _i89 = 0; _i89 < _list87.size; ++_i89)
{
- _elem68 = iprot.readI64();
- longArrayValue.add(_elem68);
+ _elem88 = iprot.readI64();
+ longArrayValue.add(_elem88);
}
iprot.readListEnd();
}
@@ -804,13 +852,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
case FLOAT_ARRAY_VALUE:
java.util.List<java.lang.Integer> floatArrayValue;
{
- org.apache.thrift.protocol.TList _list70 = iprot.readListBegin();
- floatArrayValue = new
java.util.ArrayList<java.lang.Integer>(_list70.size);
- int _elem71;
- for (int _i72 = 0; _i72 < _list70.size; ++_i72)
+ org.apache.thrift.protocol.TList _list90 = iprot.readListBegin();
+ floatArrayValue = new
java.util.ArrayList<java.lang.Integer>(_list90.size);
+ int _elem91;
+ for (int _i92 = 0; _i92 < _list90.size; ++_i92)
{
- _elem71 = iprot.readI32();
- floatArrayValue.add(_elem71);
+ _elem91 = iprot.readI32();
+ floatArrayValue.add(_elem91);
}
iprot.readListEnd();
}
@@ -818,13 +866,13 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
case DOUBLE_ARRAY_VALUE:
java.util.List<java.lang.Double> doubleArrayValue;
{
- org.apache.thrift.protocol.TList _list73 = iprot.readListBegin();
- doubleArrayValue = new
java.util.ArrayList<java.lang.Double>(_list73.size);
- double _elem74;
- for (int _i75 = 0; _i75 < _list73.size; ++_i75)
+ org.apache.thrift.protocol.TList _list93 = iprot.readListBegin();
+ doubleArrayValue = new
java.util.ArrayList<java.lang.Double>(_list93.size);
+ double _elem94;
+ for (int _i95 = 0; _i95 < _list93.size; ++_i95)
{
- _elem74 = iprot.readDouble();
- doubleArrayValue.add(_elem74);
+ _elem94 = iprot.readDouble();
+ doubleArrayValue.add(_elem94);
}
iprot.readListEnd();
}
@@ -832,17 +880,31 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
case STRING_ARRAY_VALUE:
java.util.List<java.lang.String> stringArrayValue;
{
- org.apache.thrift.protocol.TList _list76 = iprot.readListBegin();
- stringArrayValue = new
java.util.ArrayList<java.lang.String>(_list76.size);
- @org.apache.thrift.annotation.Nullable java.lang.String _elem77;
- for (int _i78 = 0; _i78 < _list76.size; ++_i78)
+ org.apache.thrift.protocol.TList _list96 = iprot.readListBegin();
+ stringArrayValue = new
java.util.ArrayList<java.lang.String>(_list96.size);
+ @org.apache.thrift.annotation.Nullable java.lang.String _elem97;
+ for (int _i98 = 0; _i98 < _list96.size; ++_i98)
{
- _elem77 = iprot.readString();
- stringArrayValue.add(_elem77);
+ _elem97 = iprot.readString();
+ stringArrayValue.add(_elem97);
}
iprot.readListEnd();
}
return stringArrayValue;
+ case BYTES_ARRAY_VALUE:
+ java.util.List<java.nio.ByteBuffer> bytesArrayValue;
+ {
+ org.apache.thrift.protocol.TList _list99 = iprot.readListBegin();
+ bytesArrayValue = new
java.util.ArrayList<java.nio.ByteBuffer>(_list99.size);
+ @org.apache.thrift.annotation.Nullable java.nio.ByteBuffer
_elem100;
+ for (int _i101 = 0; _i101 < _list99.size; ++_i101)
+ {
+ _elem100 = iprot.readBinary();
+ bytesArrayValue.add(_elem100);
+ }
+ iprot.readListEnd();
+ }
+ return bytesArrayValue;
default:
throw new java.lang.IllegalStateException("setField wasn't null, but
didn't match any of the case statements!");
}
@@ -902,9 +964,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.Integer> intArrayValue =
(java.util.List<java.lang.Integer>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32,
intArrayValue.size()));
- for (int _iter79 : intArrayValue)
+ for (int _iter102 : intArrayValue)
{
- oprot.writeI32(_iter79);
+ oprot.writeI32(_iter102);
}
oprot.writeListEnd();
}
@@ -913,9 +975,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.Long> longArrayValue =
(java.util.List<java.lang.Long>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I64,
longArrayValue.size()));
- for (long _iter80 : longArrayValue)
+ for (long _iter103 : longArrayValue)
{
- oprot.writeI64(_iter80);
+ oprot.writeI64(_iter103);
}
oprot.writeListEnd();
}
@@ -924,9 +986,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.Integer> floatArrayValue =
(java.util.List<java.lang.Integer>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.I32,
floatArrayValue.size()));
- for (int _iter81 : floatArrayValue)
+ for (int _iter104 : floatArrayValue)
{
- oprot.writeI32(_iter81);
+ oprot.writeI32(_iter104);
}
oprot.writeListEnd();
}
@@ -935,9 +997,9 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.Double> doubleArrayValue =
(java.util.List<java.lang.Double>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.DOUBLE,
doubleArrayValue.size()));
- for (double _iter82 : doubleArrayValue)
+ for (double _iter105 : doubleArrayValue)
{
- oprot.writeDouble(_iter82);
+ oprot.writeDouble(_iter105);
}
oprot.writeListEnd();
}
@@ -946,9 +1008,20 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
java.util.List<java.lang.String> stringArrayValue =
(java.util.List<java.lang.String>)value_;
{
oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
stringArrayValue.size()));
- for (java.lang.String _iter83 : stringArrayValue)
+ for (java.lang.String _iter106 : stringArrayValue)
{
- oprot.writeString(_iter83);
+ oprot.writeString(_iter106);
+ }
+ oprot.writeListEnd();
+ }
+ return;
+ case BYTES_ARRAY_VALUE:
+ java.util.List<java.nio.ByteBuffer> bytesArrayValue =
(java.util.List<java.nio.ByteBuffer>)value_;
+ {
+ oprot.writeListBegin(new
org.apache.thrift.protocol.TList(org.apache.thrift.protocol.TType.STRING,
bytesArrayValue.size()));
+ for (java.nio.ByteBuffer _iter107 : bytesArrayValue)
+ {
+ oprot.writeBinary(_iter107);
}
oprot.writeListEnd();
}
@@ -993,6 +1066,8 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
return DOUBLE_ARRAY_VALUE_FIELD_DESC;
case STRING_ARRAY_VALUE:
return STRING_ARRAY_VALUE_FIELD_DESC;
+ case BYTES_ARRAY_VALUE:
+ return BYTES_ARRAY_VALUE_FIELD_DESC;
default:
throw new java.lang.IllegalArgumentException("Unknown field id " +
setField);
}
@@ -1243,6 +1318,19 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
value_ =
java.util.Objects.requireNonNull(value,"_Fields.STRING_ARRAY_VALUE");
}
+ public java.util.List<java.nio.ByteBuffer> getBytesArrayValue() {
+ if (getSetField() == _Fields.BYTES_ARRAY_VALUE) {
+ return (java.util.List<java.nio.ByteBuffer>)getFieldValue();
+ } else {
+ throw new java.lang.RuntimeException("Cannot get field 'bytesArrayValue'
because union is currently set to " + getFieldDesc(getSetField()).name);
+ }
+ }
+
+ public void setBytesArrayValue(java.util.List<java.nio.ByteBuffer> value) {
+ setField_ = _Fields.BYTES_ARRAY_VALUE;
+ value_ =
java.util.Objects.requireNonNull(value,"_Fields.BYTES_ARRAY_VALUE");
+ }
+
public boolean isSetBoolValue() {
return setField_ == _Fields.BOOL_VALUE;
}
@@ -1323,6 +1411,11 @@ public class Literal extends
org.apache.thrift.TUnion<Literal, Literal._Fields>
}
+ public boolean isSetBytesArrayValue() {
+ return setField_ == _Fields.BYTES_ARRAY_VALUE;
+ }
+
+
public boolean equals(java.lang.Object other) {
if (other instanceof Literal) {
return equals((Literal)other);
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/LiteralContext.java
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/LiteralContext.java
index 3f5b09f0274..3f99fc36e60 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/request/context/LiteralContext.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/request/context/LiteralContext.java
@@ -29,6 +29,7 @@ import org.apache.pinot.common.request.Literal;
import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.spi.data.FieldSpec.DataType;
import org.apache.pinot.spi.utils.BigDecimalUtils;
+import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.CommonConstants.NullValuePlaceHolder;
import org.apache.pinot.spi.utils.PinotDataType;
@@ -135,6 +136,11 @@ public class LiteralContext {
_value = RequestUtils.getStringArrayValue(literal);
_pinotDataType = PinotDataType.STRING_ARRAY;
break;
+ case BYTES_ARRAY_VALUE:
+ _type = DataType.BYTES;
+ _value = RequestUtils.getBytesArrayValue(literal);
+ _pinotDataType = PinotDataType.BYTES_ARRAY;
+ break;
default:
throw new IllegalStateException("Unsupported field type: " +
literal.getSetField());
}
@@ -147,15 +153,18 @@ public class LiteralContext {
_pinotDataType = getPinotDataType(type, value);
}
- // TODO: Revisit MV support for BOOLEAN, BIG_DECIMAL, BYTES and UUID.
+ // TODO: Revisit MV support for BOOLEAN, BIG_DECIMAL and UUID.
+ // https://github.com/apache/pinot/issues/19338
@Nullable
private static PinotDataType getPinotDataType(DataType type, @Nullable
Object value) {
if (value == null) {
return null;
}
if (type == DataType.BYTES) {
- Preconditions.checkState(value.getClass().getComponentType() ==
byte.class, "Bytes array is not supported");
- return PinotDataType.BYTES;
+ Class<?> componentType = value.getClass().getComponentType();
+ Preconditions.checkState(componentType == byte.class || componentType ==
byte[].class,
+ "Expected byte[] or byte[][], got: %s", value.getClass());
+ return componentType == byte.class ? PinotDataType.BYTES :
PinotDataType.BYTES_ARRAY;
}
boolean singleValue = !value.getClass().isArray();
switch (type) {
@@ -297,7 +306,7 @@ public class LiteralContext {
@Override
public int hashCode() {
- return Objects.hash(_value, _type);
+ return Arrays.deepHashCode(new Object[]{_value, _type});
}
@Override
@@ -309,7 +318,7 @@ public class LiteralContext {
return false;
}
LiteralContext that = (LiteralContext) o;
- return _type.equals(that._type) && Objects.equals(_value, that._value);
+ return _type.equals(that._type) && Objects.deepEquals(_value, that._value);
}
@Override
@@ -333,6 +342,14 @@ public class LiteralContext {
return "'" + Arrays.toString((double[]) _value) + "'";
case STRING_ARRAY:
return "'" + Arrays.toString((String[]) _value) + "'";
+ case BYTES_ARRAY: {
+ byte[][] bytesArray = (byte[][]) _value;
+ String[] hexValues = new String[bytesArray.length];
+ for (int i = 0; i < bytesArray.length; i++) {
+ hexValues[i] = BytesUtils.toHexString(bytesArray[i]);
+ }
+ return "'" + Arrays.toString(hexValues) + "'";
+ }
default:
throw new IllegalStateException("Unsupported PinotDataType: " +
_pinotDataType);
}
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
index 52312c2a741..6d5e25b7704 100644
--- a/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
+++ b/pinot-common/src/main/java/org/apache/pinot/common/utils/DataSchema.java
@@ -589,6 +589,17 @@ public class DataSchema {
if (value instanceof Timestamp[]) {
return fromTimestampArray((Timestamp[]) value);
}
+ if (value instanceof byte[][]) {
+ byte[][] bytesArray = (byte[][]) value;
+ ByteArray[] internalBytesArray = new ByteArray[bytesArray.length];
+ for (int i = 0; i < bytesArray.length; i++) {
+ internalBytesArray[i] = new ByteArray(bytesArray[i]);
+ }
+ return internalBytesArray;
+ }
+ if (value instanceof UUID[]) {
+ return fromUuidArray(value);
+ }
return value;
default:
return value;
diff --git
a/pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java
b/pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java
index 20e1765d0f1..0f5a3b4d0c6 100644
---
a/pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java
+++
b/pinot-common/src/main/java/org/apache/pinot/common/utils/request/RequestUtils.java
@@ -29,6 +29,7 @@ import it.unimi.dsi.fastutil.doubles.DoubleArrayList;
import it.unimi.dsi.fastutil.ints.IntArrayList;
import it.unimi.dsi.fastutil.longs.LongArrayList;
import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.ArrayList;
import java.util.Arrays;
@@ -179,6 +180,14 @@ public class RequestUtils {
return Literal.stringArrayValue(Arrays.asList(value));
}
+ public static Literal getLiteral(byte[][] value) {
+ List<ByteBuffer> bytesArray = new ArrayList<>(value.length);
+ for (byte[] bytes : value) {
+ bytesArray.add(ByteBuffer.wrap(bytes.clone()));
+ }
+ return Literal.bytesArrayValue(bytesArray);
+ }
+
public static Literal getLiteral(@Nullable Object object) {
if (object == null) {
return getNullLiteral();
@@ -228,6 +237,9 @@ public class RequestUtils {
if (object instanceof String[]) {
return getLiteral((String[]) object);
}
+ if (object instanceof byte[][]) {
+ return getLiteral((byte[][]) object);
+ }
return getLiteral(object.toString());
}
@@ -255,6 +267,9 @@ public class RequestUtils {
case BOOLEAN:
literal.setBoolValue(node.booleanValue());
break;
+ case BINARY:
+ literal.setBinaryValue(node.getValueAs(byte[].class));
+ break;
case NULL:
literal.setNullValue(true);
break;
@@ -329,6 +344,10 @@ public class RequestUtils {
return getLiteralExpression(getLiteral(value));
}
+ public static Expression getLiteralExpression(byte[][] value) {
+ return getLiteralExpression(getLiteral(value));
+ }
+
public static Expression getLiteralExpression(SqlLiteral node) {
return getLiteralExpression(getLiteral(node));
}
@@ -370,6 +389,8 @@ public class RequestUtils {
return getDoubleArrayValue(literal);
case STRING_ARRAY_VALUE:
return getStringArrayValue(literal);
+ case BYTES_ARRAY_VALUE:
+ return getBytesArrayValue(literal);
default:
throw new IllegalStateException("Unsupported field type: " + type);
}
@@ -419,6 +440,19 @@ public class RequestUtils {
return literal.getStringArrayValue().toArray(new String[0]);
}
+ public static byte[][] getBytesArrayValue(Literal literal) {
+ List<ByteBuffer> list = literal.getBytesArrayValue();
+ int size = list.size();
+ byte[][] array = new byte[size][];
+ for (int i = 0; i < size; i++) {
+ ByteBuffer buffer = list.get(i).duplicate();
+ byte[] bytes = new byte[buffer.remaining()];
+ buffer.get(bytes);
+ array[i] = bytes;
+ }
+ return array;
+ }
+
public static Pair<ColumnDataType, Object> getLiteralTypeAndValue(Literal
literal) {
Literal._Fields type = literal.getSetField();
switch (type) {
@@ -450,6 +484,8 @@ public class RequestUtils {
return Pair.of(ColumnDataType.DOUBLE_ARRAY,
getDoubleArrayValue(literal));
case STRING_ARRAY_VALUE:
return Pair.of(ColumnDataType.STRING_ARRAY,
getStringArrayValue(literal));
+ case BYTES_ARRAY_VALUE:
+ return Pair.of(ColumnDataType.BYTES_ARRAY,
getBytesArrayValue(literal));
default:
throw new IllegalStateException("Unsupported field type: " + type);
}
@@ -659,6 +695,9 @@ public class RequestUtils {
case STRING_ARRAY_VALUE:
return literal.getStringArrayValue().stream().map(value -> "'" + value
+ "'").collect(Collectors.toList())
.toString();
+ case BYTES_ARRAY_VALUE:
+ return Arrays.stream(getBytesArrayValue(literal)).map(value -> "X'" +
BytesUtils.toHexString(value) + "'")
+ .collect(Collectors.toList()).toString();
default:
throw new IllegalStateException("Unsupported field type: " + type);
}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/function/scalar/ArrayFunctionsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/function/scalar/ArrayFunctionsTest.java
new file mode 100644
index 00000000000..70ad0c2a334
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/function/scalar/ArrayFunctionsTest.java
@@ -0,0 +1,34 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.function.scalar;
+
+import org.testng.Assert;
+import org.testng.annotations.Test;
+
+
+/// Tests array construction behavior that depends on runtime Java element
types.
+public class ArrayFunctionsTest {
+
+ @Test
+ public void testBytesArrayValueConstructor() {
+ byte[][] expected = {{0}, {1, 2}};
+
+ Assert.assertEquals(ArrayFunctions.arrayValueConstructor(expected[0],
expected[1]), expected);
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/request/LiteralSerDeTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/request/LiteralSerDeTest.java
new file mode 100644
index 00000000000..1056f9a11ba
--- /dev/null
+++
b/pinot-common/src/test/java/org/apache/pinot/common/request/LiteralSerDeTest.java
@@ -0,0 +1,168 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.pinot.common.request;
+
+import java.nio.ByteBuffer;
+import java.util.List;
+import org.apache.pinot.common.utils.request.RequestUtils;
+import org.apache.pinot.sql.parsers.CalciteSqlParser;
+import org.apache.thrift.TDeserializer;
+import org.apache.thrift.TException;
+import org.apache.thrift.TSerializer;
+import org.apache.thrift.protocol.TBinaryProtocol;
+import org.apache.thrift.protocol.TCompactProtocol;
+import org.apache.thrift.protocol.TProtocolFactory;
+import org.testng.annotations.Test;
+
+import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertTrue;
+
+
+/// Verifies Thrift literal conversion and wire serialization.
+public class LiteralSerDeTest {
+ private static final short BYTES_ARRAY_FIELD_ID = 17;
+
+ @Test
+ public void testBytesArrayRoundTrip()
+ throws TException {
+ List<byte[][]> values = List.of(new byte[0][],
+ new byte[][]{{}, {0}, {1, 2}, {(byte) 0xff}});
+ List<TProtocolFactory> protocolFactories =
+ List.of(new TCompactProtocol.Factory(), new TBinaryProtocol.Factory());
+
+ for (TProtocolFactory protocolFactory : protocolFactories) {
+ for (byte[][] expected : values) {
+ PinotQuery query = new PinotQuery();
+
query.setSelectList(List.of(RequestUtils.getLiteralExpression(expected)));
+
+ byte[] serialized = new TSerializer(protocolFactory).serialize(query);
+ PinotQuery deserialized = new PinotQuery();
+ new TDeserializer(protocolFactory).deserialize(deserialized,
serialized);
+
+ Literal literal = deserialized.getSelectList().get(0).getLiteral();
+ assertTrue(literal.isSetBytesArrayValue());
+ assertEquals(literal.getSetField().getThriftFieldId(),
BYTES_ARRAY_FIELD_ID);
+ assertEquals(RequestUtils.getBytesArrayValue(literal), expected);
+ }
+ }
+ }
+
+ @Test
+ public void testBytesArrayConversionRespectsByteBufferBounds() {
+ ByteBuffer sliced = ByteBuffer.wrap(new byte[]{9, 0, 1, 2, 9});
+ sliced.position(1);
+ sliced.limit(4);
+ ByteBuffer direct = ByteBuffer.allocateDirect(2);
+ direct.put(new byte[]{3, 4});
+ direct.flip();
+
+ Literal literal = Literal.bytesArrayValue(List.of(sliced, direct));
+
+ assertEquals(RequestUtils.getBytesArrayValue(literal), new byte[][]{{0, 1,
2}, {3, 4}});
+ assertEquals(sliced.position(), 1);
+ assertEquals(direct.position(), 0);
+ }
+
+ @Test
+ public void testBytesArrayDeepCopy() {
+ Literal literal = RequestUtils.getLiteral(new byte[][]{{0}, {1, 2}});
+ Literal copy = literal.deepCopy();
+
+ assertEquals(copy, literal);
+ literal.getBytesArrayValue().get(0).put(0, (byte) 9);
+ assertEquals(RequestUtils.getBytesArrayValue(copy), new byte[][]{{0}, {1,
2}});
+ }
+
+ @Test
+ public void testSingleStageQueryUsesNativeBytesArrayLiteral()
+ throws TException {
+ for (String sql : List.of("SELECT ARRAY[X'00', X'0102'] FROM myTable",
+ "SELECT id FROM myTable WHERE ARRAYS_OVERLAP(bytesMV, ARRAY[X'01'])",
+ "SELECT ARRAY[X'02'], COUNT(*) FROM myTable GROUP BY ARRAY[X'02']",
+ "SELECT COUNT(*) FROM myTable HAVING
ARRAYS_OVERLAP(ARRAYAGG(bytesColumn, 'BYTES'), ARRAY[X'03'])",
+ "SELECT id FROM myTable "
+ + "ORDER BY CASE WHEN ARRAYS_OVERLAP(bytesMV, ARRAY[X'04']) THEN 1
ELSE 0 END")) {
+ PinotQuery query = CalciteSqlParser.compileToPinotQuery(sql);
+ assertTrue(containsBytesArrayLiteral(query));
+
+ byte[] serialized = new TSerializer(new
TCompactProtocol.Factory()).serialize(query);
+ PinotQuery deserialized = new PinotQuery();
+ new TDeserializer(new
TCompactProtocol.Factory()).deserialize(deserialized, serialized);
+ assertTrue(containsBytesArrayLiteral(deserialized));
+ }
+
+ PinotQuery nested = CalciteSqlParser.compileToPinotQuery(
+ "SELECT ARRAY_LENGTH(ARRAY[X'05', X'0607']) FROM myTable");
+ assertEquals(nested.getSelectList().get(0).getLiteral().getIntValue(), 2);
+ }
+
+ @Test
+ public void testExistingLiteralArmsStillRoundTrip()
+ throws TException {
+ for (Literal literal : List.of(RequestUtils.getLiteral(new byte[]{1, 2}),
+ RequestUtils.getLiteral(new String[]{"a", "b"}))) {
+ byte[] serialized = new TSerializer(new
TCompactProtocol.Factory()).serialize(literal);
+ Literal deserialized = new Literal();
+ new TDeserializer(new
TCompactProtocol.Factory()).deserialize(deserialized, serialized);
+ assertEquals(deserialized, literal);
+ }
+ }
+
+ private static boolean containsBytesArrayLiteral(Expression expression) {
+ if (expression.isSetLiteral()) {
+ return expression.getLiteral().isSetBytesArrayValue();
+ } else if (expression.isSetFunctionCall()) {
+ for (Expression operand : expression.getFunctionCall().getOperands()) {
+ if (containsBytesArrayLiteral(operand)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
+ private static boolean containsBytesArrayLiteral(PinotQuery query) {
+ for (Expression expression : query.getSelectList()) {
+ if (containsBytesArrayLiteral(expression)) {
+ return true;
+ }
+ }
+ if (query.isSetFilterExpression() &&
containsBytesArrayLiteral(query.getFilterExpression())) {
+ return true;
+ }
+ if (query.isSetGroupByList()) {
+ for (Expression expression : query.getGroupByList()) {
+ if (containsBytesArrayLiteral(expression)) {
+ return true;
+ }
+ }
+ }
+ if (query.isSetHavingExpression() &&
containsBytesArrayLiteral(query.getHavingExpression())) {
+ return true;
+ }
+ if (query.isSetOrderByList()) {
+ for (Expression expression : query.getOrderByList()) {
+ if (containsBytesArrayLiteral(expression)) {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+}
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/request/context/LiteralContextTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/request/context/LiteralContextTest.java
index a89180cf566..7aeb4cd91de 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/request/context/LiteralContextTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/request/context/LiteralContextTest.java
@@ -19,6 +19,7 @@
package org.apache.pinot.common.request.context;
import java.math.BigDecimal;
+import java.nio.ByteBuffer;
import java.sql.Timestamp;
import java.util.List;
import java.util.UUID;
@@ -28,6 +29,7 @@ import org.apache.pinot.spi.utils.BigDecimalUtils;
import org.apache.pinot.spi.utils.BytesUtils;
import org.apache.pinot.spi.utils.CommonConstants.NullValuePlaceHolder;
import org.apache.pinot.spi.utils.UuidUtils;
+import org.testng.annotations.DataProvider;
import org.testng.annotations.Test;
import static org.testng.Assert.*;
@@ -236,6 +238,48 @@ public class LiteralContextTest {
assertEquals(literalContext.toString(), "'deadbeef'");
}
+ @Test
+ public void testBytesArrayLiteral() {
+ Literal literal = Literal.bytesArrayValue(List.of(ByteBuffer.wrap(new
byte[0]), ByteBuffer.wrap(new byte[]{0}),
+ ByteBuffer.wrap(new byte[]{1, 2}), ByteBuffer.wrap(new byte[]{(byte)
0xff})));
+ LiteralContext literalContext = new LiteralContext(literal);
+
+ assertFalse(literalContext.isSingleValue());
+ assertEquals(literalContext.getType(), DataType.BYTES);
+ assertEquals(literalContext.getValue(), new byte[][]{{}, {0}, {1, 2},
{(byte) 0xff}});
+ assertEquals(literalContext.toString(), "'[, 00, 0102, ff]'");
+
+ LiteralContext equalContext =
+ new LiteralContext(DataType.BYTES, new byte[][]{{}, {0}, {1, 2},
{(byte) 0xff}});
+ assertEquals(equalContext, literalContext);
+ assertEquals(equalContext.hashCode(), literalContext.hashCode());
+ assertNotEquals(new LiteralContext(DataType.BYTES, new byte[][]{{}, {0},
{1, 3}, {(byte) 0xff}}), literalContext);
+ }
+
+ @Test(dataProvider = "arrayBackedValues")
+ public void testArrayBackedEqualityAndHashCode(DataType dataType, Object
value, Object equalValue,
+ Object differentValue) {
+ LiteralContext literalContext = new LiteralContext(dataType, value);
+ LiteralContext equalContext = new LiteralContext(dataType, equalValue);
+
+ assertEquals(equalContext, literalContext);
+ assertEquals(equalContext.hashCode(), literalContext.hashCode());
+ assertNotEquals(new LiteralContext(dataType, differentValue),
literalContext);
+ }
+
+ @DataProvider(name = "arrayBackedValues")
+ public Object[][] arrayBackedValues() {
+ return new Object[][]{
+ {DataType.INT, new int[]{1, 2}, new int[]{1, 2}, new int[]{1, 3}},
+ {DataType.LONG, new long[]{1L, 2L}, new long[]{1L, 2L}, new long[]{1L,
3L}},
+ {DataType.FLOAT, new float[]{1.0f, 2.0f}, new float[]{1.0f, 2.0f}, new
float[]{1.0f, 3.0f}},
+ {DataType.DOUBLE, new double[]{1.0, 2.0}, new double[]{1.0, 2.0}, new
double[]{1.0, 3.0}},
+ {DataType.STRING, new String[]{"one", "two"}, new String[]{"one",
"two"}, new String[]{"one", "three"}},
+ {DataType.BYTES, new byte[]{1, 2}, new byte[]{1, 2}, new byte[]{1, 3}},
+ {DataType.BYTES, new byte[][]{{1}, {2}}, new byte[][]{{1}, {2}}, new
byte[][]{{1}, {3}}}
+ };
+ }
+
@Test
public void testUuidLiteral() {
UUID uuid = UUID.fromString("550e8400-e29b-41d4-a716-446655440000");
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/DataSchemaTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/DataSchemaTest.java
index ba3bdceef4e..28fdaa09d80 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/utils/DataSchemaTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/DataSchemaTest.java
@@ -232,6 +232,23 @@ public class DataSchemaTest {
Assert.assertEquals(BYTES.format(bytesValue),
BytesUtils.toHexString(bytesValue));
}
+ @Test
+ public void testObjectToInternalBytesArray() {
+ Assert.assertEquals((ByteArray[]) OBJECT.toInternal(new byte[0][]), new
ByteArray[0]);
+
+ byte[][] externalBytesArray = {{}, {0}, {1, 2, (byte) 0xFF}};
+ ByteArray[] expected = {new ByteArray(externalBytesArray[0]), new
ByteArray(externalBytesArray[1]),
+ new ByteArray(externalBytesArray[2])};
+ Assert.assertEquals((ByteArray[]) OBJECT.toInternal(externalBytesArray),
expected);
+ }
+
+ @Test
+ public void testObjectToInternalUuidArray() {
+ java.util.UUID[] externalUuidArray = {JAVA_UUID, JAVA_UUID_2};
+ ByteArray[] expected = {new ByteArray(UuidUtils.toBytes(JAVA_UUID)), new
ByteArray(UuidUtils.toBytes(JAVA_UUID_2))};
+ Assert.assertEquals((ByteArray[]) OBJECT.toInternal(externalUuidArray),
expected);
+ }
+
/// The null placeholder must be resolved on the *logical* type. UUID is the
only type whose placeholder differs
/// from its stored type's: it needs the 16-byte nil UUID, while BYTES
supplies a zero-length one. Every other
/// logical type must agree with its stored type, otherwise callers that
resolve the stored type (DataBlockBuilder,
diff --git
a/pinot-common/src/test/java/org/apache/pinot/common/utils/request/RequestUtilsTest.java
b/pinot-common/src/test/java/org/apache/pinot/common/utils/request/RequestUtilsTest.java
index c489740fb07..e9acb5336de 100644
---
a/pinot-common/src/test/java/org/apache/pinot/common/utils/request/RequestUtilsTest.java
+++
b/pinot-common/src/test/java/org/apache/pinot/common/utils/request/RequestUtilsTest.java
@@ -30,6 +30,7 @@ import org.apache.pinot.common.request.ExpressionType;
import org.apache.pinot.common.request.Function;
import org.apache.pinot.common.request.Identifier;
import org.apache.pinot.common.request.Literal;
+import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
import org.apache.pinot.spi.utils.UuidUtils;
import org.apache.pinot.sql.parsers.CalciteSqlParser;
import org.apache.pinot.sql.parsers.PinotSqlType;
@@ -110,6 +111,27 @@ public class RequestUtilsTest {
assertEquals(expression.getLiteral().getBinaryValue(),
UuidUtils.toBytes(uuid));
}
+ @Test
+ public void testBytesArrayLiteralRepresentations() {
+ byte[][] expected = {{0}, {(byte) 0xde, (byte) 0xad, (byte) 0xbe, (byte)
0xef}};
+ Literal nativeLiteral = RequestUtils.getLiteral(expected);
+ assertTrue(nativeLiteral.isSetBytesArrayValue());
+ assertEquals(RequestUtils.getBytesArrayValue(nativeLiteral), expected);
+ assertEquals(RequestUtils.getLiteralTypeAndValue(nativeLiteral).getLeft(),
ColumnDataType.BYTES_ARRAY);
+ assertEquals(RequestUtils.prettyPrint(nativeLiteral), "[X'00',
X'deadbeef']");
+
+ Expression expression = CalciteSqlParser.compileToPinotQuery(
+ "SELECT ARRAY[X'00', X'DEADBEEF'] FROM
myTable").getSelectList().get(0);
+ assertTrue(expression.isSetLiteral());
+ assertTrue(expression.getLiteral().isSetBytesArrayValue());
+ assertEquals(RequestUtils.getBytesArrayValue(expression.getLiteral()),
expected);
+
+ expression = CalciteSqlParser.compileToPinotQuery(
+ "SELECT ARRAYS_OVERLAP(ARRAY[X'00', X'0102'], ARRAY[X'03',
X'0102'])").getSelectList().get(0);
+ assertTrue(expression.isSetLiteral());
+ assertTrue(expression.getLiteral().getBoolValue());
+ }
+
@Test
public void testParseQuery() {
SqlNodeAndOptions result = RequestUtils.parseQuery("select foo from
countries where bar > 1");
diff --git a/pinot-common/src/thrift/query.thrift
b/pinot-common/src/thrift/query.thrift
index 728031ca695..92f919c512a 100644
--- a/pinot-common/src/thrift/query.thrift
+++ b/pinot-common/src/thrift/query.thrift
@@ -96,6 +96,7 @@ union Literal {
14: optional list<i32> floatArrayValue;
15: optional list<double> doubleArrayValue;
16: optional list<string> stringArrayValue;
+ 17: optional list<binary> bytesArrayValue;
}
struct Identifier {
diff --git
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java
index 1919d1ee4c1..a2c8ad3544f 100644
---
a/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java
+++
b/pinot-core/src/main/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunction.java
@@ -45,6 +45,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
private final float[] _floatArrayLiteral;
private final double[] _doubleArrayLiteral;
private final String[] _stringArrayLiteral;
+ private final byte[][] _bytesArrayLiteral;
// NOTE:
// This class can be shared across multiple threads, and the result arrays
are lazily initialized and cached. They
@@ -55,6 +56,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
private volatile float[][] _floatArrayResult;
private volatile double[][] _doubleArrayResult;
private volatile String[][] _stringArrayResult;
+ private volatile byte[][][] _bytesArrayResult;
public ArrayLiteralTransformFunction(LiteralContext literalContext) {
_dataType = literalContext.getType();
@@ -65,6 +67,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_floatArrayLiteral = new float[0];
_doubleArrayLiteral = new double[0];
_stringArrayLiteral = new String[0];
+ _bytesArrayLiteral = new byte[0][];
return;
}
switch (_dataType) {
@@ -74,6 +77,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_floatArrayLiteral = null;
_doubleArrayLiteral = null;
_stringArrayLiteral = null;
+ _bytesArrayLiteral = null;
break;
case LONG:
_longArrayLiteral = (long[]) value;
@@ -81,6 +85,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_floatArrayLiteral = null;
_doubleArrayLiteral = null;
_stringArrayLiteral = null;
+ _bytesArrayLiteral = null;
break;
case FLOAT:
_floatArrayLiteral = (float[]) value;
@@ -88,6 +93,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_longArrayLiteral = null;
_doubleArrayLiteral = null;
_stringArrayLiteral = null;
+ _bytesArrayLiteral = null;
break;
case DOUBLE:
_doubleArrayLiteral = (double[]) value;
@@ -95,6 +101,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_longArrayLiteral = null;
_floatArrayLiteral = null;
_stringArrayLiteral = null;
+ _bytesArrayLiteral = null;
break;
case STRING:
_stringArrayLiteral = (String[]) value;
@@ -102,6 +109,15 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_longArrayLiteral = null;
_floatArrayLiteral = null;
_doubleArrayLiteral = null;
+ _bytesArrayLiteral = null;
+ break;
+ case BYTES:
+ _bytesArrayLiteral = (byte[][]) value;
+ _intArrayLiteral = null;
+ _longArrayLiteral = null;
+ _floatArrayLiteral = null;
+ _doubleArrayLiteral = null;
+ _stringArrayLiteral = null;
break;
default:
throw new IllegalStateException(
@@ -119,6 +135,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_floatArrayLiteral = new float[0];
_doubleArrayLiteral = new double[0];
_stringArrayLiteral = new String[0];
+ _bytesArrayLiteral = new byte[0][];
return;
}
for (ExpressionContext literalContext : literalContexts) {
@@ -136,6 +153,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_floatArrayLiteral = null;
_doubleArrayLiteral = null;
_stringArrayLiteral = null;
+ _bytesArrayLiteral = null;
break;
case LONG:
_longArrayLiteral = new long[literalContexts.size()];
@@ -146,6 +164,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_floatArrayLiteral = null;
_doubleArrayLiteral = null;
_stringArrayLiteral = null;
+ _bytesArrayLiteral = null;
break;
case FLOAT:
_floatArrayLiteral = new float[literalContexts.size()];
@@ -156,6 +175,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_longArrayLiteral = null;
_doubleArrayLiteral = null;
_stringArrayLiteral = null;
+ _bytesArrayLiteral = null;
break;
case DOUBLE:
_doubleArrayLiteral = new double[literalContexts.size()];
@@ -166,6 +186,7 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_longArrayLiteral = null;
_floatArrayLiteral = null;
_stringArrayLiteral = null;
+ _bytesArrayLiteral = null;
break;
case STRING:
_stringArrayLiteral = new String[literalContexts.size()];
@@ -176,6 +197,18 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
_longArrayLiteral = null;
_floatArrayLiteral = null;
_doubleArrayLiteral = null;
+ _bytesArrayLiteral = null;
+ break;
+ case BYTES:
+ _bytesArrayLiteral = new byte[literalContexts.size()][];
+ for (int i = 0; i < _bytesArrayLiteral.length; i++) {
+ _bytesArrayLiteral[i] =
literalContexts.get(i).getLiteral().getBytesValue();
+ }
+ _intArrayLiteral = null;
+ _longArrayLiteral = null;
+ _floatArrayLiteral = null;
+ _doubleArrayLiteral = null;
+ _stringArrayLiteral = null;
break;
default:
throw new IllegalStateException(
@@ -204,6 +237,10 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
return _stringArrayLiteral;
}
+ public byte[][] getBytesArrayLiteral() {
+ return _bytesArrayLiteral;
+ }
+
@Override
public String getName() {
return FUNCTION_NAME;
@@ -485,7 +522,14 @@ public class ArrayLiteralTransformFunction implements
TransformFunction {
@Override
public byte[][][] transformToBytesValuesMV(ValueBlock valueBlock) {
- throw new UnsupportedOperationException();
+ int numDocs = valueBlock.getNumDocs();
+ byte[][][] bytesArrayResult = _bytesArrayResult;
+ if (bytesArrayResult == null || bytesArrayResult.length < numDocs) {
+ bytesArrayResult = new byte[numDocs][][];
+ Arrays.fill(bytesArrayResult, _bytesArrayLiteral);
+ _bytesArrayResult = bytesArrayResult;
+ }
+ return bytesArrayResult;
}
@Override
diff --git
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunctionTest.java
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunctionTest.java
index 2bbdab74e0d..bac0e366177 100644
---
a/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunctionTest.java
+++
b/pinot-core/src/test/java/org/apache/pinot/core/operator/transform/function/ArrayLiteralTransformFunctionTest.java
@@ -22,8 +22,11 @@ import java.util.ArrayList;
import java.util.List;
import org.apache.pinot.common.request.Literal;
import org.apache.pinot.common.request.context.ExpressionContext;
+import org.apache.pinot.common.request.context.LiteralContext;
+import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.core.operator.blocks.ProjectionBlock;
import org.apache.pinot.spi.data.FieldSpec.DataType;
+import org.apache.pinot.spi.utils.BytesUtils;
import org.mockito.Mock;
import org.mockito.MockitoAnnotations;
import org.testng.Assert;
@@ -123,6 +126,28 @@ public class ArrayLiteralTransformFunctionTest {
});
}
+ @Test
+ public void testBytesArrayLiteralTransformFunction() {
+ byte[][] expected = {BytesUtils.toBytes("00"),
BytesUtils.toBytes("deadbeef")};
+ List<ExpressionContext> arrayExpressions = List.of(
+ ExpressionContext.forLiteral(DataType.BYTES, expected[0]),
+ ExpressionContext.forLiteral(DataType.BYTES, expected[1]));
+
+ List<ArrayLiteralTransformFunction> bytesArrays = List.of(new
ArrayLiteralTransformFunction(arrayExpressions),
+ new ArrayLiteralTransformFunction(new
LiteralContext(RequestUtils.getLiteral(expected))));
+ for (ArrayLiteralTransformFunction bytesArray : bytesArrays) {
+ Assert.assertEquals(bytesArray.getResultMetadata().getDataType(),
DataType.BYTES);
+ Assert.assertFalse(bytesArray.getResultMetadata().isSingleValue());
+ Assert.assertEquals(bytesArray.getBytesArrayLiteral(), expected);
+
+ byte[][][] values =
bytesArray.transformToBytesValuesMV(_projectionBlock);
+ Assert.assertEquals(values.length, NUM_DOCS);
+ for (byte[][] value : values) {
+ Assert.assertEquals(value, expected);
+ }
+ }
+ }
+
@Test
public void testEmptyArrayTransform() {
List<ExpressionContext> arrayExpressions = new ArrayList<>();
diff --git
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesMvTypeTest.java
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesMvTypeTest.java
index 5e7f2c8c337..25b440b931f 100644
---
a/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesMvTypeTest.java
+++
b/pinot-integration-tests/src/test/java/org/apache/pinot/integration/tests/custom/BytesMvTypeTest.java
@@ -35,6 +35,7 @@ import org.apache.pinot.spi.utils.builder.TableConfigBuilder;
import org.testng.annotations.Test;
import static org.testng.Assert.assertEquals;
+import static org.testng.Assert.assertFalse;
import static org.testng.Assert.assertTrue;
@@ -183,6 +184,66 @@ public class BytesMvTypeTest extends
CustomDataQueryClusterIntegrationTest {
}
}
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testBytesArrayLiteral(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ String arrayLiteral = "ARRAY[X'07', X'0708', X'07090A']";
+ for (boolean withFrom : new boolean[]{true, false}) {
+ String query = withFrom ? String.format("SELECT %s FROM %s WHERE %s = 7
LIMIT 1", arrayLiteral,
+ getTableName(), ID_COLUMN) : "SELECT " + arrayLiteral;
+ JsonNode result = postQuery(query).get("resultTable");
+
assertEquals(result.get("dataSchema").get("columnDataTypes").get(0).asText(),
"BYTES_ARRAY");
+ JsonNode values = result.get("rows").get(0).get(0);
+ assertEquals(values.size(), MV_LENGTH);
+ assertEquals(values.get(0).asText(), "07");
+ assertEquals(values.get(1).asText(), "0708");
+ assertEquals(values.get(2).asText(), "07090a");
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testBytesArrayLiteralRejectsUnsupportedElements(boolean
useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ for (String expression : List.of("ARRAY[X'00', NULL]", "ARRAY[NULL,
X'00']", "ARRAY[X'00', 1]",
+ "ARRAY[1, X'00']")) {
+ JsonNode response = postQuery("SELECT " + expression);
+ assertFalse(response.path("exceptions").isEmpty(),
+ "Expected unsupported BYTES array elements to be rejected: " +
response.toPrettyString());
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testArraysOverlapWithLiteral(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ for (String mvCol : MV_COLUMNS) {
+ String positiveQuery = String.format(
+ "SELECT COUNT(*) FROM %s WHERE ARRAYS_OVERLAP(%s, ARRAY[X'07'])",
getTableName(), mvCol);
+ JsonNode rows = postQuery(positiveQuery).get("resultTable").get("rows");
+ assertEquals(rows.get(0).get(0).asLong(), 1L);
+
+ String negativeQuery = String.format(
+ "SELECT COUNT(*) FROM %s WHERE ARRAYS_OVERLAP(%s, ARRAY[X'FF'])",
getTableName(), mvCol);
+ rows = postQuery(negativeQuery).get("resultTable").get("rows");
+ assertEquals(rows.get(0).get(0).asLong(), 0L);
+ }
+ }
+
+ @Test(dataProvider = "useBothQueryEngines")
+ public void testArraysOverlapWithLiterals(boolean useMultiStageQueryEngine)
+ throws Exception {
+ setUseMultiStageQueryEngine(useMultiStageQueryEngine);
+ JsonNode result = postQuery(
+ "SELECT ARRAYS_OVERLAP(ARRAY[X'00', X'0102'], ARRAY[X'03',
X'0102'])").get("resultTable");
+ assertTrue(result.get("rows").get(0).get(0).asBoolean());
+
+ result = postQuery(
+ "SELECT ARRAYS_OVERLAP(ARRAY[X'00', X'0102'], ARRAY[X'03',
X'04'])").get("resultTable");
+ assertFalse(result.get("rows").get(0).get(0).asBoolean());
+ }
+
@Test(dataProvider = "useBothQueryEngines")
public void testCountStar(boolean useMultiStageQueryEngine)
throws Exception {
diff --git
a/pinot-query-planner/src/main/java/org/apache/pinot/query/parser/CalciteRexExpressionParser.java
b/pinot-query-planner/src/main/java/org/apache/pinot/query/parser/CalciteRexExpressionParser.java
index d6f270d6f56..e4b7a3f35db 100644
---
a/pinot-query-planner/src/main/java/org/apache/pinot/query/parser/CalciteRexExpressionParser.java
+++
b/pinot-query-planner/src/main/java/org/apache/pinot/query/parser/CalciteRexExpressionParser.java
@@ -139,13 +139,20 @@ public class CalciteRexExpressionParser {
return RequestUtils.getNullLiteral();
}
// NOTE: Value is stored in internal format in RexExpression.Literal.
- // Do not convert TIMESTAMP/BOOLEAN_ARRAY/TIMESTAMP_ARRAY to
external format because they are not explicitly
- // supported in single-stage engine Literal.
+ // Do not convert TIMESTAMP/BOOLEAN_ARRAY/TIMESTAMP_ARRAY/UUID_ARRAY
to external format because they are not
+ // explicitly supported in single-stage engine Literal.
ColumnDataType dataType = literal.getDataType();
if (dataType == ColumnDataType.BOOLEAN) {
value = BooleanUtils.isTrueInternalValue(value);
} else if (dataType == ColumnDataType.BYTES || dataType ==
ColumnDataType.UUID) {
value = ((ByteArray) value).getBytes();
+ } else if (dataType == ColumnDataType.BYTES_ARRAY) {
+ ByteArray[] byteArrays = (ByteArray[]) value;
+ byte[][] bytes = new byte[byteArrays.length][];
+ for (int i = 0; i < byteArrays.length; i++) {
+ bytes[i] = byteArrays[i].getBytes();
+ }
+ value = bytes;
}
return RequestUtils.getLiteral(value);
}
diff --git
a/pinot-query-planner/src/test/java/org/apache/pinot/query/parser/CalciteRexExpressionParserTest.java
b/pinot-query-planner/src/test/java/org/apache/pinot/query/parser/CalciteRexExpressionParserTest.java
index c604a8dce59..08aca0c1d3f 100644
---
a/pinot-query-planner/src/test/java/org/apache/pinot/query/parser/CalciteRexExpressionParserTest.java
+++
b/pinot-query-planner/src/test/java/org/apache/pinot/query/parser/CalciteRexExpressionParserTest.java
@@ -20,6 +20,7 @@ package org.apache.pinot.query.parser;
import org.apache.pinot.common.request.Literal;
import org.apache.pinot.common.utils.DataSchema.ColumnDataType;
+import org.apache.pinot.common.utils.request.RequestUtils;
import org.apache.pinot.query.planner.logical.RexExpression;
import org.apache.pinot.spi.utils.ByteArray;
import org.apache.pinot.spi.utils.UuidUtils;
@@ -43,4 +44,16 @@ public class CalciteRexExpressionParserTest {
assertTrue(literal.isSetBinaryValue());
assertEquals(literal.getBinaryValue(), UuidUtils.toBytes(UUID_VALUE));
}
+
+ @Test
+ public void testBytesArrayLiteralUsesBytesArrayValue() {
+ byte[][] expected = {{0}, {1, 2}};
+ ByteArray[] internalValue = {new ByteArray(expected[0]), new
ByteArray(expected[1])};
+ RexExpression.Literal bytesArrayLiteral = new
RexExpression.Literal(ColumnDataType.BYTES_ARRAY, internalValue);
+
+ Literal literal = CalciteRexExpressionParser.toLiteral(bytesArrayLiteral);
+
+ assertTrue(literal.isSetBytesArrayValue());
+ assertEquals(RequestUtils.getBytesArrayValue(literal), expected);
+ }
}
diff --git
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/serde/RexExpressionSerDeTest.java
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/serde/RexExpressionSerDeTest.java
index ffd8a62e76c..d09fcd8d02e 100644
---
a/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/serde/RexExpressionSerDeTest.java
+++
b/pinot-query-planner/src/test/java/org/apache/pinot/query/planner/serde/RexExpressionSerDeTest.java
@@ -38,7 +38,8 @@ public class RexExpressionSerDeTest {
ColumnDataType.BIG_DECIMAL, ColumnDataType.BOOLEAN,
ColumnDataType.TIMESTAMP, ColumnDataType.STRING,
ColumnDataType.BYTES, ColumnDataType.UUID, ColumnDataType.INT_ARRAY,
ColumnDataType.LONG_ARRAY,
ColumnDataType.FLOAT_ARRAY, ColumnDataType.DOUBLE_ARRAY,
ColumnDataType.BOOLEAN_ARRAY,
- ColumnDataType.TIMESTAMP_ARRAY, ColumnDataType.STRING_ARRAY,
ColumnDataType.UUID_ARRAY,
+ ColumnDataType.TIMESTAMP_ARRAY, ColumnDataType.STRING_ARRAY,
ColumnDataType.BYTES_ARRAY,
+ ColumnDataType.UUID_ARRAY,
ColumnDataType.UNKNOWN);
private static final Random RANDOM = new Random();
@@ -167,6 +168,12 @@ public class RexExpressionSerDeTest {
verifyLiteralSerDe(new RexExpression.Literal(ColumnDataType.STRING_ARRAY,
values));
}
+ @Test
+ public void testBytesArrayLiteral() {
+ ByteArray[] values = {new ByteArray(new byte[0]), new ByteArray(new
byte[]{1, 2, 3})};
+ verifyLiteralSerDe(new RexExpression.Literal(ColumnDataType.BYTES_ARRAY,
values));
+ }
+
@Test
public void testUuidArrayLiteral() {
ByteArray[] values = {
diff --git
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/TransformOperatorTest.java
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/TransformOperatorTest.java
index ca33f5c1fd1..71759056b4a 100644
---
a/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/TransformOperatorTest.java
+++
b/pinot-query-runtime/src/test/java/org/apache/pinot/query/runtime/operator/TransformOperatorTest.java
@@ -114,6 +114,58 @@ public class TransformOperatorTest {
assertEquals(resultRows.get(1), new Object[]{5.0, -1.0});
}
+ @Test
+ public void shouldHandleBytesArrayLiteralTransform() {
+ DataSchema inputSchema = new DataSchema(new String[]{"intCol"}, new
ColumnDataType[]{ColumnDataType.INT});
+ when(_input.nextBlock()).thenReturn(
+ OperatorTestUtil.block(inputSchema, new Object[]{1}, new Object[]{2},
new Object[]{3}));
+ DataSchema resultSchema =
+ new DataSchema(new String[]{"bytesArray"}, new
ColumnDataType[]{ColumnDataType.BYTES_ARRAY});
+ ByteArray first = new ByteArray(new byte[]{0});
+ ByteArray second = new ByteArray(new byte[]{1, 2});
+ List<RexExpression> operands = List.of(new
RexExpression.Literal(ColumnDataType.BYTES, first),
+ new RexExpression.Literal(ColumnDataType.BYTES, second));
+ List<RexExpression> projects = List.of(
+ new RexExpression.FunctionCall(ColumnDataType.BYTES_ARRAY,
"ARRAY_VALUE_CONSTRUCTOR", operands));
+
+ TransformOperator operator = getOperator(inputSchema, resultSchema,
projects);
+ List<Object[]> resultRows = ((MseBlock.Data)
operator.nextBlock()).asRowHeap().getRows();
+ assertEquals(resultRows.size(), 3);
+ for (Object[] resultRow : resultRows) {
+ assertEquals((ByteArray[]) resultRow[0], new ByteArray[]{first, second});
+ }
+ }
+
+ @Test
+ public void shouldHandleDynamicBytesArrayTransform() {
+ DataSchema inputSchema =
+ new DataSchema(new String[]{"left", "right"}, new
ColumnDataType[]{ColumnDataType.BYTES, ColumnDataType.BYTES});
+ ByteArray literal = new ByteArray(new byte[]{0});
+ ByteArray left0 = new ByteArray(new byte[]{1});
+ ByteArray right0 = new ByteArray(new byte[]{2});
+ ByteArray left1 = new ByteArray(new byte[]{3});
+ ByteArray right1 = new ByteArray(new byte[]{4});
+ ByteArray left2 = new ByteArray(new byte[]{5});
+ ByteArray right2 = new ByteArray(new byte[]{6});
+ when(_input.nextBlock()).thenReturn(OperatorTestUtil.block(inputSchema,
new Object[]{left0, right0},
+ new Object[]{left1, right1}, new Object[]{left2, right2}));
+ DataSchema resultSchema =
+ new DataSchema(new String[]{"bytesArray"}, new
ColumnDataType[]{ColumnDataType.BYTES_ARRAY});
+ List<RexExpression> operands = List.of(new
RexExpression.Literal(ColumnDataType.BYTES, literal),
+ new RexExpression.InputRef(0), new RexExpression.InputRef(1));
+ List<RexExpression> projects = List.of(
+ new RexExpression.FunctionCall(ColumnDataType.BYTES_ARRAY,
"ARRAY_VALUE_CONSTRUCTOR", operands));
+
+ TransformOperator operator = getOperator(inputSchema, resultSchema,
projects);
+ List<Object[]> resultRows = ((MseBlock.Data)
operator.nextBlock()).asRowHeap().getRows();
+ assertEquals(resultRows.size(), 3);
+ ByteArray[][] expected = {{literal, left0, right0}, {literal, left1,
right1}, {literal, left2, right2}};
+ for (int i = 0; i < expected.length; i++) {
+ ByteArray[] actual = (ByteArray[]) resultRows.get(i)[0];
+ assertEquals(actual, expected[i]);
+ }
+ }
+
@Test
public void shouldRenderUuidToStringAsCanonicalText() {
String uuid = "550e8400-e29b-41d4-a716-446655440000";
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
