This is an automated email from the ASF dual-hosted git repository.
parthchandra pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
The following commit(s) were added to refs/heads/main by this push:
new bf18cec52f feat: Implement TimeType support - Infrastructure - shuffle
(#4398)
bf18cec52f is described below
commit bf18cec52fbbbb0528754a5900f39c9ffbb1d3e5
Author: Bolin Lin <[email protected]>
AuthorDate: Fri Jul 17 12:05:15 2026 -0400
feat: Implement TimeType support - Infrastructure - shuffle (#4398)
* feat: support native shuffle for timetype
* feat: support columnar shuffle
* feat: support TimeType in list shuffle serialization
* feat: support TimeType hash partitioning
* test: cover JVM TimeType range partitioning
* test: refine make_time shuffle coverage
---
native/shuffle/src/spark_unsafe/list.rs | 8 ++-
native/shuffle/src/spark_unsafe/row.rs | 36 +++++++++++-
native/spark-expr/src/hash_funcs/utils.rs | 17 ++++++
.../comet/codegen/CometBatchKernelCodegen.scala | 6 +-
.../codegen/CometBatchKernelCodegenInput.scala | 15 ++++-
.../codegen/CometBatchKernelCodegenOutput.scala | 8 ++-
.../shuffle/CometShuffleExchangeExec.scala | 6 ++
.../sql-tests/expressions/datetime/make_time.sql | 13 ++---
.../expressions/datetime/make_time_shuffle.sql | 64 ++++++++++++++++++++++
.../make_time_shuffle_range_partition_jvm.sql | 34 ++++++++++++
.../make_time_shuffle_range_partition_native.sql | 34 ++++++++++++
11 files changed, 227 insertions(+), 14 deletions(-)
diff --git a/native/shuffle/src/spark_unsafe/list.rs
b/native/shuffle/src/spark_unsafe/list.rs
index 14f9feb843..22dce2cf79 100644
--- a/native/shuffle/src/spark_unsafe/list.rs
+++ b/native/shuffle/src/spark_unsafe/list.rs
@@ -24,7 +24,8 @@ use arrow::array::{
builder::{
ArrayBuilder, BinaryBuilder, BooleanBuilder, Date32Builder,
Decimal128Builder,
Float32Builder, Float64Builder, Int16Builder, Int32Builder,
Int64Builder, Int8Builder,
- ListBuilder, NullBuilder, StringBuilder, StructBuilder,
TimestampMicrosecondBuilder,
+ ListBuilder, NullBuilder, StringBuilder, StructBuilder,
Time64NanosecondBuilder,
+ TimestampMicrosecondBuilder,
},
MapBuilder,
};
@@ -179,6 +180,7 @@ impl SparkUnsafeArray {
impl_append_to_builder!(append_ints_to_builder, Int32Builder, i32);
impl_append_to_builder!(append_longs_to_builder, Int64Builder, i64);
+ impl_append_to_builder!(append_time64s_to_builder,
Time64NanosecondBuilder, i64);
impl_append_to_builder!(append_shorts_to_builder, Int16Builder, i16);
impl_append_to_builder!(append_bytes_to_builder, Int8Builder, i8);
impl_append_to_builder!(append_floats_to_builder, Float32Builder, f32);
@@ -377,6 +379,10 @@ pub fn append_to_builder<const NULLABLE: bool>(
let builder = downcast_builder_ref!(Int64Builder, builder);
array.append_longs_to_builder::<NULLABLE>(builder);
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ let builder = downcast_builder_ref!(Time64NanosecondBuilder,
builder);
+ array.append_time64s_to_builder::<NULLABLE>(builder);
+ }
DataType::Float32 => {
let builder = downcast_builder_ref!(Float32Builder, builder);
array.append_floats_to_builder::<NULLABLE>(builder);
diff --git a/native/shuffle/src/spark_unsafe/row.rs
b/native/shuffle/src/spark_unsafe/row.rs
index 449371463d..e0ebf8e7d0 100644
--- a/native/shuffle/src/spark_unsafe/row.rs
+++ b/native/shuffle/src/spark_unsafe/row.rs
@@ -29,7 +29,8 @@ use arrow::array::{
ArrayBuilder, BinaryBuilder, BinaryDictionaryBuilder, BooleanBuilder,
Date32Builder,
Decimal128Builder, Float32Builder, Float64Builder, Int16Builder,
Int32Builder,
Int64Builder, Int8Builder, ListBuilder, MapBuilder, NullBuilder,
StringBuilder,
- StringDictionaryBuilder, StructBuilder, TimestampMicrosecondBuilder,
+ StringDictionaryBuilder, StructBuilder, Time64NanosecondBuilder,
+ TimestampMicrosecondBuilder,
},
types::Int32Type,
Array, ArrayRef, RecordBatch, RecordBatchOptions,
@@ -278,6 +279,12 @@ pub(super) fn append_field(
.append_value(row.get_timestamp(idx))
);
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ append_field_to_builder!(
+ Time64NanosecondBuilder,
+ |builder: &mut Time64NanosecondBuilder|
builder.append_value(row.get_long(idx))
+ );
+ }
DataType::Binary => {
append_field_to_builder!(BinaryBuilder, |builder: &mut
BinaryBuilder| builder
.append_value(row.get_binary(idx)));
@@ -439,6 +446,13 @@ fn append_nested_struct_fields_field_major(
|row: &SparkUnsafeRow, idx| row.get_timestamp(idx)
);
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ process_field!(
+ Time64NanosecondBuilder,
+ field_idx,
+ |row: &SparkUnsafeRow, idx| row.get_long(idx)
+ );
+ }
DataType::Binary => {
let field_builder = get_field_builder!(struct_builder,
BinaryBuilder, field_idx);
@@ -656,6 +670,9 @@ fn append_list_column_batch(
DataType::Timestamp(TimeUnit::Microsecond, _) => {
process_primitive_lists!(TimestampMicrosecondBuilder,
append_timestamps_to_builder);
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ process_primitive_lists!(Time64NanosecondBuilder,
append_time64s_to_builder);
+ }
// For complex element types, fall back to per-row dispatch
_ => {
for i in row_start..row_end {
@@ -880,6 +897,13 @@ fn append_struct_fields_field_major(
|row: &SparkUnsafeRow, idx| row.get_timestamp(idx)
);
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ process_field!(
+ Time64NanosecondBuilder,
+ field_idx,
+ |row: &SparkUnsafeRow, idx| row.get_long(idx)
+ );
+ }
DataType::Binary => {
let field_builder = get_field_builder!(struct_builder,
BinaryBuilder, field_idx);
@@ -1165,6 +1189,13 @@ fn append_columns(
.append_value(row.get_timestamp(idx))
);
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ append_column_to_builder!(
+ Time64NanosecondBuilder,
+ |builder: &mut Time64NanosecondBuilder, row: &SparkUnsafeRow,
idx| builder
+ .append_value(row.get_long(idx))
+ );
+ }
DataType::Map(field, _) => {
let map_builder = downcast_builder_ref!(
MapBuilder<Box<dyn ArrayBuilder>, Box<dyn ArrayBuilder>>,
@@ -1266,6 +1297,9 @@ fn make_builders(
DataType::Timestamp(TimeUnit::Microsecond, _) => {
Box::new(TimestampMicrosecondBuilder::with_capacity(row_num).with_data_type(dt.clone()))
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ Box::new(Time64NanosecondBuilder::with_capacity(row_num))
+ }
DataType::Map(field, _) => {
let (key_field, value_field, map_field_names) =
get_map_key_value_fields(field)?;
let key_dt = key_field.data_type();
diff --git a/native/spark-expr/src/hash_funcs/utils.rs
b/native/spark-expr/src/hash_funcs/utils.rs
index d634b27bf8..979f787b59 100644
--- a/native/spark-expr/src/hash_funcs/utils.rs
+++ b/native/spark-expr/src/hash_funcs/utils.rs
@@ -380,6 +380,10 @@ macro_rules! hash_list_with_primitive_elements {
let elem_array =
$values.as_any().downcast_ref::<TimestampMicrosecondArray>().unwrap();
$crate::hash_list_primitive!(offsets: $offsets, $list_array,
elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ let elem_array =
$values.as_any().downcast_ref::<Time64NanosecondArray>().unwrap();
+ $crate::hash_list_primitive!(offsets: $offsets, $list_array,
elem_array, $hashes_buffer, $hash_method, |v: i64| v.to_le_bytes());
+ }
_ => {
// Fall back to recursive approach for complex element types
$crate::hash_list_array!($list_array_type,
$fallback_offset_type, $col, $hashes_buffer, $recursive_hash_method);
@@ -471,6 +475,10 @@ macro_rules! hash_list_with_primitive_elements {
let elem_array =
$values.as_any().downcast_ref::<TimestampMicrosecondArray>().unwrap();
$crate::hash_list_primitive!(fixed_size: $list_size,
$list_array, elem_array, $hashes_buffer, $hash_method, |v: i64|
v.to_le_bytes());
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ let elem_array =
$values.as_any().downcast_ref::<Time64NanosecondArray>().unwrap();
+ $crate::hash_list_primitive!(fixed_size: $list_size,
$list_array, elem_array, $hashes_buffer, $hash_method, |v: i64|
v.to_le_bytes());
+ }
_ => {
// Fall back to recursive approach for complex element types
if $list_array.null_count() == 0 {
@@ -689,6 +697,15 @@ macro_rules! create_hashes_internal {
$hash_method
);
}
+ DataType::Time64(TimeUnit::Nanosecond) => {
+ $crate::hash_array_primitive!(
+ Time64NanosecondArray,
+ col,
+ i64,
+ $hashes_buffer,
+ $hash_method
+ );
+ }
DataType::Utf8 => {
$crate::hash_array!(StringArray, col, $hashes_buffer,
$hash_method);
}
diff --git
a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala
b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala
index 107fb5e7f8..f56f95d065 100644
---
a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala
+++
b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegen.scala
@@ -28,7 +28,7 @@ import org.apache.spark.sql.catalyst.expressions.codegen._
import org.apache.spark.sql.internal.SQLConf
import org.apache.spark.sql.types._
-import org.apache.comet.shims.CometExprTraitShim
+import org.apache.comet.shims.{CometExprTraitShim, CometTypeShim}
/**
* Compiles a bound [[Expression]] plus an Arrow input schema into a
[[CometBatchKernel]] that
@@ -49,7 +49,7 @@ import org.apache.comet.shims.CometExprTraitShim
* The generated kernel is the `InternalRow` that Spark's
`BoundReference.genCode` reads from. See
* [[generateSource]] for how the wiring is set up.
*/
-object CometBatchKernelCodegen extends Logging with CometExprTraitShim {
+object CometBatchKernelCodegen extends Logging with CometExprTraitShim with
CometTypeShim {
/**
* Resolve an Arrow vector class by simple name through the codegen object's
own classloader.
@@ -67,6 +67,7 @@ object CometBatchKernelCodegen extends Logging with
CometExprTraitShim {
case "Float8Vector" => classOf[Float8Vector]
case "DecimalVector" => classOf[DecimalVector]
case "DateDayVector" => classOf[DateDayVector]
+ case "TimeNanoVector" => classOf[TimeNanoVector]
case "TimeStampMicroVector" => classOf[TimeStampMicroVector]
case "TimeStampMicroTZVector" => classOf[TimeStampMicroTZVector]
case "VarCharVector" => classOf[VarCharVector]
@@ -86,6 +87,7 @@ object CometBatchKernelCodegen extends Logging with
CometExprTraitShim {
case _: DecimalType => true
case _: StringType | _: BinaryType => true
case DateType | TimestampType | TimestampNTZType => true
+ case dt if isTimeType(dt) => true
case _: YearMonthIntervalType | _: DayTimeIntervalType => true
case ArrayType(inner, _) => isSupportedDataType(inner)
case st: StructType => st.fields.forall(f =>
isSupportedDataType(f.dataType))
diff --git
a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenInput.scala
b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenInput.scala
index 09bfc52bd4..f31c712c62 100644
---
a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenInput.scala
+++
b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenInput.scala
@@ -27,6 +27,7 @@ import
org.apache.spark.sql.catalyst.expressions.{BoundReference, Expression}
import org.apache.spark.sql.types._
import org.apache.comet.codegen.CometBatchKernelCodegen.{ArrayColumnSpec,
ArrowColumnSpec, MapColumnSpec, ScalarColumnSpec, StructColumnSpec}
+import org.apache.comet.shims.CometTypeShim
import org.apache.comet.vector.CometPlainVector
/**
@@ -44,7 +45,7 @@ import org.apache.comet.vector.CometPlainVector
* stashing references in an `OpenHashSet`) get distinct identities, and JIT
escape analysis
* usually scalarizes the allocation when the value is consumed locally.
*/
-private[codegen] object CometBatchKernelCodegenInput {
+private[codegen] object CometBatchKernelCodegenInput extends CometTypeShim {
/**
* Primitive Arrow vector classes wrapped in [[CometPlainVector]] at
input-cast time so per-row
@@ -61,6 +62,7 @@ private[codegen] object CometBatchKernelCodegenInput {
classOf[Float4Vector],
classOf[Float8Vector],
classOf[DateDayVector],
+ classOf[TimeNanoVector],
classOf[TimeStampMicroVector],
classOf[TimeStampMicroTZVector])
private val cometPlainVectorName: String = classOf[CometPlainVector].getName
@@ -133,6 +135,7 @@ private[codegen] object CometBatchKernelCodegenInput {
val longCases = withOrd.collect {
case (ArrowColumnSpec(cls, _), ord)
if cls == classOf[BigIntVector] ||
+ cls == classOf[TimeNanoVector] ||
cls == classOf[TimeStampMicroVector] ||
cls == classOf[TimeStampMicroTZVector] =>
s" case $ord: return this.col$ord.getLong(this.rowIdx);"
@@ -592,6 +595,7 @@ private[codegen] object CometBatchKernelCodegenInput {
case ShortType => s"getShort($idx)"
case IntegerType | DateType => s"getInt($idx)"
case LongType | TimestampType | TimestampNTZType => s"getLong($idx)"
+ case dt if isTimeType(dt) => s"getLong($idx)"
case FloatType => s"getFloat($idx)"
case DoubleType => s"getDouble($idx)"
case d: DecimalType => s"getDecimal($idx, ${d.precision}, ${d.scale})"
@@ -697,6 +701,11 @@ private[codegen] object CometBatchKernelCodegenInput {
| public long getLong(int i) {
| return $childField.getLong(startIndex + i);
| }""".stripMargin
+ case dt if isTimeType(dt) =>
+ s""" @Override
+ | public long getLong(int i) {
+ | return $childField.getLong(startIndex + i);
+ | }""".stripMargin
case FloatType =>
s""" @Override
| public float getFloat(int i) {
@@ -847,6 +856,8 @@ private[codegen] object CometBatchKernelCodegenInput {
s" case $fi: return ${path}_f$fi.getInt(this.rowIdx);"
case LongType | TimestampType | TimestampNTZType =>
s" case $fi: return ${path}_f$fi.getLong(this.rowIdx);"
+ case dt if isTimeType(dt) =>
+ s" case $fi: return ${path}_f$fi.getLong(this.rowIdx);"
case FloatType =>
s" case $fi: return ${path}_f$fi.getFloat(this.rowIdx);"
case DoubleType =>
@@ -897,7 +908,7 @@ private[codegen] object CometBatchKernelCodegenInput {
val longCases = scalarOrd.collect {
case (f, fi)
if f.sparkType == LongType || f.sparkType == TimestampType ||
- f.sparkType == TimestampNTZType =>
+ f.sparkType == TimestampNTZType || isTimeType(f.sparkType) =>
fieldReadScalar(fi, LongType, f.nullable)
}
val floatCases =
diff --git
a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala
b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala
index fe282cdc65..4160c478d1 100644
---
a/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala
+++
b/spark/src/main/scala/org/apache/comet/codegen/CometBatchKernelCodegenOutput.scala
@@ -31,13 +31,14 @@ import org.apache.spark.sql.comet.util.Utils
import org.apache.spark.sql.types._
import org.apache.comet.CometArrowAllocator
+import org.apache.comet.shims.CometTypeShim
/**
* Output-side emitters for the codegen kernel: [[allocateOutput]],
[[emitOutputWriter]]
* (top-level write entry), [[emitWrite]] (recursive per-type write), the
output vector-class
* lookup. Paired with [[CometBatchKernelCodegenInput]] on the read side.
*/
-private[codegen] object CometBatchKernelCodegenOutput {
+private[codegen] object CometBatchKernelCodegenOutput extends CometTypeShim {
/**
* Spark `DataType` to an Arrow `Field` with names Comet expects on FFI
export. Spark's
@@ -169,6 +170,7 @@ private[codegen] object CometBatchKernelCodegenOutput {
case _: StringType => classOf[VarCharVector].getName
case BinaryType => classOf[VarBinaryVector].getName
case DateType => classOf[DateDayVector].getName
+ case dt if isTimeType(dt) => classOf[TimeNanoVector].getName
case TimestampType => classOf[TimeStampMicroTZVector].getName
case TimestampNTZType => classOf[TimeStampMicroVector].getName
case _: YearMonthIntervalType => classOf[IntervalYearVector].getName
@@ -216,6 +218,9 @@ private[codegen] object CometBatchKernelCodegenOutput {
// DayTimeIntervalType -> DurationVector.set(int, long micros).
val set = if (nested) "setSafe" else "set"
OutputEmit("", s"$targetVec.$set($idx, $source);")
+ case dt if isTimeType(dt) =>
+ val set = if (nested) "setSafe" else "set"
+ OutputEmit("", s"$targetVec.$set($idx, $source);")
case dt: DecimalType =>
// DecimalOutputShortFastPath: precision <= 18 fits in a signed long, so
pass the unscaled
// value to `setSafe(int, long)` and skip the BigDecimal allocation.
@@ -398,6 +403,7 @@ private[codegen] object CometBatchKernelCodegenOutput {
case ShortType => s"$target.getShort($idx)"
case IntegerType | DateType => s"$target.getInt($idx)"
case LongType | TimestampType | TimestampNTZType =>
s"$target.getLong($idx)"
+ case dt if isTimeType(dt) => s"$target.getLong($idx)"
case FloatType => s"$target.getFloat($idx)"
case DoubleType => s"$target.getDouble($idx)"
case dt: DecimalType => s"$target.getDecimal($idx, ${dt.precision},
${dt.scale})"
diff --git
a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala
index 7629c3e59f..c9fe324bd8 100644
---
a/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala
+++
b/spark/src/main/scala/org/apache/spark/sql/comet/execution/shuffle/CometShuffleExchangeExec.scala
@@ -397,6 +397,8 @@ object CometShuffleExchangeExec
// Decimals with precision > 18 require Java BigDecimal conversion
before hashing
// d.precision <= 18
true
+ case dt if isTimeType(dt) =>
+ true
case _ =>
false
}
@@ -412,6 +414,8 @@ object CometShuffleExchangeExec
_: FloatType | _: DoubleType | _: StringType | _: BinaryType | _:
TimestampType |
_: TimestampNTZType | _: DecimalType | _: DateType | _: NullType =>
true
+ case dt if isTimeType(dt) =>
+ true
case StructType(fields) =>
fields.nonEmpty && fields.forall(f =>
supportedSerializableDataType(f.dataType))
case ArrayType(elementType, _) =>
@@ -535,6 +539,8 @@ object CometShuffleExchangeExec
_: FloatType | _: DoubleType | _: StringType | _: BinaryType | _:
TimestampType |
_: TimestampNTZType | _: DecimalType | _: DateType | _: NullType =>
true
+ case dt if isTimeType(dt) =>
+ true
case StructType(fields) =>
fields.nonEmpty && fields.forall(f =>
supportedSerializableDataType(f.dataType)) &&
// Java Arrow stream reader cannot work on duplicate field name
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/make_time.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/make_time.sql
index 8270e8f322..17391712cb 100644
--- a/spark/src/test/resources/sql-tests/expressions/datetime/make_time.sql
+++ b/spark/src/test/resources/sql-tests/expressions/datetime/make_time.sql
@@ -33,17 +33,16 @@ INSERT INTO test_make_time VALUES
(12, 30, NULL),
(NULL, NULL, NULL)
--- column arguments (spark_answer_only: shuffle does not support TimeType yet;
TODO: promote to
--- full native-verification once SPARK-51779 lands)
-query spark_answer_only
+-- column arguments (TODO: promote to full native-verification once
SPARK-51779 lands)
+query
SELECT hours, minutes, secs, make_time(hours, minutes, secs) FROM
test_make_time ORDER BY hours, minutes, secs
--- literal hour, column minutes and secs (spark_answer_only: shuffle does not
support TimeType yet)
-query spark_answer_only
+-- literal hour, column minutes and secs
+query
SELECT make_time(10, minutes, secs) FROM test_make_time ORDER BY minutes, secs
--- column hours, literal minutes and secs (spark_answer_only: shuffle does not
support TimeType yet)
-query spark_answer_only
+-- column hours, literal minutes and secs
+query
SELECT make_time(hours, 15, 30.5) FROM test_make_time ORDER BY hours
-- all literals
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle.sql
new file mode 100644
index 0000000000..024865b904
--- /dev/null
+++
b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle.sql
@@ -0,0 +1,64 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 4.1
+-- Config: spark.sql.timeType.enabled=true
+-- Config: spark.comet.native.shuffle.partitioning.roundrobin.enabled=true
+-- ConfigMatrix: spark.comet.exec.shuffle.mode=native,jvm
+
+statement
+CREATE TABLE test_make_time_shuffle(hours int, minutes int, secs
decimal(16,6)) USING parquet
+
+statement
+INSERT INTO test_make_time_shuffle VALUES (12, 30, 45.123456), (0, 0, 0.0),
(23, 59, 59.999999), (1, 2, 3.5),
+ (NULL, 0, 0.0), (NULL, NULL, NULL)
+
+query
+SELECT /*+ REPARTITION(3) */ make_time(hours, minutes, secs) AS t FROM
test_make_time_shuffle
+
+query
+SELECT /*+ REPARTITION(3) */ hours, make_time(hours, minutes, secs) AS t, secs
FROM test_make_time_shuffle
+
+query
+SELECT /*+ REPARTITION(3) */ named_struct('t', make_time(hours, minutes,
secs)) AS s FROM test_make_time_shuffle
+
+query
+SELECT /*+ REPARTITION(3) */ array(make_time(hours, minutes, secs)) AS arr
FROM test_make_time_shuffle
+
+query
+SELECT /*+ REPARTITION(3) */
+ array(make_time(hours, minutes, secs), make_time(0, 0, 0), CAST(NULL AS
TIME)) AS arr
+FROM test_make_time_shuffle
+
+query
+SELECT /*+ REPARTITION(3) */
+ CASE
+ WHEN hours IS NULL THEN CAST(array() AS ARRAY<TIME>)
+ ELSE array(make_time(hours, minutes, secs), make_time(23, 59,
59.999999))
+ END AS arr
+FROM test_make_time_shuffle
+
+query
+SELECT /*+ REPARTITION(3) */ map(make_time(hours, minutes, secs), hours) AS m
FROM test_make_time_shuffle
+WHERE hours IS NOT NULL AND minutes IS NOT NULL AND secs IS NOT NULL
+
+query
+SELECT /*+ REPARTITION(3, t) */ t
+FROM (
+ SELECT make_time(hours, minutes, secs) AS t
+ FROM test_make_time_shuffle
+)
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_jvm.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_jvm.sql
new file mode 100644
index 0000000000..36316adb87
--- /dev/null
+++
b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_jvm.sql
@@ -0,0 +1,34 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 4.1
+-- Config: spark.sql.timeType.enabled=true
+-- Config: spark.comet.exec.shuffle.mode=jvm
+
+statement
+CREATE TABLE test_make_time_range_shuffle_jvm(hours int, minutes int, secs
decimal(16,6)) USING parquet
+
+statement
+INSERT INTO test_make_time_range_shuffle_jvm VALUES
+ (12, 30, 45.123456), (0, 0, 0.0), (23, 59, 59.999999), (NULL, NULL, NULL)
+
+query
+SELECT /*+ REPARTITION_BY_RANGE(3, t) */ t
+FROM (
+ SELECT make_time(hours, minutes, secs) AS t
+ FROM test_make_time_range_shuffle_jvm
+)
diff --git
a/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_native.sql
b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_native.sql
new file mode 100644
index 0000000000..3627c87fe0
--- /dev/null
+++
b/spark/src/test/resources/sql-tests/expressions/datetime/make_time_shuffle_range_partition_native.sql
@@ -0,0 +1,34 @@
+-- Licensed to the Apache Software Foundation (ASF) under one
+-- or more contributor license agreements. See the NOTICE file
+-- distributed with this work for additional information
+-- regarding copyright ownership. The ASF licenses this file
+-- to you under the Apache License, Version 2.0 (the
+-- "License"); you may not use this file except in compliance
+-- with the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing,
+-- software distributed under the License is distributed on an
+-- "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+-- KIND, either express or implied. See the License for the
+-- specific language governing permissions and limitations
+-- under the License.
+
+-- MinSparkVersion: 4.1
+-- Config: spark.sql.timeType.enabled=true
+-- Config: spark.comet.exec.shuffle.mode=native
+
+statement
+CREATE TABLE test_make_time_range_shuffle_native(hours int, minutes int, secs
decimal(16,6)) USING parquet
+
+statement
+INSERT INTO test_make_time_range_shuffle_native VALUES
+ (12, 30, 45.123456), (0, 0, 0.0), (23, 59, 59.999999), (NULL, NULL, NULL)
+
+query expect_fallback(unsupported range partitioning data type for native
shuffle)
+SELECT /*+ REPARTITION_BY_RANGE(3, t) */ t
+FROM (
+ SELECT make_time(hours, minutes, secs) AS t
+ FROM test_make_time_range_shuffle_native
+)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
