This is an automated email from the ASF dual-hosted git repository.
andygrove pushed a change to branch andygrove/warn-compatible-convert-none
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
from 68e7b4d872 Merge remote-tracking branch 'apache/main' into
andygrove/warn-compatible-convert-none
add 92ad99e974 fix: keep the dictionary hash fast path off nested and
reseeded buffers (#5757)
add 99d3100c60 fix: preserve ANSI errors for rejected TIMESTAMP_NTZ casts
(#5752)
add 6368c4e619 test: restore Comet coverage for recursive HAVING and ORDER
BY (#5755)
add bb9e74020a fix: apply the parent struct's null mask before hashing its
fields (#5754)
add 5af898b2ef fix: read Iceberg tables partitioned by an unknown
transform (#5759)
add 31a69ee548 test: restore Parquet V2 writer and delta encoding coverage
(#5760)
add cbe3b39035 perf: evaluate posexplode array expressions once per batch
(#5737)
add cb4a261c05 chore: Add benches for datetime funcs (#5767)
add 6f25df70b3 perf: cache expected schemas for remote shuffle decoding
(#5722)
add e696f53bec feat: normalize marked Variant arrays at the native Parquet
boundary (#5715)
add 8d25379e6e bench: add a benchmark for the Spark hash kernels (#5765)
add 3734905902 test: restore Spark 4.1 Variant shredding suites (#5745)
add 8e6846850c feat: support native concat_ws with string arrays (#5725)
add 8c7b706a32 feat: native dynamic filter pushdown for hash join into
Parquet scans (#5699)
add 03875d4776 fix: native Iceberg write panics on an evolved partition
spec and on a timestamptz partition path (#5729)
add 4c14448c27 refactor: share one helper for pushing a struct's null mask
into its children (#5769)
add b56268d149 fix: let AQE optimize queries over Comet caches (#5733)
add fd5592f991 fix: dispatch Iceberg system functions wrapped as
ApplyFunctionExpression (#5773)
add 49a4b6c526 feat: enable codegen dispatch for `lpad` and `rpad` (#5764)
add 0d1348f353 feat: address remaining issues for `CreateArray` (#5766)
add b886032434 perf: use Arrow cast for date to timestamp NTZ (#5735)
add f6714a6cdd fix: attach tokio runtime threads to the JVM as daemon
threads (#5748)
add fefee03d94 ci: label pull requests by changed paths and title prefix
(#5762)
add 17f54da8ca test: cover ambiguous exact nested Parquet field matches
(#5751)
add 4eeb1f80f0 fix: check nested TIMESTAMP_MILLIS overflow in unfiltered
scans (#5740)
add e256ddb1f4 chore: run Preflight mandatory (#5784)
add 79d1be1eb3 fix: match iceberg-java's exception for unclustered input
to a clustered Iceberg write (#5779)
add 5627ab8c01 perf: reduce allocations when collecting cache statistics
(#5734)
add 424c31aa79 fix: enable FIRST/LAST partial merge (#5041)
add b77a2bc2e8 Merge remote-tracking branch 'apache/main' into
andygrove/warn-compatible-convert-none
No new revisions were added by this update.
Summary of changes:
.github/labeler.yml | 137 ++
.github/workflows/README.md | 103 +-
.github/workflows/ci.yml | 66 +-
.github/workflows/label_prs.yml | 87 ++
.github/workflows/pr_build_linux.yml | 2 +
.github/workflows/pr_build_macos.yml | 2 +
dev/ci/pr-type-label.mjs | 53 +
dev/ci/pr-type-label.test.mjs | 44 +
dev/diffs/3.4.3.diff | 16 +-
dev/diffs/3.5.9.diff | 16 +-
dev/diffs/4.0.4.diff | 40 +-
dev/diffs/4.1.3.diff | 88 +-
docs/source/contributor-guide/development.md | 8 +-
.../expression-audits/string_funcs.md | 7 +-
docs/source/user-guide/latest/expressions.md | 6 +-
docs/source/user-guide/latest/iceberg-writes.md | 9 +
docs/source/user-guide/latest/metrics.md | 20 +
docs/source/user-guide/latest/tuning.md | 20 +
native/Cargo.lock | 1 +
native/common/src/lib.rs | 2 +
native/common/src/struct_nulls.rs | 163 ++
native/core/Cargo.toml | 5 +
.../core/benches/parquet_timestamp_conversion.rs | 160 ++
native/core/src/execution/jni_api.rs | 102 +-
.../core/src/execution/operators/dynamic_filter.rs | 572 +++++++
.../execution/operators/dynamic_filter/tests.rs | 1642 ++++++++++++++++++++
native/core/src/execution/operators/filter.rs | 280 ++++
.../execution/operators/iceberg_partition_path.rs | 553 +++++++
.../core/src/execution/operators/iceberg_write.rs | 568 ++++++-
native/core/src/execution/operators/mod.rs | 5 +
native/core/src/execution/planner.rs | 536 ++++++-
native/core/src/parquet/cast_column.rs | 16 +
native/core/src/parquet/cast_column/variant.rs | 596 +++++++
.../core/src/parquet/cast_column/variant/tests.rs | 550 +++++++
native/core/src/parquet/parquet_support.rs | 347 ++++-
native/core/src/parquet/schema_adapter.rs | 135 +-
native/jni-bridge/src/errors.rs | 87 +-
native/proto/src/proto/operator.proto | 3 +
native/spark-expr/Cargo.toml | 44 +
native/spark-expr/benches/common/mod.rs | 44 +-
.../benches/{contains.rs => date_diff.rs} | 37 +-
.../{contains.rs => date_from_unix_date.rs} | 28 +-
.../benches/{levenshtein.rs => day_month_name.rs} | 31 +-
native/spark-expr/benches/extract_date_part.rs | 80 +
native/spark-expr/benches/hash.rs | 272 ++++
.../spark-expr/benches/{contains.rs => hours.rs} | 33 +-
.../benches/{contains.rs => make_date.rs} | 31 +-
.../benches/{contains.rs => make_interval.rs} | 39 +-
.../benches/{contains.rs => make_time.rs} | 42 +-
.../benches/{contains.rs => next_day.rs} | 33 +-
.../{array_position.rs => seconds_to_timestamp.rs} | 38 +-
.../src/array_funcs/get_array_struct_fields.rs | 32 +-
native/spark-expr/src/comet_scalar_funcs.rs | 3 +
native/spark-expr/src/conversion_funcs/string.rs | 43 +-
native/spark-expr/src/conversion_funcs/temporal.rs | 160 +-
native/spark-expr/src/hash_funcs/murmur3.rs | 196 ++-
native/spark-expr/src/hash_funcs/utils.rs | 27 +-
native/spark-expr/src/hash_funcs/xxhash64.rs | 131 +-
native/spark-expr/src/string_funcs/concat_ws.rs | 64 +
native/spark-expr/src/string_funcs/mod.rs | 2 +
.../src/struct_funcs/get_struct_field.rs | 38 +-
native/spark-expr/tests/spark_expr_reg.rs | 69 +
.../main/scala/org/apache/comet/CometConf.scala | 13 +
.../main/scala/org/apache/comet/GenerateDocs.scala | 11 +-
spark/src/main/scala/org/apache/comet/Native.scala | 16 +-
.../comet/codegen/CometBatchKernelCodegen.scala | 12 +-
.../apache/comet/iceberg/IcebergReflection.scala | 38 +
.../org/apache/comet/rules/CometExecRule.scala | 27 +-
.../org/apache/comet/serde/QueryPlanSerde.scala | 1 +
.../main/scala/org/apache/comet/serde/arrays.scala | 124 +-
.../org/apache/comet/serde/icebergFunctions.scala | 170 +-
.../scala/org/apache/comet/serde/literals.scala | 203 ++-
.../serde/operator/CometIcebergNativeScan.scala | 9 +-
.../scala/org/apache/comet/serde/statics.scala | 97 +-
.../scala/org/apache/comet/serde/strings.scala | 42 +-
.../sql/comet/CometInMemoryTableScanExec.scala | 15 +-
.../apache/spark/sql/comet/CometMetricNode.scala | 17 +
.../arrow/ArrowCachedBatchSerializer.scala | 258 ++-
.../shuffle/NativeBatchDecoderIterator.scala | 144 +-
.../org/apache/spark/sql/comet/operators.scala | 53 +-
.../shims/ShimCometInMemoryTableScanExec.scala} | 5 +-
.../shims/ShimCometInMemoryTableScanExec.scala} | 4 +-
.../org/apache/comet/shims/CometTypeShim.scala | 23 +
.../org/apache/comet/shims/CometTypeShim.scala | 3 +
....scala => ShimCometInMemoryTableScanExec.scala} | 4 +-
.../expressions/aggregate/partial_merge.sql | 30 +-
.../partial_merge_first_last.sql} | 22 +-
.../sql-tests/expressions/array/array_append.sql | 7 +
.../expressions/array/element_at_ansi.sql | 41 +
.../sql-tests/expressions/array/slice.sql | 18 +
.../expressions/cast/cast_timestamp_ntz.sql | 11 +
.../sql-tests/expressions/map/create_map.sql | 11 +
.../sql-tests/expressions/string/concat_ws.sql | 21 +-
.../sql-tests/expressions/string/string_lpad.sql | 19 +-
.../string_lpad_fallback.sql} | 27 +-
.../sql-tests/expressions/string/string_rpad.sql | 19 +-
.../string_rpad_fallback.sql} | 27 +-
.../apache/comet/CometArrayExpressionSuite.scala | 126 +-
.../apache/comet/CometHashExpressionSuite.scala | 97 +-
.../org/apache/comet/CometIcebergNativeSuite.scala | 120 +-
.../comet/CometIcebergSystemFunctionSuite.scala | 76 +-
.../org/apache/comet/CometIcebergTestBase.scala | 10 +-
.../comet/CometIcebergWriteActionSuite.scala | 328 +++-
.../org/apache/comet/CometNativeCastSuite.scala | 10 +
.../apache/comet/CometStringExpressionSuite.scala | 105 +-
.../apache/comet/exec/CometAggregateSuite.scala | 31 +-
.../apache/comet/exec/CometGenerateExecSuite.scala | 14 +
.../comet/exec/CometInMemoryCacheSuite.scala | 291 +++-
.../org/apache/comet/exec/CometJoinSuite.scala | 366 ++++-
.../apache/comet/exec/CometNativeReaderSuite.scala | 57 +-
.../comet/iceberg/IcebergReflectionSuite.scala | 51 +
.../apache/comet/parquet/ParquetReadSuite.scala | 24 +-
.../org/apache/comet/serde/CometLiteralSuite.scala | 126 ++
.../apache/spark/CometRuntimeShutdownSuite.scala | 113 ++
.../sql/benchmark/CometConcatWsBenchmark.scala | 109 ++
.../benchmark/CometInMemoryCacheBenchmark.scala | 43 +
...scala => CometPaddingExpressionBenchmark.scala} | 50 +-
.../CometCelebornShufflePlanningSuite.scala | 5 +-
.../shuffle/CometCelebornShuffleReaderSuite.scala | 61 +-
...tiveBatchDecoderIteratorConcurrencyChecks.scala | 384 +++++
...NativeBatchDecoderIteratorLifecycleChecks.scala | 204 ++-
121 files changed, 11648 insertions(+), 1259 deletions(-)
create mode 100644 .github/labeler.yml
create mode 100644 .github/workflows/label_prs.yml
create mode 100644 dev/ci/pr-type-label.mjs
create mode 100644 dev/ci/pr-type-label.test.mjs
create mode 100644 native/common/src/struct_nulls.rs
create mode 100644 native/core/benches/parquet_timestamp_conversion.rs
create mode 100644 native/core/src/execution/operators/dynamic_filter.rs
create mode 100644 native/core/src/execution/operators/dynamic_filter/tests.rs
create mode 100644 native/core/src/execution/operators/filter.rs
create mode 100644
native/core/src/execution/operators/iceberg_partition_path.rs
create mode 100644 native/core/src/parquet/cast_column/variant.rs
create mode 100644 native/core/src/parquet/cast_column/variant/tests.rs
copy native/spark-expr/benches/{contains.rs => date_diff.rs} (66%)
copy native/spark-expr/benches/{contains.rs => date_from_unix_date.rs} (71%)
copy native/spark-expr/benches/{levenshtein.rs => day_month_name.rs} (66%)
create mode 100644 native/spark-expr/benches/extract_date_part.rs
create mode 100644 native/spark-expr/benches/hash.rs
copy native/spark-expr/benches/{contains.rs => hours.rs} (66%)
copy native/spark-expr/benches/{contains.rs => make_date.rs} (71%)
copy native/spark-expr/benches/{contains.rs => make_interval.rs} (62%)
copy native/spark-expr/benches/{contains.rs => make_time.rs} (64%)
copy native/spark-expr/benches/{contains.rs => next_day.rs} (71%)
copy native/spark-expr/benches/{array_position.rs => seconds_to_timestamp.rs}
(68%)
create mode 100644 native/spark-expr/src/string_funcs/concat_ws.rs
copy
spark/src/main/{spark-4.x/org/apache/spark/sql/comet/shims/ShimCometShuffleWriteProcessor.scala
=>
spark-3.4/org/apache/spark/sql/comet/shims/ShimCometInMemoryTableScanExec.scala}
(82%)
copy
spark/src/main/{spark-4.x/org/apache/spark/sql/comet/shims/ShimCometShuffleWriteProcessor.scala
=>
spark-3.5/org/apache/spark/sql/comet/shims/ShimCometInMemoryTableScanExec.scala}
(86%)
copy
spark/src/main/spark-4.x/org/apache/spark/sql/comet/shims/{ShimCometShuffleWriteProcessor.scala
=> ShimCometInMemoryTableScanExec.scala} (86%)
copy spark/src/test/resources/sql-tests/expressions/{math/asinh.sql =>
aggregate/partial_merge_first_last.sql} (55%)
copy
spark/src/test/resources/sql-tests/expressions/{array/higher_order_function_fallback.sql
=> string/string_lpad_fallback.sql} (61%)
copy
spark/src/test/resources/sql-tests/expressions/{array/higher_order_function_fallback.sql
=> string/string_rpad_fallback.sql} (61%)
create mode 100644
spark/src/test/scala/org/apache/comet/serde/CometLiteralSuite.scala
create mode 100644
spark/src/test/scala/org/apache/spark/CometRuntimeShutdownSuite.scala
create mode 100644
spark/src/test/scala/org/apache/spark/sql/benchmark/CometConcatWsBenchmark.scala
copy
spark/src/test/scala/org/apache/spark/sql/benchmark/{CometPredicateExpressionBenchmark.scala
=> CometPaddingExpressionBenchmark.scala} (50%)
create mode 100644
spark/src/test/scala/org/apache/spark/sql/comet/execution/shuffle/NativeBatchDecoderIteratorConcurrencyChecks.scala
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
