This is an automated email from the ASF dual-hosted git repository.
mbutrovich pushed a change to branch df55
in repository https://gitbox.apache.org/repos/asf/datafusion-comet.git
from c245c414f0 try 4g MEMORY_OFFHEAP_SIZE in CometTPCHQuerySuite
add 2af9cec38b feat: support `explode_outer` (#5192)
add 5b8a45e061 refactor: replace hand-coded rollup of expression fallback
reasons onto operators (#5236)
add 07307b5c53 perf: cache Iceberg reflection lookups on the planning path
(#5222)
add 0fe9b38779 feat: support timestampadd and timestampdiff via codegen
dispatch (#5030)
add 4cfd0739c3 perf: vectorize integer-to-decimal cast (#4939)
add f262b13d21 feat: support _metadata constant columns in native Parquet
scan (#5237)
add d3b1072dbe test: rename CometCastSuite to CometNativeCastSuite (#5268)
add 1feef7aba8 perf: compute spark_size list lengths with Arrow length
kernel (#5233)
add f22d9f17bb refactor: use arity helper for Int to Decimal128
reinterpretation (#5193)
add 874e7a64af Merge branch 'main' into df55
No new revisions were added by this update.
Summary of changes:
.github/workflows/pr_build_linux.yml | 2 +-
.github/workflows/pr_build_macos.yml | 2 +-
docs/source/contributor-guide/development.md | 8 +-
.../expression-audits/conversion_funcs.md | 1 +
.../expression-audits/generator_funcs.md | 4 +-
.../contributor-guide/optimizing_expressions.md | 3 +-
.../contributor-guide/spark_configs_support.md | 2 +-
.../user-guide/latest/compatibility/scans.md | 4 +-
docs/source/user-guide/latest/expressions.md | 21 +-
docs/source/user-guide/latest/operators.md | 12 +-
.../user-guide/latest/understanding-comet-plans.md | 32 +-
native/core/src/execution/columnar_to_row.rs | 181 +++++---
.../execution/expressions/list_empty_to_null.rs | 311 +++++++++++++
.../src/execution/expressions/list_positions.rs | 172 ++++++-
native/core/src/execution/expressions/mod.rs | 1 +
native/core/src/execution/planner.rs | 110 ++++-
native/spark-expr/Cargo.toml | 7 +
native/spark-expr/benches/array_size.rs | 75 +++-
native/spark-expr/benches/cast_int_to_decimal.rs | 98 ++++
native/spark-expr/benches/make_decimal.rs | 164 +++++++
native/spark-expr/src/array_funcs/size.rs | 256 ++++++++---
native/spark-expr/src/conversion_funcs/numeric.rs | 159 +++++--
native/spark-expr/src/conversion_funcs/string.rs | 4 +-
.../src/math_funcs/internal/make_decimal.rs | 219 ++++++++-
.../main/scala/org/apache/comet/CometConf.scala | 14 +
.../apache/comet/CometSparkSessionExtensions.scala | 62 +--
.../org/apache/comet/expressions/CometCast.scala | 3 +-
.../apache/comet/iceberg/IcebergReflection.scala | 236 ++++++----
.../org/apache/comet/rules/CometExecRule.scala | 86 +++-
.../org/apache/comet/rules/CometScanRule.scala | 56 +--
.../comet/serde/CometBloomFilterMightContain.scala | 2 -
.../apache/comet/serde/CometScalarFunction.scala | 4 +-
.../org/apache/comet/serde/CometSortOrder.scala | 2 -
.../org/apache/comet/serde/QueryPlanSerde.scala | 62 +--
.../scala/org/apache/comet/serde/aggregates.scala | 50 +--
.../scala/org/apache/comet/serde/arithmetic.scala | 8 +-
.../main/scala/org/apache/comet/serde/arrays.scala | 57 +--
.../scala/org/apache/comet/serde/bitwise.scala | 4 +-
.../scala/org/apache/comet/serde/conditional.scala | 6 -
.../apache/comet/serde/contraintExpressions.scala | 47 +-
.../scala/org/apache/comet/serde/datetime.scala | 39 +-
.../apache/comet/serde/decimalExpressions.scala | 6 +-
.../main/scala/org/apache/comet/serde/json.scala | 4 +-
.../main/scala/org/apache/comet/serde/maps.scala | 10 +-
.../main/scala/org/apache/comet/serde/math.scala | 26 +-
.../org/apache/comet/serde/namedExpressions.scala | 6 +-
.../serde/operator/CometIcebergNativeScan.scala | 160 ++++---
.../comet/serde/operator/CometNativeScan.scala | 25 +-
.../org/apache/comet/serde/operator/package.scala | 50 ++-
.../scala/org/apache/comet/serde/predicates.scala | 6 +-
.../scala/org/apache/comet/serde/statics.scala | 11 +-
.../scala/org/apache/comet/serde/strings.scala | 20 +-
.../scala/org/apache/comet/serde/structs.scala | 5 +-
.../scala/org/apache/comet/serde/unixtime.scala | 5 +-
.../main/scala/org/apache/comet/serde/url.scala | 4 +-
.../spark/sql/comet/CometNativeScanExec.scala | 7 +-
.../apache/spark/sql/comet/CometWindowExec.scala | 32 +-
.../org/apache/spark/sql/comet/operators.scala | 41 +-
.../org/apache/comet/shims/ShimFileFormat.scala | 38 +-
.../apache/comet/serde/CometToPrettyString.scala | 2 -
.../org/apache/comet/shims/ShimFileFormat.scala | 18 +-
.../org/apache/comet/shims/ShimFileFormat.scala | 18 +-
.../org/apache/comet/shims/CometExprShim.scala | 8 +-
.../org/apache/comet/shims/ShimFileFormat.scala | 18 +-
.../org/apache/comet/shims/ShimFileFormat.scala | 17 +-
.../org/apache/comet/serde/CometMapSort.scala | 4 +-
.../apache/comet/serde/CometToPrettyString.scala | 2 -
.../org/apache/comet/shims/CometExprShim4x.scala | 8 +-
.../sql-tests/expressions/array/explode.sql | 500 +++++++++++++++++++++
.../sql-tests/expressions/array/posexplode.sql | 213 ++++++++-
.../expressions/cast/cast_string_to_date_ansi.sql | 2 +-
.../expressions/cast/cast_string_trim.sql | 2 +-
.../expressions/datetime/date_add_unit_alias.sql | 61 +++
.../sql-tests/expressions/datetime/timediff.sql | 48 ++
.../expressions/datetime/timestampadd.sql | 113 +++++
.../expressions/datetime/timestampdiff.sql | 99 ++++
.../approved-plans-v1_4/q9/extended.txt | 2 +-
.../org/apache/comet/CometExpressionSuite.scala | 14 +-
...tCastSuite.scala => CometNativeCastSuite.scala} | 26 +-
.../apache/comet/exec/CometGenerateExecSuite.scala | 211 ++++++++-
.../comet/iceberg/IcebergReflectionSuite.scala | 98 +++-
.../apache/comet/parquet/ParquetReadSuite.scala | 190 ++++++++
.../apache/comet/rules/CometExecRuleSuite.scala | 100 ++++-
.../apache/comet/rules/CometScanRuleSuite.scala | 20 +
.../scala/org/apache/spark/sql/CometTestBase.scala | 4 +
85 files changed, 4037 insertions(+), 803 deletions(-)
create mode 100644 native/core/src/execution/expressions/list_empty_to_null.rs
create mode 100644 native/spark-expr/benches/cast_int_to_decimal.rs
create mode 100644 native/spark-expr/benches/make_decimal.rs
copy spark/src/main/{spark-3.5 =>
spark-4.0}/org/apache/comet/shims/ShimFileFormat.scala (62%)
copy spark/src/main/{spark-3.5 =>
spark-4.1}/org/apache/comet/shims/ShimFileFormat.scala (62%)
rename spark/src/main/{spark-4.x =>
spark-4.2}/org/apache/comet/shims/ShimFileFormat.scala (63%)
create mode 100644
spark/src/test/resources/sql-tests/expressions/array/explode.sql
create mode 100644
spark/src/test/resources/sql-tests/expressions/datetime/date_add_unit_alias.sql
create mode 100644
spark/src/test/resources/sql-tests/expressions/datetime/timediff.sql
create mode 100644
spark/src/test/resources/sql-tests/expressions/datetime/timestampadd.sql
create mode 100644
spark/src/test/resources/sql-tests/expressions/datetime/timestampdiff.sql
rename spark/src/test/scala/org/apache/comet/{CometCastSuite.scala =>
CometNativeCastSuite.scala} (98%)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
