This is an automated email from the ASF dual-hosted git repository.
blaginin pushed a change to branch db/mq-is-back
in repository https://gitbox.apache.org/repos/asf/datafusion.git
from 004f39220f Merge queue: make dev checks required + add .asf.yaml
validation
add aadae6bda3 Fix/support duplicate column names #6543 (#21126)
add e8831716f1 Use spot instances for extended tests (#21221)
add e57198ab92 feat(metric): Add output skewness metric to detect skewed
plans easier (#21211)
add cc5a3a7049 chore: Cleanup Cargo profiles (#21214)
add 06b4791d7f fix: Fix `main` compilation failure (#21242)
add 80a8bb8a7b feat: add sort pushdown benchmark and SLT tests (#21213)
add a0869e95ed chore(benchmark): Fix/update compile profile benchmark
(#21223)
add 81faba3293 Basic Extension Type Registry Implementation (#20312)
add 3b42dcf76f chore(deps): bump serialize-javascript,
terser-webpack-plugin and copy-webpack-plugin in
/datafusion/wasmtest/datafusion-wasm-app (#21235)
add eb68cf3aba chore(deps-dev): bump node-forge from 1.3.2 to 1.4.0 in
/datafusion/wasmtest/datafusion-wasm-app (#21225)
add 51d06df043 chore(deps): bump cryptography from 46.0.5 to 46.0.6
(#21224)
add 2aab55941d Fix FilterExec tree render missing fetch display (#21230)
add 0a8bb2933a fix: Revert "Fix/support duplicate column names #6543
(#21126)" (#21254)
add 010e5ee04b ci: use ubuntu-slim runner for lightweight CI jobs (#21252)
add 0be5982a91 perf: sort-merge join (SMJ) batch deferred filtering and
move mark joins to bitwise stream. Near-unique LEFT and FULL SMJ 20-50x faster
(#21184)
add 9385c70c77 kill `check_run_id` and `pr_number` from extended tests
(#21228)
add 3555322440 [Minor] add non topk benchmarks for utf8/utf8view string
aggregates (#21073)
add 2c08ddb4fe ci: Add datafusion/sql as a folder to trigger extended
tests for on changes (#21255)
add 11c2fbcb15 Misc minor optimization in the Physical Optimizer (#21216)
add 4a366752ac chore: Replace `TryInto` impl by `TryFrom` (#21203)
add f830ee39df Refactor parquet datasource into an explicit state machine
(#21190)
add 5ff80e4340 Provide session to the udtf call (#20222)
add 14a85fa383 chore: remove as_any from aggregate and window functions
(#21209)
add ccaf8022da Add flat vs. struct field projection benchmarks (#21257)
add 9c1e7abd2f Refactor: expose predicate constant inference from
physical-expr (#21167)
add 6412c3aaa0 Add end-to-end Parquet tests for List and LargeList struct
schema evolution (#20840)
add 2d9e268bc7 docs: Add `RESET` Command Documentation (#21245)
add 0bf9defb1d fix: Fix three bugs in query decorrelation (#21208)
add d87d8f6cac perf: Optimize `string_to_array` for scalar args (#21131)
add a120e4d1bd chore(deps): bump taiki-e/install-action from 2.69.7 to
2.70.3 (#21271)
add dcf818e4c5 chore(deps): bump rustyline from 17.0.2 to 18.0.0 (#21276)
add 26783124f1 chore(deps): bump ctor from 0.6.3 to 0.8.0 (#21282)
add 7138a832e3 chore(deps): bump snmalloc-rs from 0.3.8 to 0.7.4 (#21280)
add 4460ae0d26 chore(deps): bump sha1 from 0.10.6 to 0.11.0 (#21277)
add 55d1995624 chore(deps): bump astral-sh/setup-uv from 7.6.0 to 8.0.0
(#21272)
add 3d177fcc67 chore(deps): bump github/codeql-action from 4.34.1 to
4.35.1 (#21273)
add 19eb849b21 chore(deps): bump pygments from 2.19.2 to 2.20.0 (#21256)
add a9dc1dcac1 chore: fix upgrade guide link for object_store release
notes (#21283)
add c473c1852f feat(memory_pool): add `TrackConsumersPool::metrics()` to
expose cons… (#21147)
add e74b83ab02 fix: date overflow panic (#21233)
add 9de1253f2a Update repeat UDF to emit utf8view when input is utf8view
(#20645)
add fee90be613 Misc minor optimizations to query optimizer performance
(#21128)
add e75ed5b42c feat(sql): unparse array_has as ANY for Postgres (#20654)
add f3e8291ebd chore(deps): bump the all-other-cargo-deps group across 1
directory with 7 updates (#21274)
add bc2b36cf56 fix: `SELECT * EXCLUDE(...)` silently returns empty rows
when all columns are excluded (#21259)
add 1e68674705 doc: Add documentation explaining the behavior of `null`
values in struct comparisons (#21226)
add 2d5f016dda feat: feature-gate `sqllogictests` datafusion-substrait
behind optional 'substrait' feature (#21268)
add ba873e0d1c chore(deps): bump runs-on/action from 2.0.3 to 2.1.0
(#21134)
add 141f12b02e Merge branch 'main' into db/mq-is-back
No new revisions were added by this update.
Summary of changes:
.github/workflows/audit.yml | 2 +-
.github/workflows/codeql.yml | 6 +-
.github/workflows/dev.yml | 2 +-
.github/workflows/docs.yaml | 2 +-
.github/workflows/docs_pr.yaml | 2 +-
.github/workflows/extended.yml | 21 +-
.github/workflows/labeler.yml | 2 +-
.github/workflows/large_files.yml | 2 +-
.github/workflows/rust.yml | 38 +-
.github/workflows/stale.yml | 2 +-
.github/workflows/take.yml | 2 +-
.gitignore | 3 +
Cargo.lock | 269 ++--
Cargo.toml | 62 +-
benchmarks/Cargo.toml | 2 +-
benchmarks/README.md | 2 +-
benchmarks/bench.sh | 30 +
benchmarks/compile_profile.py | 43 +-
benchmarks/queries/sort_pushdown/q1.sql | 6 +
benchmarks/queries/sort_pushdown/q2.sql | 7 +
benchmarks/queries/sort_pushdown/q3.sql | 5 +
benchmarks/queries/sort_pushdown/q4.sql | 5 +
benchmarks/src/bin/dfbench.rs | 5 +-
benchmarks/src/lib.rs | 1 +
benchmarks/src/smj.rs | 80 +-
benchmarks/src/sort_pushdown.rs | 282 ++++
datafusion-cli/Cargo.toml | 2 +-
datafusion-cli/src/functions.rs | 14 +-
datafusion-examples/README.md | 31 +-
.../main.rs | 26 +-
.../examples/extension_types/temperature.rs | 323 ++++
datafusion-examples/examples/udf/advanced_udaf.rs | 11 +-
datafusion-examples/examples/udf/advanced_udwf.rs | 11 +-
datafusion-examples/examples/udf/main.rs | 8 +-
datafusion-examples/examples/udf/simple_udtf.rs | 5 +-
.../examples/udf/table_list_udtf.rs | 128 ++
datafusion/catalog/src/table.rs | 56 +-
datafusion/common/Cargo.toml | 2 +
datafusion/common/src/config.rs | 26 +-
.../src/types/canonical_extensions}/mod.rs | 2 +-
.../common/src/types/canonical_extensions/uuid.rs | 103 ++
datafusion/common/src/types/extension.rs | 80 +
datafusion/common/src/types/mod.rs | 3 +
.../core/benches/parquet_struct_projection.rs | 85 +-
datafusion/core/benches/topk_aggregate.rs | 338 ++--
datafusion/core/src/dataframe/mod.rs | 6 +
.../core/src/datasource/listing_table_factory.rs | 6 +
datafusion/core/src/execution/session_state.rs | 39 +-
.../core/src/execution/session_state_defaults.rs | 8 +
datafusion/core/src/lib.rs | 6 +-
datafusion/core/tests/core_integration.rs | 3 +
.../core/tests/{catalog => extension_types}/mod.rs | 2 +-
.../core/tests/extension_types/pretty_printing.rs | 78 +
datafusion/core/tests/fuzz_cases/join_fuzz.rs | 135 ++
datafusion/core/tests/parquet/expr_adapter.rs | 524 ++++++-
.../tests/physical_optimizer/enforce_sorting.rs | 3 +-
.../tests/physical_optimizer/limit_pushdown.rs | 35 +
datafusion/core/tests/sql/explain_analyze.rs | 5 +-
.../tests/user_defined/user_defined_aggregates.rs | 9 -
.../user_defined/user_defined_table_functions.rs | 7 +-
.../user_defined/user_defined_window_functions.rs | 13 -
datafusion/datasource-arrow/src/file_format.rs | 5 +
datafusion/datasource-parquet/src/mod.rs | 4 +
datafusion/datasource-parquet/src/opener.rs | 1258 ++++++++++-----
datafusion/datasource-parquet/src/reader.rs | 2 +-
datafusion/datasource/src/source.rs | 19 +-
datafusion/datasource/src/url.rs | 5 +
datafusion/execution/src/memory_pool/pool.rs | 83 +
datafusion/expr-common/src/interval_arithmetic.rs | 2 +
datafusion/expr/Cargo.toml | 3 +-
datafusion/expr/src/expr_fn.rs | 9 -
.../src/extension_types/array_formatter_factory.rs | 67 +
.../generator => expr/src/extension_types}/mod.rs | 9 +-
datafusion/expr/src/lib.rs | 1 +
datafusion/expr/src/logical_plan/builder.rs | 9 +
datafusion/expr/src/registry.rs | 307 +++-
datafusion/expr/src/test/function_stub.rs | 22 -
datafusion/expr/src/udaf.rs | 21 +-
datafusion/expr/src/udf_eq.rs | 4 +-
datafusion/expr/src/udwf.rs | 22 +-
.../ffi/src/proto/logical_extension_codec.rs | 4 +-
.../ffi/src/proto/physical_extension_codec.rs | 8 +-
datafusion/ffi/src/session/mod.rs | 30 +-
datafusion/ffi/src/udaf/mod.rs | 22 +-
datafusion/ffi/src/udtf.rs | 100 +-
datafusion/ffi/src/udwf/mod.rs | 14 +-
.../functions-aggregate/src/approx_distinct.rs | 5 -
.../functions-aggregate/src/approx_median.rs | 5 -
.../src/approx_percentile_cont.rs | 5 -
.../src/approx_percentile_cont_with_weight.rs | 5 -
datafusion/functions-aggregate/src/array_agg.rs | 4 -
datafusion/functions-aggregate/src/average.rs | 5 -
.../functions-aggregate/src/bit_and_or_xor.rs | 5 -
datafusion/functions-aggregate/src/bool_and_or.rs | 9 -
datafusion/functions-aggregate/src/correlation.rs | 6 -
datafusion/functions-aggregate/src/count.rs | 4 -
datafusion/functions-aggregate/src/covariance.rs | 8 -
datafusion/functions-aggregate/src/first_last.rs | 9 -
datafusion/functions-aggregate/src/grouping.rs | 6 -
datafusion/functions-aggregate/src/median.rs | 4 -
datafusion/functions-aggregate/src/min_max.rs | 8 -
datafusion/functions-aggregate/src/nth_value.rs | 5 -
.../functions-aggregate/src/percentile_cont.rs | 4 -
datafusion/functions-aggregate/src/regr.rs | 5 -
datafusion/functions-aggregate/src/stddev.rs | 11 -
datafusion/functions-aggregate/src/string_agg.rs | 5 -
datafusion/functions-aggregate/src/sum.rs | 5 -
datafusion/functions-aggregate/src/variance.rs | 8 -
datafusion/functions-nested/Cargo.toml | 5 +
.../functions-nested/benches/string_to_array.rs | 244 +++
datafusion/functions-nested/src/string.rs | 544 ++++---
datafusion/functions-table/src/generate_series.rs | 13 +-
datafusion/functions-window/src/cume_dist.rs | 6 -
datafusion/functions-window/src/lead_lag.rs | 5 -
datafusion/functions-window/src/macros.rs | 28 -
datafusion/functions-window/src/nth_value.rs | 5 -
datafusion/functions-window/src/ntile.rs | 5 -
datafusion/functions-window/src/rank.rs | 5 -
datafusion/functions-window/src/row_number.rs | 5 -
datafusion/functions/Cargo.toml | 2 +-
datafusion/functions/src/core/named_struct.rs | 4 +-
datafusion/functions/src/core/struct.rs | 4 +-
datafusion/functions/src/string/repeat.rs | 98 +-
datafusion/optimizer/src/analyzer/type_coercion.rs | 6 +-
.../optimizer/src/common_subexpr_eliminate.rs | 6 +-
datafusion/optimizer/src/decorrelate.rs | 34 +-
.../optimizer/src/extract_leaf_expressions.rs | 46 +-
datafusion/optimizer/src/optimize_unions.rs | 4 +-
datafusion/optimizer/src/push_down_filter.rs | 63 +-
datafusion/optimizer/src/push_down_limit.rs | 2 +-
.../optimizer/src/scalar_subquery_to_join.rs | 31 +-
.../src/simplify_expressions/expr_simplifier.rs | 61 +-
datafusion/optimizer/src/utils.rs | 46 +-
.../physical-expr-common/src/metrics/baseline.rs | 95 +-
.../physical-expr-common/src/metrics/value.rs | 82 +-
datafusion/physical-expr/src/aggregate.rs | 4 -
datafusion/physical-expr/src/utils/mod.rs | 94 +-
.../physical-optimizer/src/enforce_sorting/mod.rs | 5 +-
.../src/enforce_sorting/sort_pushdown.rs | 13 +-
.../physical-optimizer/src/join_selection.rs | 6 +-
.../physical-optimizer/src/limit_pushdown.rs | 119 +-
.../src/limited_distinct_aggregation.rs | 12 +-
.../physical-optimizer/src/output_requirements.rs | 9 +-
.../physical-optimizer/src/sanity_checker.rs | 35 +-
datafusion/physical-plan/src/filter.rs | 63 +-
datafusion/physical-plan/src/joins/mod.rs | 1 -
.../src/joins/semi_anti_sort_merge_join/mod.rs | 25 -
.../src/joins/semi_anti_sort_merge_join/tests.rs | 801 ----------
.../bitwise_stream.rs} | 98 +-
.../src/joins/sort_merge_join/exec.rs | 34 +-
.../src/joins/sort_merge_join/filter.rs | 405 ++---
.../{stream.rs => materializing_stream.rs} | 605 +++----
.../src/joins/sort_merge_join/metrics.rs | 10 +-
.../physical-plan/src/joins/sort_merge_join/mod.rs | 3 +-
.../src/joins/sort_merge_join/tests.rs | 1651 +++++++++++++++-----
datafusion/proto-common/src/from_proto/mod.rs | 1 -
datafusion/proto/tests/cases/mod.rs | 9 -
.../proto/tests/cases/roundtrip_logical_plan.rs | 8 +-
.../proto/tests/cases/roundtrip_physical_plan.rs | 5 +-
datafusion/session/src/session.rs | 4 +
datafusion/spark/Cargo.toml | 2 +-
datafusion/spark/src/function/aggregate/avg.rs | 6 +-
datafusion/spark/src/function/aggregate/collect.rs | 10 +-
datafusion/spark/src/function/aggregate/try_sum.rs | 5 -
datafusion/sql/src/unparser/dialect.rs | 23 +-
datafusion/sql/src/unparser/expr.rs | 20 +-
datafusion/sql/tests/cases/plan_to_sql.rs | 11 +
datafusion/sqllogictest/Cargo.toml | 7 +-
datafusion/sqllogictest/bin/sqllogictests.rs | 22 +-
datafusion/sqllogictest/src/engines/mod.rs | 2 +
datafusion/sqllogictest/src/lib.rs | 1 +
datafusion/sqllogictest/test_files/array.slt | 68 +
.../test_files/datetime/arith_date_interval.slt | 12 +
.../test_files/dynamic_filter_pushdown_config.slt | 2 +-
.../sqllogictest/test_files/explain_analyze.slt | 105 +-
.../sqllogictest/test_files/explain_tree.slt | 92 +-
datafusion/sqllogictest/test_files/limit.slt | 21 +-
.../test_files/schema_evolution_nested.slt | 124 ++
datafusion/sqllogictest/test_files/select.slt | 28 +-
.../sqllogictest/test_files/sort_merge_join.slt | 30 +
.../sqllogictest/test_files/sort_pushdown.slt | 457 ++++++
.../test_files/string/string_literal.slt | 24 +
datafusion/sqllogictest/test_files/subquery.slt | 68 +
datafusion/wasmtest/Cargo.toml | 2 +-
.../wasmtest/datafusion-wasm-app/package-lock.json | 508 ++----
.../wasmtest/datafusion-wasm-app/package.json | 2 +-
dev/update_config_docs.sh | 17 +-
.../library-user-guide/functions/adding-udfs.md | 15 +-
docs/source/library-user-guide/upgrading/53.0.0.md | 2 +-
docs/source/library-user-guide/upgrading/54.0.0.md | 45 +-
docs/source/user-guide/configs.md | 15 +
docs/source/user-guide/explain-usage.md | 1 +
docs/source/user-guide/sql/scalar_functions.md | 4 +
docs/source/user-guide/sql/struct_coercion.md | 20 +
docs/source/user-guide/sql/subqueries.md | 12 +
uv.lock | 106 +-
196 files changed, 8274 insertions(+), 4261 deletions(-)
create mode 100644 benchmarks/queries/sort_pushdown/q1.sql
create mode 100644 benchmarks/queries/sort_pushdown/q2.sql
create mode 100644 benchmarks/queries/sort_pushdown/q3.sql
create mode 100644 benchmarks/queries/sort_pushdown/q4.sql
create mode 100644 benchmarks/src/sort_pushdown.rs
copy datafusion-examples/examples/{external_dependency =>
extension_types}/main.rs (75%)
create mode 100644 datafusion-examples/examples/extension_types/temperature.rs
create mode 100644 datafusion-examples/examples/udf/table_list_udtf.rs
copy datafusion/{core/tests/catalog =>
common/src/types/canonical_extensions}/mod.rs (98%)
create mode 100644 datafusion/common/src/types/canonical_extensions/uuid.rs
create mode 100644 datafusion/common/src/types/extension.rs
copy datafusion/core/tests/{catalog => extension_types}/mod.rs (97%)
create mode 100644 datafusion/core/tests/extension_types/pretty_printing.rs
create mode 100644
datafusion/expr/src/extension_types/array_formatter_factory.rs
copy datafusion/{spark/src/function/generator =>
expr/src/extension_types}/mod.rs (84%)
create mode 100644 datafusion/functions-nested/benches/string_to_array.rs
delete mode 100644
datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/mod.rs
delete mode 100644
datafusion/physical-plan/src/joins/semi_anti_sort_merge_join/tests.rs
rename datafusion/physical-plan/src/joins/{semi_anti_sort_merge_join/stream.rs
=> sort_merge_join/bitwise_stream.rs} (93%)
rename datafusion/physical-plan/src/joins/sort_merge_join/{stream.rs =>
materializing_stream.rs} (80%)
create mode 100644
datafusion/sqllogictest/test_files/schema_evolution_nested.slt
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
