This is an automated email from the ASF dual-hosted git repository. blaginin pushed a commit to branch unprotect in repository https://gitbox.apache.org/repos/asf/datafusion-sandbox.git
commit fe5473f182df06db058f0fa9c81eb014de8d5d46 Merge: 0a16920b0 db7ce03e6 Author: blaginin <[email protected]> AuthorDate: Thu Oct 23 21:29:29 2025 +0100 Merge branch 'sandbox-main' into unprotect # Conflicts: # .asf.yaml .github/actions/setup-macos-builder/action.yaml | 47 - .github/actions/setup-rust-runtime/action.yaml | 9 - .github/dependabot.yml | 2 +- .github/workflows/audit.yml | 6 +- .github/workflows/dev.yml | 5 +- .github/workflows/rust.yml | 47 +- .gitignore | 1 + Cargo.lock | 1002 +++++++++++--------- Cargo.toml | 92 +- benchmarks/Cargo.toml | 2 +- benchmarks/README.md | 47 + benchmarks/bench.sh | 61 +- benchmarks/compile_profile.py | 180 ++++ benchmarks/src/bin/dfbench.rs | 6 +- benchmarks/src/hj.rs | 273 ++++++ benchmarks/src/imdb/run.rs | 2 +- benchmarks/src/lib.rs | 1 + benchmarks/src/tpch/run.rs | 2 +- .../scripts/license_header.sh | 11 +- datafusion-cli/CONTRIBUTING.md | 10 +- datafusion-cli/README.md | 7 +- datafusion-cli/src/main.rs | 12 +- .../examples/composed_extension_codec.rs | 14 +- datafusion-examples/examples/custom_file_casts.rs | 10 +- .../examples/date_time_functions.rs | 16 +- datafusion-examples/examples/expr_api.rs | 2 +- .../examples/flight/flight_server.rs | 2 +- .../examples/flight/flight_sql_server.rs | 6 +- datafusion/catalog-listing/Cargo.toml | 2 +- datafusion/catalog-listing/README.md | 8 +- datafusion/catalog/Cargo.toml | 2 +- datafusion/catalog/README.md | 7 +- datafusion/catalog/src/information_schema.rs | 5 +- datafusion/catalog/src/listing_schema.rs | 1 + datafusion/catalog/src/stream.rs | 2 +- datafusion/catalog/src/view.rs | 2 +- datafusion/common-runtime/README.md | 7 +- datafusion/common/Cargo.toml | 2 +- datafusion/common/README.md | 7 +- datafusion/common/src/cast.rs | 19 +- datafusion/common/src/config.rs | 8 + datafusion/common/src/dfschema.rs | 99 +- .../common/src/file_options/parquet_writer.rs | 13 +- datafusion/common/src/nested_struct.rs | 12 +- datafusion/common/src/param_value.rs | 2 +- datafusion/common/src/scalar/mod.rs | 555 +++++++++-- datafusion/common/src/types/native.rs | 26 +- datafusion/core/Cargo.toml | 3 +- datafusion/core/README.md | 13 +- datafusion/core/benches/sql_planner.rs | 6 +- datafusion/core/src/dataframe/mod.rs | 36 +- datafusion/core/src/dataframe/parquet.rs | 7 +- datafusion/core/src/datasource/file_format/csv.rs | 2 +- .../core/src/datasource/listing_table_factory.rs | 11 +- .../core/src/datasource/physical_plan/parquet.rs | 4 +- datafusion/core/src/execution/context/mod.rs | 58 +- datafusion/core/src/execution/session_state.rs | 8 + datafusion/core/src/physical_planner.rs | 272 +++--- datafusion/core/src/test_util/mod.rs | 2 +- datafusion/core/tests/dataframe/mod.rs | 240 ++++- .../tests/fuzz_cases/record_batch_generator.rs | 55 +- .../core/tests/fuzz_cases/topk_filter_pushdown.rs | 4 +- datafusion/core/tests/fuzz_cases/window_fuzz.rs | 6 +- datafusion/core/tests/macro_hygiene/mod.rs | 1 + .../memory_limit_validation/sort_mem_validation.rs | 8 +- datafusion/core/tests/parquet/filter_pushdown.rs | 101 +- datafusion/core/tests/parquet/schema_adapter.rs | 4 +- .../tests/physical_optimizer/enforce_sorting.rs | 2 +- .../physical_optimizer/filter_pushdown/mod.rs | 34 +- .../physical_optimizer/projection_pushdown.rs | 2 +- .../core/tests/physical_optimizer/test_utils.rs | 4 +- datafusion/core/tests/sql/sql_api.rs | 4 +- .../tests/user_defined/user_defined_aggregates.rs | 14 +- .../user_defined/user_defined_scalar_functions.rs | 95 +- .../user_defined/user_defined_window_functions.rs | 32 +- datafusion/datasource-avro/Cargo.toml | 2 +- datafusion/datasource-avro/README.md | 10 +- .../src/avro_to_arrow/arrow_array_reader.rs | 14 +- datafusion/datasource-csv/Cargo.toml | 2 +- datafusion/datasource-csv/README.md | 7 +- datafusion/datasource-json/Cargo.toml | 2 +- datafusion/datasource-json/README.md | 7 +- datafusion/datasource-parquet/Cargo.toml | 2 +- datafusion/datasource-parquet/README.md | 10 +- datafusion/datasource-parquet/src/file_format.rs | 84 +- datafusion/datasource-parquet/src/metadata.rs | 6 +- datafusion/datasource-parquet/src/metrics.rs | 17 + datafusion/datasource-parquet/src/opener.rs | 62 +- datafusion/datasource-parquet/src/row_filter.rs | 5 +- datafusion/datasource-parquet/src/source.rs | 7 + datafusion/datasource/Cargo.toml | 2 +- datafusion/datasource/README.md | 7 +- datafusion/datasource/src/file_scan_config.rs | 61 +- datafusion/datasource/src/schema_adapter.rs | 2 +- datafusion/doc/Cargo.toml | 1 + datafusion/doc/README.md | 7 +- datafusion/doc/src/lib.rs | 8 + .../src/function/map/mod.rs => doc/src/udaf.rs} | 32 +- datafusion/doc/src/udf.rs | 132 +++ datafusion/doc/src/udwf.rs | 47 + datafusion/execution/README.md | 7 +- datafusion/execution/src/task.rs | 8 + datafusion/expr-common/Cargo.toml | 1 + datafusion/expr-common/README.md | 7 +- datafusion/expr-common/src/casts.rs | 70 +- datafusion/expr-common/src/signature.rs | 13 +- datafusion/expr-common/src/statistics.rs | 2 +- .../expr-common/src/type_coercion/aggregates.rs | 53 +- datafusion/expr-common/src/type_coercion/binary.rs | 130 ++- .../src/type_coercion/binary/tests/arithmetic.rs | 57 +- datafusion/expr/Cargo.toml | 2 +- datafusion/expr/README.md | 7 +- datafusion/expr/src/conditional_expressions.rs | 8 +- datafusion/expr/src/expr.rs | 4 +- datafusion/expr/src/expr_schema.rs | 4 +- datafusion/expr/src/lib.rs | 19 +- datafusion/expr/src/logical_plan/builder.rs | 205 ++-- datafusion/expr/src/logical_plan/ddl.rs | 2 + datafusion/expr/src/logical_plan/display.rs | 8 +- datafusion/expr/src/logical_plan/plan.rs | 61 +- datafusion/expr/src/logical_plan/statement.rs | 3 +- datafusion/expr/src/registry.rs | 26 +- datafusion/expr/src/test/function_stub.rs | 22 +- datafusion/expr/src/type_coercion/functions.rs | 17 +- datafusion/expr/src/type_coercion/mod.rs | 10 +- datafusion/expr/src/udaf.rs | 48 +- datafusion/expr/src/udf.rs | 116 --- datafusion/expr/src/udwf.rs | 31 - datafusion/expr/src/window_state.rs | 16 +- datafusion/ffi/README.md | 22 +- datafusion/ffi/src/plan_properties.rs | 5 +- datafusion/ffi/src/udaf/accumulator_args.rs | 5 +- datafusion/ffi/src/udwf/partition_evaluator.rs | 2 +- .../ffi/src/udwf/partition_evaluator_args.rs | 2 +- datafusion/functions-aggregate-common/Cargo.toml | 1 + datafusion/functions-aggregate-common/README.md | 7 +- .../src/aggregate/avg_distinct.rs | 2 + .../src/aggregate/avg_distinct/decimal.rs | 282 ++++++ .../functions-aggregate-common/src/min_max.rs | 66 +- .../functions-aggregate-common/src/tdigest.rs | 4 +- datafusion/functions-aggregate/README.md | 7 +- .../functions-aggregate/src/approx_distinct.rs | 60 +- .../src/approx_percentile_cont.rs | 18 +- .../src/approx_percentile_cont_with_weight.rs | 31 +- datafusion/functions-aggregate/src/average.rs | 174 +++- .../functions-aggregate/src/bit_and_or_xor.rs | 6 +- datafusion/functions-aggregate/src/correlation.rs | 11 +- datafusion/functions-aggregate/src/first_last.rs | 20 +- datafusion/functions-aggregate/src/lib.rs | 2 + datafusion/functions-aggregate/src/median.rs | 8 +- datafusion/functions-aggregate/src/min_max.rs | 24 +- datafusion/functions-aggregate/src/regr.rs | 2 +- datafusion/functions-aggregate/src/sum.rs | 47 +- datafusion/functions-nested/README.md | 10 +- datafusion/functions-nested/src/array_has.rs | 74 +- datafusion/functions-nested/src/cardinality.rs | 6 - datafusion/functions-nested/src/concat.rs | 5 +- datafusion/functions-nested/src/distance.rs | 6 +- datafusion/functions-nested/src/except.rs | 8 +- datafusion/functions-nested/src/extract.rs | 35 +- datafusion/functions-nested/src/flatten.rs | 16 +- datafusion/functions-nested/src/length.rs | 35 +- datafusion/functions-nested/src/make_array.rs | 6 +- datafusion/functions-nested/src/planner.rs | 2 +- datafusion/functions-nested/src/position.rs | 4 +- datafusion/functions-nested/src/remove.rs | 18 +- datafusion/functions-nested/src/replace.rs | 6 +- datafusion/functions-nested/src/resize.rs | 6 +- datafusion/functions-nested/src/reverse.rs | 6 +- datafusion/functions-nested/src/sort.rs | 63 +- datafusion/functions-nested/src/string.rs | 104 +- datafusion/functions-nested/src/utils.rs | 8 +- datafusion/functions-table/README.md | 7 +- datafusion/functions-window-common/README.md | 7 +- datafusion/functions-window/README.md | 7 +- datafusion/functions-window/src/lead_lag.rs | 2 +- datafusion/functions-window/src/nth_value.rs | 2 +- datafusion/functions-window/src/rank.rs | 2 +- datafusion/functions/README.md | 7 +- datafusion/functions/benches/regx.rs | 13 +- datafusion/functions/src/core/getfield.rs | 4 +- datafusion/functions/src/core/named_struct.rs | 2 +- datafusion/functions/src/core/nvl2.rs | 2 +- datafusion/functions/src/datetime/from_unixtime.rs | 4 +- datafusion/functions/src/datetime/to_date.rs | 4 +- datafusion/functions/src/datetime/to_timestamp.rs | 13 +- datafusion/functions/src/datetime/to_unixtime.rs | 2 +- datafusion/functions/src/macros.rs | 10 +- datafusion/functions/src/math/abs.rs | 12 +- datafusion/functions/src/math/cot.rs | 8 + datafusion/functions/src/math/factorial.rs | 12 +- datafusion/functions/src/math/gcd.rs | 8 + datafusion/functions/src/math/iszero.rs | 8 + datafusion/functions/src/math/lcm.rs | 12 +- datafusion/functions/src/math/log.rs | 8 + datafusion/functions/src/math/monotonicity.rs | 241 ++++- datafusion/functions/src/math/nans.rs | 8 + datafusion/functions/src/math/nanvl.rs | 8 + datafusion/functions/src/math/power.rs | 12 +- datafusion/functions/src/math/random.rs | 10 +- datafusion/functions/src/math/round.rs | 10 +- datafusion/functions/src/math/signum.rs | 10 +- datafusion/functions/src/math/trunc.rs | 11 +- datafusion/functions/src/regex/regexpreplace.rs | 136 ++- datafusion/functions/src/string/concat.rs | 2 +- datafusion/functions/src/string/contains.rs | 2 +- datafusion/functions/src/string/replace.rs | 2 +- datafusion/macros/Cargo.toml | 5 +- datafusion/macros/README.md | 11 +- datafusion/macros/src/user_doc.rs | 2 +- datafusion/optimizer/README.md | 7 +- datafusion/optimizer/src/analyzer/type_coercion.rs | 47 +- .../optimizer/src/common_subexpr_eliminate.rs | 26 +- .../optimizer/src/extract_equijoin_predicate.rs | 123 ++- datafusion/optimizer/src/optimizer.rs | 7 +- datafusion/optimizer/src/push_down_filter.rs | 71 +- .../src/simplify_expressions/expr_simplifier.rs | 228 ++++- .../src/simplify_expressions/unwrap_cast.rs | 2 +- .../optimizer/src/simplify_expressions/utils.rs | 27 +- datafusion/physical-expr-adapter/README.md | 32 +- datafusion/physical-expr-common/README.md | 11 +- datafusion/physical-expr/README.md | 7 +- datafusion/physical-expr/src/expressions/binary.rs | 10 +- .../src/expressions/binary/kernels.rs | 4 +- datafusion/physical-expr/src/expressions/cast.rs | 2 +- .../src/expressions/dynamic_filters.rs | 39 +- .../physical-expr/src/expressions/in_list.rs | 4 +- .../physical-expr/src/expressions/try_cast.rs | 2 +- datafusion/physical-expr/src/planner.rs | 3 +- datafusion/physical-expr/src/utils/mod.rs | 33 +- datafusion/physical-expr/src/window/standard.rs | 8 +- datafusion/physical-optimizer/README.md | 8 +- datafusion/physical-optimizer/src/lib.rs | 2 +- datafusion/physical-plan/README.md | 7 +- datafusion/physical-plan/benches/spill_io.rs | 14 +- datafusion/physical-plan/src/display.rs | 2 +- datafusion/physical-plan/src/filter.rs | 14 +- datafusion/physical-plan/src/filter_pushdown.rs | 4 +- .../physical-plan/src/joins/hash_join/exec.rs | 19 +- .../physical-plan/src/joins/nested_loop_join.rs | 25 +- .../src/joins/sort_merge_join/exec.rs | 25 +- datafusion/physical-plan/src/joins/utils.rs | 35 +- datafusion/physical-plan/src/limit.rs | 3 +- .../physical-plan/src/sorts/multi_level_merge.rs | 9 +- datafusion/physical-plan/src/spill/mod.rs | 49 +- .../physical-plan/src/spill/spill_manager.rs | 2 + .../src/windows/bounded_window_agg_exec.rs | 51 +- datafusion/physical-plan/src/windows/mod.rs | 158 ++- datafusion/proto-common/Cargo.toml | 2 +- datafusion/proto-common/README.md | 14 +- datafusion/proto-common/gen/Cargo.toml | 4 +- .../proto-common/proto/datafusion_common.proto | 4 + datafusion/proto-common/src/from_proto/mod.rs | 16 +- datafusion/proto-common/src/generated/pbjson.rs | 26 +- datafusion/proto-common/src/generated/prost.rs | 111 ++- datafusion/proto-common/src/to_proto/mod.rs | 39 +- datafusion/proto/Cargo.toml | 9 +- datafusion/proto/README.md | 14 +- datafusion/proto/gen/Cargo.toml | 4 +- datafusion/proto/proto/datafusion.proto | 1 + datafusion/proto/src/bytes/mod.rs | 17 +- datafusion/proto/src/bytes/registry.rs | 8 + .../proto/src/generated/datafusion_proto_common.rs | 111 ++- datafusion/proto/src/generated/pbjson.rs | 18 + datafusion/proto/src/generated/prost.rs | 78 +- datafusion/proto/src/lib.rs | 2 +- datafusion/proto/src/logical_plan/file_formats.rs | 283 +++--- datafusion/proto/src/logical_plan/mod.rs | 3 + datafusion/proto/src/physical_plan/from_proto.rs | 31 +- datafusion/proto/src/physical_plan/mod.rs | 400 ++++---- .../proto/tests/cases/roundtrip_logical_plan.rs | 9 +- .../proto/tests/cases/roundtrip_physical_plan.rs | 42 +- datafusion/pruning/Cargo.toml | 1 + datafusion/{catalog => pruning}/README.md | 11 +- datafusion/pruning/src/pruning_predicate.rs | 8 +- datafusion/session/Cargo.toml | 2 +- datafusion/session/README.md | 7 +- datafusion/spark/README.md | 19 +- datafusion/spark/src/function/array/spark_array.rs | 2 +- .../spark/src/function/bitmap/bitmap_count.rs | 26 +- datafusion/spark/src/function/bitwise/bit_count.rs | 2 +- datafusion/spark/src/function/bitwise/bit_shift.rs | 2 +- datafusion/spark/src/function/datetime/date_sub.rs | 2 +- .../src/function/datetime/make_dt_interval.rs | 485 ++++++++++ .../spark/src/function/datetime/make_interval.rs | 581 ++++++++++++ datafusion/spark/src/function/datetime/mod.rs | 23 +- .../spark/src/function/map/map_from_arrays.rs | 105 ++ .../spark/src/function/map/map_from_entries.rs | 133 +++ datafusion/spark/src/function/map/mod.rs | 26 +- datafusion/spark/src/function/map/utils.rs | 231 +++++ datafusion/spark/src/function/math/factorial.rs | 4 +- datafusion/spark/src/function/math/hex.rs | 7 +- datafusion/spark/src/function/string/luhn_check.rs | 2 +- datafusion/spark/src/lib.rs | 2 + datafusion/sql/Cargo.toml | 4 +- datafusion/sql/README.md | 15 +- datafusion/sql/src/cte.rs | 8 +- datafusion/sql/src/expr/function.rs | 63 +- datafusion/sql/src/parser.rs | 145 ++- datafusion/sql/src/planner.rs | 4 +- datafusion/sql/src/query.rs | 78 +- datafusion/sql/src/relation/mod.rs | 2 +- datafusion/sql/src/select.rs | 10 +- datafusion/sql/src/statement.rs | 3 +- datafusion/sql/src/unparser/dialect.rs | 18 + datafusion/sql/src/unparser/expr.rs | 184 +++- datafusion/sql/tests/cases/params.rs | 23 +- datafusion/sql/tests/sql_integration.rs | 2 +- datafusion/sqllogictest/Cargo.toml | 6 +- datafusion/sqllogictest/README.md | 22 +- datafusion/sqllogictest/src/test_context.rs | 61 +- datafusion/sqllogictest/test_files/aggregate.slt | 257 +++-- .../test_files/aggregate_skip_partial.slt | 7 - datafusion/sqllogictest/test_files/array.slt | 268 +++++- datafusion/sqllogictest/test_files/async_udf.slt | 107 +++ .../test_files/create_external_table.slt | 6 + .../sqllogictest/test_files/create_function.slt | 5 - datafusion/sqllogictest/test_files/ddl.slt | 53 +- datafusion/sqllogictest/test_files/errors.slt | 2 +- .../sqllogictest/test_files/explain_tree.slt | 14 +- datafusion/sqllogictest/test_files/expr.slt | 6 - datafusion/sqllogictest/test_files/group_by.slt | 8 - .../sqllogictest/test_files/information_schema.slt | 2 + datafusion/sqllogictest/test_files/join.slt.part | 10 + .../test_files/join_is_not_distinct_from.slt | 321 +++++++ datafusion/sqllogictest/test_files/limit.slt | 2 +- .../sqllogictest/test_files/pipe_operator.slt | 91 ++ .../sqllogictest/test_files/push_down_filter.slt | 2 +- .../test_files/spark/datetime/make_dt_interval.slt | 125 ++- .../test_files/spark/datetime/make_interval.slt | 112 +++ .../test_files/spark/map/map_from_arrays.slt | 136 +++ .../test_files/spark/map/map_from_entries.slt | 164 ++++ .../test_files/string/string_literal.slt | 20 + .../sqllogictest/test_files/string/string_view.slt | 2 +- datafusion/sqllogictest/test_files/timestamps.slt | 33 + datafusion/sqllogictest/test_files/topk.slt | 16 +- datafusion/sqllogictest/test_files/union.slt | 8 +- datafusion/sqllogictest/test_files/window.slt | 89 ++ datafusion/substrait/README.md | 7 +- .../src/logical_plan/consumer/expr/literal.rs | 42 +- .../src/logical_plan/consumer/expr/mod.rs | 2 +- .../src/logical_plan/consumer/expr/subquery.rs | 5 +- .../src/logical_plan/consumer/rel/join_rel.rs | 4 +- .../src/logical_plan/consumer/rel/read_rel.rs | 2 +- .../src/logical_plan/producer/expr/literal.rs | 45 +- .../src/logical_plan/producer/rel/aggregate_rel.rs | 2 +- .../src/logical_plan/producer/rel/join.rs | 5 +- .../substrait/src/logical_plan/producer/types.rs | 4 +- datafusion/substrait/src/physical_plan/consumer.rs | 2 +- .../substrait/tests/cases/consumer_integration.rs | 42 +- .../tests/cases/roundtrip_logical_plan.rs | 90 +- .../substrait/tests/cases/substrait_validations.rs | 2 +- datafusion/substrait/tests/utils.rs | 2 +- datafusion/wasmtest/Cargo.toml | 2 +- datafusion/wasmtest/README.md | 16 +- dev/release/README.md | 43 +- dev/release/publish_homebrew.sh | 92 -- dev/rust_lint.sh | 8 + docs/scripts/update_committer_list.py | 266 ++++++ docs/source/_static/theme_overrides.css | 26 + docs/source/contributor-guide/governance.md | 82 +- .../gsoc_application_guidelines_2025.md} | 2 +- .../gsoc_project_ideas_2025.md} | 2 +- docs/source/contributor-guide/gsoc/index.rst | 36 + docs/source/index.rst | 7 +- docs/source/library-user-guide/query-optimizer.md | 2 +- docs/source/library-user-guide/upgrading.md | 51 +- docs/source/user-guide/configs.md | 1 + docs/source/user-guide/expressions.md | 4 +- docs/source/user-guide/introduction.md | 48 +- docs/source/user-guide/sql/scalar_functions.md | 396 ++++++++ docs/source/user-guide/sql/select.md | 96 ++ rust-toolchain.toml | 2 +- test-utils/src/array_gen/random_data.rs | 14 +- testing | 2 +- typos.toml | 3 +- 376 files changed, 11779 insertions(+), 3266 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
