This is an automated email from the ASF dual-hosted git repository. blaginin pushed a commit to branch bash-push in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 667aa8c67bd67f032d44d6265da6f7d7bb9c9ff5 Merge: 6f330d43c2 3de195a6d4 Author: blaginin <[email protected]> AuthorDate: Sat Nov 8 14:03:23 2025 +0000 Merge branch 'main' into sandbox-main # Conflicts: # README.md .github/workflows/audit.yml | 2 +- .github/workflows/extended.yml | 5 +- .github/workflows/rust.yml | 17 +- Cargo.lock | 730 +++-- Cargo.toml | 37 +- benchmarks/Cargo.toml | 3 + benchmarks/src/tpch/run.rs | 13 + ci/scripts/rust_clippy.sh | 2 +- datafusion-cli/Cargo.toml | 2 +- datafusion-cli/src/functions.rs | 4 +- datafusion-cli/src/main.rs | 18 +- datafusion-cli/src/object_storage/instrumented.rs | 8 +- datafusion-cli/src/print_options.rs | 2 +- datafusion-examples/Cargo.toml | 17 +- datafusion-examples/README.md | 25 +- .../examples/advanced_parquet_index.rs | 3 +- .../date_time.rs} | 16 +- .../{ => builtin_functions}/function_factory.rs | 3 +- .../examples/builtin_functions/main.rs | 94 + .../examples/{ => builtin_functions}/regexp.rs | 5 +- datafusion-examples/examples/csv_json_opener.rs | 10 +- .../examples/default_column_values.rs | 2 +- .../examples/external_dependency/query-aws-s3.rs | 1 - .../flight/{flight_client.rs => client.rs} | 8 +- datafusion-examples/examples/flight/main.rs | 94 + .../flight/{flight_server.rs => server.rs} | 8 +- .../flight/{flight_sql_server.rs => sql_server.rs} | 4 +- datafusion-examples/examples/json_shredding.rs | 2 +- datafusion-examples/examples/parquet_encrypted.rs | 8 +- .../examples/parquet_encrypted_with_kms.rs | 4 +- datafusion-examples/examples/parquet_index.rs | 3 +- datafusion-examples/examples/sql_query.rs | 1 - datafusion-examples/examples/thread_pools.rs | 2 +- .../examples/{ => udf}/advanced_udaf.rs | 5 +- .../examples/{ => udf}/advanced_udf.rs | 55 +- .../examples/{ => udf}/advanced_udwf.rs | 5 +- .../examples/{ => udf}/async_udf.rs | 5 +- datafusion-examples/examples/udf/main.rs | 133 + .../examples/{ => udf}/simple_udaf.rs | 5 +- .../examples/{ => udf}/simple_udf.rs | 3 +- .../examples/{ => udf}/simple_udtf.rs | 3 +- .../examples/{ => udf}/simple_udwf.rs | 3 +- datafusion/catalog-listing/Cargo.toml | 3 + datafusion/catalog-listing/src/config.rs | 5 +- datafusion/catalog-listing/src/options.rs | 36 +- datafusion/catalog-listing/src/table.rs | 2 +- datafusion/catalog/Cargo.toml | 3 + datafusion/common-runtime/Cargo.toml | 3 + datafusion/common/Cargo.toml | 7 +- datafusion/common/src/cast.rs | 14 +- datafusion/common/src/config.rs | 107 +- datafusion/common/src/datatype.rs | 173 ++ datafusion/common/src/dfschema.rs | 42 +- datafusion/common/src/diagnostic.rs | 7 +- datafusion/common/src/encryption.rs | 32 +- datafusion/common/src/error.rs | 17 +- .../common/src/file_options/parquet_writer.rs | 22 +- datafusion/common/src/hash_utils.rs | 280 +- datafusion/common/src/lib.rs | 5 + datafusion/common/src/metadata.rs | 370 +++ datafusion/common/src/nested_struct.rs | 9 +- datafusion/common/src/param_value.rs | 58 +- datafusion/common/src/pyarrow.rs | 28 +- datafusion/common/src/scalar/mod.rs | 176 +- datafusion/common/src/scalar/struct_builder.rs | 13 +- datafusion/common/src/stats.rs | 46 +- datafusion/common/src/table_reference.rs | 59 +- datafusion/common/src/test_util.rs | 2 +- datafusion/common/src/tree_node.rs | 13 +- datafusion/common/src/types/builtin.rs | 35 + datafusion/common/src/types/logical.rs | 12 +- datafusion/common/src/types/native.rs | 32 +- datafusion/common/src/utils/memory.rs | 8 +- datafusion/common/src/utils/mod.rs | 46 +- datafusion/common/src/utils/proxy.rs | 16 +- datafusion/core/Cargo.toml | 7 + datafusion/core/benches/aggregate_query_sql.rs | 4 +- datafusion/core/benches/csv_load.rs | 4 +- datafusion/core/benches/dataframe.rs | 3 +- datafusion/core/benches/distinct_query_sql.rs | 6 +- datafusion/core/benches/filter_query_sql.rs | 3 +- datafusion/core/benches/map_query_sql.rs | 3 +- datafusion/core/benches/parquet_query_sql.rs | 5 +- datafusion/core/benches/spm.rs | 3 +- datafusion/core/benches/sql_planner.rs | 204 +- datafusion/core/benches/sql_planner_extended.rs | 233 ++ datafusion/core/benches/struct_query_sql.rs | 3 +- datafusion/core/benches/topk_aggregate.rs | 8 +- datafusion/core/benches/window_query_sql.rs | 4 +- datafusion/core/src/dataframe/mod.rs | 314 ++- datafusion/core/src/dataframe/parquet.rs | 21 +- datafusion/core/src/datasource/file_format/mod.rs | 2 +- .../core/src/datasource/file_format/options.rs | 14 + .../core/src/datasource/file_format/parquet.rs | 90 +- datafusion/core/src/datasource/mod.rs | 1 + .../core/src/datasource/physical_plan/avro.rs | 6 +- .../core/src/datasource/physical_plan/csv.rs | 6 +- .../core/src/datasource/physical_plan/json.rs | 4 +- .../core/src/datasource/physical_plan/parquet.rs | 55 +- datafusion/core/src/execution/context/csv.rs | 11 +- datafusion/core/src/execution/context/mod.rs | 151 +- datafusion/core/src/execution/session_state.rs | 26 +- .../core/src/execution/session_state_defaults.rs | 8 +- datafusion/core/src/lib.rs | 63 +- datafusion/core/src/physical_planner.rs | 26 +- datafusion/core/src/test_util/parquet.rs | 3 +- .../core/tests/dataframe/dataframe_functions.rs | 47 +- datafusion/core/tests/dataframe/mod.rs | 336 ++- datafusion/core/tests/datasource/mod.rs | 1 + .../core/tests/datasource/object_store_access.rs | 956 +++++++ datafusion/core/tests/expr_api/mod.rs | 20 + datafusion/core/tests/expr_api/simplification.rs | 2 +- .../aggregation_fuzzer/context_generator.rs | 1 - .../aggregation_fuzzer/data_generator.rs | 2 - .../tests/fuzz_cases/aggregation_fuzzer/fuzzer.rs | 1 - .../fuzz_cases/aggregation_fuzzer/query_builder.rs | 4 +- .../tests/fuzz_cases/equivalence/projection.rs | 4 +- datafusion/core/tests/optimizer/mod.rs | 2 +- datafusion/core/tests/parquet/encryption.rs | 6 +- .../core/tests/parquet/external_access_plan.rs | 23 +- datafusion/core/tests/parquet/filter_pushdown.rs | 44 +- datafusion/core/tests/parquet/mod.rs | 75 +- datafusion/core/tests/parquet/row_group_pruning.rs | 102 +- datafusion/core/tests/parquet/schema_coercion.rs | 2 +- .../physical_optimizer/enforce_distribution.rs | 2865 ++++++++++---------- .../tests/physical_optimizer/enforce_sorting.rs | 163 +- .../enforce_sorting_monotonicity.rs | 504 ++-- .../physical_optimizer/filter_pushdown/mod.rs | 463 +++- .../physical_optimizer/filter_pushdown/util.rs | 11 +- .../physical_optimizer/projection_pushdown.rs | 6 +- .../tests/physical_optimizer/sanity_checker.rs | 4 +- datafusion/core/tests/sql/explain_analyze.rs | 190 +- datafusion/core/tests/sql/mod.rs | 1 - datafusion/core/tests/sql/select.rs | 62 +- .../tests/user_defined/user_defined_aggregates.rs | 8 +- .../core/tests/user_defined/user_defined_plan.rs | 1 - .../user_defined/user_defined_scalar_functions.rs | 4 +- datafusion/datasource-arrow/Cargo.toml | 3 + datafusion/datasource-arrow/src/source.rs | 4 +- datafusion/datasource-avro/Cargo.toml | 5 +- .../datasource-avro/src/avro_to_arrow/reader.rs | 8 +- datafusion/datasource-avro/src/source.rs | 7 +- datafusion/datasource-csv/Cargo.toml | 3 + datafusion/datasource-csv/src/source.rs | 6 +- datafusion/datasource-json/Cargo.toml | 3 + datafusion/datasource-json/src/source.rs | 3 +- datafusion/datasource-parquet/Cargo.toml | 3 + datafusion/datasource-parquet/src/file_format.rs | 101 +- datafusion/datasource-parquet/src/metadata.rs | 22 +- datafusion/datasource-parquet/src/metrics.rs | 80 +- datafusion/datasource-parquet/src/opener.rs | 43 +- datafusion/datasource-parquet/src/page_filter.rs | 9 +- datafusion/datasource-parquet/src/reader.rs | 5 +- .../datasource-parquet/src/row_group_filter.rs | 27 +- datafusion/datasource-parquet/src/source.rs | 37 +- datafusion/datasource/Cargo.toml | 3 + datafusion/datasource/src/file.rs | 4 +- datafusion/datasource/src/file_scan_config.rs | 368 ++- datafusion/datasource/src/file_stream.rs | 8 +- datafusion/datasource/src/memory.rs | 28 +- datafusion/datasource/src/mod.rs | 1 - datafusion/datasource/src/source.rs | 32 +- datafusion/datasource/src/table_schema.rs | 26 + datafusion/datasource/src/test_util.rs | 5 +- datafusion/datasource/src/url.rs | 147 +- datafusion/datasource/src/write/mod.rs | 12 +- datafusion/doc/Cargo.toml | 3 + datafusion/execution/Cargo.toml | 3 + datafusion/execution/src/config.rs | 13 +- datafusion/execution/src/disk_manager.rs | 273 +- datafusion/execution/src/memory_pool/pool.rs | 4 +- datafusion/execution/src/parquet_encryption.rs | 4 +- datafusion/execution/src/runtime_env.rs | 6 +- datafusion/expr-common/Cargo.toml | 3 + datafusion/expr-common/src/interval_arithmetic.rs | 129 +- datafusion/expr-common/src/signature.rs | 812 +++++- datafusion/expr/Cargo.toml | 3 + datafusion/expr/src/arguments.rs | 285 ++ datafusion/expr/src/expr.rs | 399 +-- datafusion/expr/src/expr_fn.rs | 6 +- datafusion/expr/src/expr_rewriter/mod.rs | 2 +- datafusion/expr/src/expr_schema.rs | 120 +- datafusion/expr/src/lib.rs | 1 + datafusion/expr/src/literal.rs | 3 +- datafusion/expr/src/logical_plan/builder.rs | 24 +- datafusion/expr/src/logical_plan/display.rs | 12 +- datafusion/expr/src/logical_plan/extension.rs | 28 +- datafusion/expr/src/logical_plan/plan.rs | 170 +- datafusion/expr/src/logical_plan/statement.rs | 21 +- datafusion/expr/src/select_expr.rs | 6 +- datafusion/expr/src/udaf.rs | 74 +- datafusion/expr/src/udf.rs | 9 +- datafusion/expr/src/udwf.rs | 2 +- datafusion/expr/src/utils.rs | 72 +- datafusion/expr/src/window_frame.rs | 1 - datafusion/ffi/Cargo.toml | 3 + datafusion/ffi/src/udaf/accumulator_args.rs | 9 + datafusion/ffi/src/udaf/mod.rs | 2 + datafusion/ffi/src/udtf.rs | 3 +- datafusion/functions-aggregate-common/Cargo.toml | 3 + .../functions-aggregate-common/src/accumulator.rs | 6 +- .../src/aggregate/groups_accumulator.rs | 5 +- .../src/aggregate/groups_accumulator/accumulate.rs | 1 - .../src/aggregate/groups_accumulator/prim_op.rs | 1 - datafusion/functions-aggregate-common/src/stats.rs | 2 +- .../functions-aggregate-common/src/tdigest.rs | 1 - datafusion/functions-aggregate-common/src/utils.rs | 10 +- datafusion/functions-aggregate/Cargo.toml | 3 + .../functions-aggregate/benches/array_agg.rs | 3 +- datafusion/functions-aggregate/benches/count.rs | 11 +- .../functions-aggregate/benches/min_max_bytes.rs | 1 + datafusion/functions-aggregate/benches/sum.rs | 6 +- .../functions-aggregate/src/approx_distinct.rs | 2 +- .../functions-aggregate/src/approx_median.rs | 2 +- .../src/approx_percentile_cont.rs | 15 +- .../src/approx_percentile_cont_with_weight.rs | 25 +- datafusion/functions-aggregate/src/array_agg.rs | 18 +- datafusion/functions-aggregate/src/average.rs | 27 +- datafusion/functions-aggregate/src/correlation.rs | 4 +- datafusion/functions-aggregate/src/count.rs | 10 +- datafusion/functions-aggregate/src/first_last.rs | 124 +- datafusion/functions-aggregate/src/median.rs | 5 +- datafusion/functions-aggregate/src/nth_value.rs | 4 +- .../functions-aggregate/src/percentile_cont.rs | 7 +- datafusion/functions-aggregate/src/stddev.rs | 9 +- datafusion/functions-aggregate/src/string_agg.rs | 15 +- datafusion/functions-nested/Cargo.toml | 7 + .../functions-nested/benches/array_expression.rs | 3 +- .../functions-nested/benches/array_reverse.rs | 78 + datafusion/functions-nested/benches/map.rs | 5 +- datafusion/functions-nested/src/expr_ext.rs | 6 +- datafusion/functions-nested/src/extract.rs | 598 +++- datafusion/functions-nested/src/flatten.rs | 39 +- datafusion/functions-nested/src/lib.rs | 1 - datafusion/functions-nested/src/macros.rs | 25 +- datafusion/functions-nested/src/planner.rs | 5 +- datafusion/functions-nested/src/range.rs | 826 +++--- datafusion/functions-nested/src/replace.rs | 3 + datafusion/functions-nested/src/reverse.rs | 284 +- datafusion/functions-table/Cargo.toml | 3 + datafusion/functions-table/src/generate_series.rs | 10 +- datafusion/functions-window-common/Cargo.toml | 3 + datafusion/functions-window-common/src/expr.rs | 1 - datafusion/functions-window-common/src/field.rs | 1 - .../functions-window-common/src/partition.rs | 1 - datafusion/functions-window/Cargo.toml | 3 + datafusion/functions-window/src/lead_lag.rs | 8 +- datafusion/functions-window/src/lib.rs | 1 - datafusion/functions/Cargo.toml | 4 + datafusion/functions/benches/ascii.rs | 3 +- datafusion/functions/benches/character_length.rs | 3 +- datafusion/functions/benches/chr.rs | 3 +- datafusion/functions/benches/concat.rs | 3 +- datafusion/functions/benches/cot.rs | 3 +- datafusion/functions/benches/date_bin.rs | 3 +- datafusion/functions/benches/date_trunc.rs | 3 +- datafusion/functions/benches/encoding.rs | 3 +- datafusion/functions/benches/find_in_set.rs | 3 +- datafusion/functions/benches/gcd.rs | 3 +- datafusion/functions/benches/initcap.rs | 3 +- datafusion/functions/benches/isnan.rs | 3 +- datafusion/functions/benches/iszero.rs | 3 +- datafusion/functions/benches/lower.rs | 3 +- datafusion/functions/benches/ltrim.rs | 6 +- datafusion/functions/benches/make_date.rs | 3 +- datafusion/functions/benches/nullif.rs | 3 +- datafusion/functions/benches/pad.rs | 37 +- datafusion/functions/benches/random.rs | 3 +- datafusion/functions/benches/regx.rs | 3 +- datafusion/functions/benches/repeat.rs | 3 +- datafusion/functions/benches/reverse.rs | 3 +- datafusion/functions/benches/signum.rs | 3 +- datafusion/functions/benches/strpos.rs | 3 +- datafusion/functions/benches/substr.rs | 3 +- datafusion/functions/benches/substr_index.rs | 3 +- datafusion/functions/benches/to_char.rs | 3 +- datafusion/functions/benches/to_hex.rs | 3 +- datafusion/functions/benches/to_timestamp.rs | 3 +- datafusion/functions/benches/trunc.rs | 3 +- datafusion/functions/benches/upper.rs | 3 +- datafusion/functions/benches/uuid.rs | 3 +- datafusion/functions/src/core/arrow_cast.rs | 28 +- datafusion/functions/src/core/expr_ext.rs | 3 +- datafusion/functions/src/core/nullif.rs | 1 - datafusion/functions/src/core/nvl2.rs | 85 +- datafusion/functions/src/datetime/common.rs | 2 - datafusion/functions/src/datetime/current_date.rs | 9 +- datafusion/functions/src/datetime/current_time.rs | 15 +- datafusion/functions/src/datetime/date_bin.rs | 6 +- datafusion/functions/src/datetime/date_trunc.rs | 244 +- datafusion/functions/src/datetime/now.rs | 60 +- datafusion/functions/src/datetime/to_local_time.rs | 10 +- datafusion/functions/src/datetime/to_timestamp.rs | 2 +- datafusion/functions/src/encoding/inner.rs | 51 +- datafusion/functions/src/math/abs.rs | 36 +- datafusion/functions/src/math/log.rs | 184 +- datafusion/functions/src/unicode/substr.rs | 8 +- datafusion/functions/src/utils.rs | 43 +- datafusion/macros/Cargo.toml | 5 +- datafusion/macros/src/user_doc.rs | 1 - datafusion/optimizer/Cargo.toml | 3 + .../optimizer/benches/projection_unnecessary.rs | 3 +- .../src/analyzer/resolve_grouping_function.rs | 6 +- datafusion/optimizer/src/analyzer/type_coercion.rs | 10 +- .../src/decorrelate_predicate_subquery.rs | 44 +- datafusion/optimizer/src/lib.rs | 3 + datafusion/optimizer/src/push_down_filter.rs | 4 - datafusion/optimizer/src/push_down_limit.rs | 1 - .../src/simplify_expressions/expr_simplifier.rs | 206 +- .../src/simplify_expressions/guarantees.rs | 10 +- .../src/simplify_expressions/unwrap_cast.rs | 1 - datafusion/physical-expr-common/Cargo.toml | 3 + datafusion/physical-expr-common/src/binary_map.rs | 2 +- .../physical-expr-common/src/binary_view_map.rs | 5 +- .../physical-expr-common/src/physical_expr.rs | 6 +- datafusion/physical-expr/Cargo.toml | 3 + datafusion/physical-expr/benches/binary_op.rs | 3 +- datafusion/physical-expr/benches/case_when.rs | 3 +- datafusion/physical-expr/benches/in_list.rs | 3 +- datafusion/physical-expr/benches/is_null.rs | 3 +- datafusion/physical-expr/src/aggregate.rs | 17 +- .../src/equivalence/properties/mod.rs | 9 +- .../src/expressions/binary/kernels.rs | 14 +- datafusion/physical-expr/src/expressions/case.rs | 1332 +++++++-- datafusion/physical-expr/src/expressions/cast.rs | 4 +- datafusion/physical-expr/src/expressions/column.rs | 6 +- .../src/expressions/dynamic_filters.rs | 4 +- .../physical-expr/src/expressions/literal.rs | 2 +- .../physical-expr/src/intervals/cp_solver.rs | 12 +- datafusion/physical-expr/src/physical_expr.rs | 16 +- datafusion/physical-expr/src/planner.rs | 5 +- datafusion/physical-expr/src/projection.rs | 192 +- datafusion/physical-optimizer/Cargo.toml | 3 + .../src/combine_partial_final_agg.rs | 1 - .../physical-optimizer/src/enforce_distribution.rs | 1 - .../physical-optimizer/src/enforce_sorting/mod.rs | 2 +- .../physical-optimizer/src/join_selection.rs | 1 - datafusion/physical-optimizer/src/utils.rs | 4 +- datafusion/physical-plan/Cargo.toml | 3 + .../physical-plan/benches/aggregate_vectorized.rs | 3 +- .../src/aggregates/group_values/metrics.rs | 214 ++ .../src/aggregates/group_values/mod.rs | 4 +- .../aggregates/group_values/multi_group_by/mod.rs | 11 - .../group_values/single_group_by/primitive.rs | 1 - datafusion/physical-plan/src/aggregates/mod.rs | 88 + .../physical-plan/src/aggregates/order/partial.rs | 2 +- .../physical-plan/src/aggregates/row_hash.rs | 60 +- .../physical-plan/src/aggregates/topk_stream.rs | 33 +- datafusion/physical-plan/src/coalesce/mod.rs | 444 +-- datafusion/physical-plan/src/coalesce_batches.rs | 132 +- datafusion/physical-plan/src/execution_plan.rs | 23 +- datafusion/physical-plan/src/filter.rs | 36 +- datafusion/physical-plan/src/joins/cross_join.rs | 2 +- .../physical-plan/src/joins/hash_join/exec.rs | 26 +- .../physical-plan/src/joins/hash_join/stream.rs | 1 - .../physical-plan/src/joins/nested_loop_join.rs | 78 +- .../src/joins/piecewise_merge_join/exec.rs | 1 - .../src/joins/sort_merge_join/stream.rs | 6 +- .../physical-plan/src/joins/stream_join_utils.rs | 3 +- .../physical-plan/src/joins/symmetric_hash_join.rs | 1 - datafusion/physical-plan/src/joins/utils.rs | 1 - datafusion/physical-plan/src/memory.rs | 30 +- datafusion/physical-plan/src/metrics/baseline.rs | 20 + datafusion/physical-plan/src/metrics/builder.rs | 62 +- datafusion/physical-plan/src/metrics/custom.rs | 3 +- datafusion/physical-plan/src/metrics/mod.rs | 33 +- datafusion/physical-plan/src/metrics/value.rs | 315 ++- datafusion/physical-plan/src/projection.rs | 36 +- datafusion/physical-plan/src/recursive_query.rs | 1 - .../src/repartition/distributor_channels.rs | 2 +- datafusion/physical-plan/src/repartition/mod.rs | 912 +++++-- datafusion/physical-plan/src/sorts/merge.rs | 1 - .../physical-plan/src/sorts/multi_level_merge.rs | 1 - datafusion/physical-plan/src/sorts/partial_sort.rs | 4 +- datafusion/physical-plan/src/sorts/sort.rs | 1 - .../src/spill/in_progress_spill_file.rs | 6 + datafusion/physical-plan/src/spill/mod.rs | 5 + .../physical-plan/src/spill/spill_manager.rs | 5 + datafusion/physical-plan/src/spill/spill_pool.rs | 1425 ++++++++++ datafusion/physical-plan/src/stream.rs | 15 +- datafusion/physical-plan/src/test/exec.rs | 1 - datafusion/physical-plan/src/tree_node.rs | 6 +- datafusion/physical-plan/src/union.rs | 16 +- datafusion/physical-plan/src/unnest.rs | 4 - .../src/windows/bounded_window_agg_exec.rs | 4 +- datafusion/proto-common/gen/Cargo.toml | 3 + datafusion/proto-common/src/from_proto/mod.rs | 13 +- datafusion/proto-common/src/lib.rs | 43 +- datafusion/proto-common/src/to_proto/mod.rs | 13 +- datafusion/proto/gen/Cargo.toml | 3 + datafusion/proto/proto/datafusion.proto | 7 + datafusion/proto/src/bytes/mod.rs | 2 +- datafusion/proto/src/generated/pbjson.rs | 53 + datafusion/proto/src/generated/prost.rs | 13 + datafusion/proto/src/lib.rs | 14 +- datafusion/proto/src/logical_plan/from_proto.rs | 28 +- datafusion/proto/src/logical_plan/mod.rs | 46 +- datafusion/proto/src/logical_plan/to_proto.rs | 26 +- datafusion/proto/src/physical_plan/from_proto.rs | 58 +- datafusion/proto/src/physical_plan/to_proto.rs | 5 +- .../proto/tests/cases/roundtrip_logical_plan.rs | 75 + .../proto/tests/cases/roundtrip_physical_plan.rs | 4 +- datafusion/proto/tests/cases/serialize.rs | 18 +- datafusion/pruning/Cargo.toml | 3 + datafusion/pruning/src/pruning_predicate.rs | 3 +- datafusion/session/Cargo.toml | 3 + datafusion/session/src/session.rs | 9 +- datafusion/spark/Cargo.toml | 3 + datafusion/spark/benches/char.rs | 3 +- datafusion/spark/src/function/array/shuffle.rs | 102 +- datafusion/spark/src/function/array/spark_array.rs | 31 +- .../spark/src/function/bitmap/bitmap_count.rs | 65 +- datafusion/spark/src/function/bitwise/bit_count.rs | 55 +- datafusion/spark/src/function/bitwise/bit_shift.rs | 3 - datafusion/spark/src/function/url/parse_url.rs | 2 - datafusion/spark/src/lib.rs | 2 +- datafusion/sql/Cargo.toml | 3 + datafusion/sql/src/expr/function.rs | 157 +- datafusion/sql/src/expr/mod.rs | 56 +- datafusion/sql/src/expr/value.rs | 10 +- datafusion/sql/src/parser.rs | 27 +- datafusion/sql/src/planner.rs | 70 +- datafusion/sql/src/resolve.rs | 12 +- datafusion/sql/src/statement.rs | 63 +- datafusion/sql/src/unparser/expr.rs | 7 +- datafusion/sql/src/unparser/plan.rs | 8 +- datafusion/sql/src/unparser/rewrite.rs | 1 - datafusion/sql/src/utils.rs | 1 - datafusion/sql/tests/cases/diagnostic.rs | 10 +- datafusion/sql/tests/cases/params.rs | 193 +- datafusion/sql/tests/common/mod.rs | 8 + datafusion/sql/tests/sql_integration.rs | 36 +- datafusion/sqllogictest/Cargo.toml | 12 +- .../src/engines/datafusion_engine/normalize.rs | 1 - .../src/engines/postgres_engine/mod.rs | 4 +- .../test_files/aggregate_repartition.slt | 136 + datafusion/sqllogictest/test_files/array.slt | 140 +- .../sqllogictest/test_files/arrow_typeof.slt | 26 +- datafusion/sqllogictest/test_files/case.slt | 90 +- datafusion/sqllogictest/test_files/coalesce.slt | 6 +- .../sqllogictest/test_files/count_star_rule.slt | 2 +- datafusion/sqllogictest/test_files/cte.slt | 2 +- .../test_files/current_date_timezone.slt | 84 +- .../test_files/current_time_timezone.slt | 4 +- datafusion/sqllogictest/test_files/dates.slt | 15 +- datafusion/sqllogictest/test_files/ddl.slt | 2 +- datafusion/sqllogictest/test_files/decimal.slt | 40 +- datafusion/sqllogictest/test_files/describe.slt | 28 +- datafusion/sqllogictest/test_files/dictionary.slt | 4 +- datafusion/sqllogictest/test_files/encoding.slt | 9 +- datafusion/sqllogictest/test_files/errors.slt | 2 +- .../sqllogictest/test_files/expr/date_part.slt | 4 +- datafusion/sqllogictest/test_files/float16.slt | 20 +- datafusion/sqllogictest/test_files/group_by.slt | 8 +- .../sqllogictest/test_files/information_schema.slt | 66 +- .../test_files/information_schema_columns.slt | 2 +- datafusion/sqllogictest/test_files/insert.slt | 6 +- .../sqllogictest/test_files/insert_to_external.slt | 4 +- datafusion/sqllogictest/test_files/interval.slt | 6 +- datafusion/sqllogictest/test_files/join_lists.slt | 1 - datafusion/sqllogictest/test_files/joins.slt | 32 +- datafusion/sqllogictest/test_files/map.slt | 46 +- datafusion/sqllogictest/test_files/math.slt | 120 +- datafusion/sqllogictest/test_files/metadata.slt | 2 +- .../sqllogictest/test_files/named_arguments.slt | 271 ++ datafusion/sqllogictest/test_files/nullif.slt | 2 +- datafusion/sqllogictest/test_files/order.slt | 32 +- datafusion/sqllogictest/test_files/parquet.slt | 35 +- .../test_files/parquet_filter_pushdown.slt | 23 +- datafusion/sqllogictest/test_files/prepare.slt | 32 + datafusion/sqllogictest/test_files/pwmj.slt | 2 +- datafusion/sqllogictest/test_files/qualify.slt | 8 +- datafusion/sqllogictest/test_files/scalar.slt | 16 + datafusion/sqllogictest/test_files/select.slt | 4 +- .../sqllogictest/test_files/simplify_expr.slt | 11 + .../sqllogictest/test_files/spark/array/array.slt | 15 + .../test_files/spark/array/shuffle.slt | 48 +- .../test_files/spark/bitmap/bitmap_count.slt | 32 + .../test_files/spark/bitwise/bit_count.slt | 12 +- datafusion/sqllogictest/test_files/struct.slt | 46 +- .../sqllogictest/test_files/subquery_sort.slt | 4 +- .../sqllogictest/test_files/table_functions.slt | 17 +- datafusion/sqllogictest/test_files/timestamps.slt | 202 +- .../sqllogictest/test_files/type_coercion.slt | 2 +- datafusion/sqllogictest/test_files/union.slt | 39 +- datafusion/sqllogictest/test_files/unnest.slt | 10 +- datafusion/sqllogictest/test_files/window.slt | 328 +-- .../sqllogictest/test_files/window_limits.slt | 24 +- datafusion/substrait/Cargo.toml | 5 +- datafusion/substrait/src/extensions.rs | 6 + datafusion/substrait/src/lib.rs | 27 +- .../logical_plan/consumer/substrait_consumer.rs | 1 - .../substrait/src/logical_plan/consumer/types.rs | 3 +- .../src/logical_plan/producer/expr/mod.rs | 4 + .../substrait/src/logical_plan/producer/plan.rs | 5 + .../substrait/src/logical_plan/producer/types.rs | 29 +- datafusion/substrait/src/physical_plan/consumer.rs | 4 +- datafusion/substrait/src/physical_plan/producer.rs | 7 +- datafusion/substrait/src/variation_const.rs | 5 + datafusion/wasmtest/Cargo.toml | 5 +- dev/update_config_docs.sh | 60 + docs/rustdoc_trim.py | 2 +- docs/source/_static/theme_overrides.css | 39 +- docs/source/_templates/docs-sidebar.html | 11 - docs/source/_templates/layout.html | 4 - docs/source/conf.py | 6 + docs/source/contributor-guide/howtos.md | 170 +- docs/source/contributor-guide/index.md | 26 +- docs/source/contributor-guide/roadmap.md | 5 +- docs/source/index.rst | 1 + .../library-user-guide/functions/adding-udfs.md | 101 +- docs/source/library-user-guide/upgrading.md | 165 +- docs/source/user-guide/arrow-introduction.md | 255 ++ docs/source/user-guide/configs.md | 65 +- docs/source/user-guide/dataframe.md | 2 + docs/source/user-guide/explain-usage.md | 13 +- docs/source/user-guide/metrics.md | 9 +- docs/source/user-guide/sql/data_types.md | 13 +- docs/source/user-guide/sql/dml.md | 2 +- docs/source/user-guide/sql/scalar_functions.md | 56 +- docs/source/user-guide/sql/subqueries.md | 4 +- rust-toolchain.toml | 2 +- test-utils/Cargo.toml | 3 + 523 files changed, 19825 insertions(+), 7748 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
