This is an automated email from the ASF dual-hosted git repository. blaginin pushed a commit to branch db/dorny-md in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 62f8237f7bf1c9d20fc07f3c3ec976e721780ef7 Merge: 4a9f34d1d2 e5e76366a6 Author: blaginin <[email protected]> AuthorDate: Sat Jan 24 17:20:01 2026 +0000 Merge branch 'upstream-main' into sandbox-main # Conflicts: # .asf.yaml # .github/workflows/rust.yml # README.md .github/actions/setup-builder/action.yaml | 14 + .github/dependabot.yml | 1 + .github/workflows/audit.yml | 6 +- .github/workflows/dependencies.yml | 6 +- .github/workflows/dev.yml | 34 +- .github/workflows/docs.yaml | 12 +- .github/workflows/docs_pr.yaml | 11 +- .github/workflows/extended.yml | 8 +- .github/workflows/labeler.yml | 2 +- .github/workflows/large_files.yml | 2 +- .github/workflows/rust.yml | 140 +- .github/workflows/stale.yml | 2 +- .github/workflows/take.yml | 32 +- Cargo.lock | 2160 +++++------- Cargo.toml | 127 +- NOTICE.txt | 2 +- benchmarks/Cargo.toml | 4 +- benchmarks/README.md | 160 +- benchmarks/bench.sh | 289 +- benchmarks/compare.py | 20 +- benchmarks/compare_tpcds.sh | 58 + .../rust_example.sh => benchmarks/compare_tpch.sh | 52 +- .../queries/clickbench/queries/sorted_data/q0.sql | 3 + benchmarks/src/bin/dfbench.rs | 23 +- benchmarks/src/bin/external_aggr.rs | 33 +- benchmarks/src/bin/imdb.rs | 24 +- benchmarks/src/bin/mem_profile.rs | 49 +- benchmarks/src/bin/tpch.rs | 65 - benchmarks/src/cancellation.rs | 28 +- benchmarks/src/clickbench.rs | 143 +- benchmarks/src/h2o.rs | 30 +- benchmarks/src/hj.rs | 424 ++- benchmarks/src/imdb/convert.rs | 14 +- benchmarks/src/imdb/run.rs | 30 +- benchmarks/src/lib.rs | 2 + benchmarks/src/nlj.rs | 18 +- benchmarks/src/smj.rs | 524 +++ benchmarks/src/sort_tpch.rs | 22 +- .../schema_adapter => benchmarks/src/tpcds}/mod.rs | 3 +- benchmarks/src/tpcds/run.rs | 356 ++ benchmarks/src/tpch/convert.rs | 162 - benchmarks/src/tpch/mod.rs | 3 - benchmarks/src/tpch/run.rs | 38 +- benchmarks/src/util/memory.rs | 2 +- benchmarks/src/util/options.rs | 20 +- benchmarks/src/util/run.rs | 2 +- ci/scripts/check_examples_docs.sh | 64 + ci/scripts/doc_prettier_check.sh | 84 + ci/scripts/license_header.sh | 62 +- ci/scripts/rust_clippy.sh | 59 +- ci/scripts/rust_docs.sh | 1 + ci/scripts/rust_example.sh | 30 +- ci/scripts/rust_fmt.sh | 51 +- ci/scripts/rust_toml_fmt.sh | 55 +- ci/scripts/typos_check.sh | 70 + ci/scripts/{license_header.sh => utils/git.sh} | 11 +- datafusion-cli/Cargo.toml | 5 +- datafusion-cli/examples/cli-session-context.rs | 2 +- datafusion-cli/src/catalog.rs | 50 +- datafusion-cli/src/cli_context.rs | 2 +- datafusion-cli/src/command.rs | 2 +- datafusion-cli/src/exec.rs | 29 +- datafusion-cli/src/functions.rs | 310 +- datafusion-cli/src/helper.rs | 10 +- datafusion-cli/src/highlighter.rs | 4 +- datafusion-cli/src/main.rs | 249 +- datafusion-cli/src/object_storage.rs | 51 +- datafusion-cli/src/object_storage/instrumented.rs | 114 +- datafusion-cli/src/print_format.rs | 66 +- datafusion-cli/src/print_options.rs | 8 +- datafusion-cli/tests/cli_integration.rs | 42 +- ...errides@explain_plan_environment_overrides.snap | 1 - .../tests/snapshots/[email protected] | 2 +- datafusion-cli/tests/snapshots/[email protected] | 2 +- .../tests/snapshots/[email protected] | 2 +- .../tests/snapshots/[email protected] | 2 +- .../tests/snapshots/[email protected] | 2 +- datafusion-cli/tests/snapshots/[email protected] | 2 +- .../tests/snapshots/cli_quick_test@batch_size.snap | 2 +- .../cli_quick_test@default_explain_plan.snap | 1 - .../tests/snapshots/[email protected] | 2 +- .../tests/snapshots/[email protected] | 2 +- datafusion-examples/Cargo.toml | 32 +- datafusion-examples/README.md | 229 +- datafusion-examples/data/README.md | 25 + datafusion-examples/data/csv/cars.csv | 26 + datafusion-examples/data/csv/regex.csv | 12 + .../examples/builtin_functions/date_time.rs | 9 +- .../examples/builtin_functions/function_factory.rs | 8 +- .../examples/builtin_functions/main.rs | 76 +- .../examples/builtin_functions/regexp.rs | 74 +- .../{ => custom_data_source}/csv_json_opener.rs | 79 +- .../{ => custom_data_source}/csv_sql_streaming.rs | 24 +- .../{ => custom_data_source}/custom_datasource.rs | 14 +- .../{ => custom_data_source}/custom_file_casts.rs | 37 +- .../{ => custom_data_source}/custom_file_format.rs | 87 +- .../default_column_values.rs | 185 +- .../file_stream_provider.rs | 44 +- .../examples/custom_data_source/main.rs | 116 + .../examples/{ => data_io}/catalog.rs | 21 +- .../examples/{ => data_io}/json_shredding.rs | 119 +- datafusion-examples/examples/data_io/main.rs | 124 + .../parquet_advanced_index.rs} | 34 +- .../{ => data_io}/parquet_embedded_index.rs | 88 +- .../examples/{ => data_io}/parquet_encrypted.rs | 42 +- .../{ => data_io}/parquet_encrypted_with_kms.rs | 5 +- .../examples/{ => data_io}/parquet_exec_visitor.rs | 64 +- .../examples/{ => data_io}/parquet_index.rs | 24 +- .../query_http_csv.rs} | 8 +- .../examples/{ => data_io}/remote_catalog.rs | 8 +- .../examples/dataframe/cache_factory.rs | 229 ++ .../examples/{ => dataframe}/dataframe.rs | 126 +- .../examples/dataframe/deserialize_to_struct.rs | 366 ++ datafusion-examples/examples/dataframe/main.rs | 93 + .../examples/deserialize_to_struct.rs | 150 - .../examples/execution_monitoring/main.rs | 92 + .../memory_pool_execution_plan.rs | 9 +- .../memory_pool_tracking.rs | 9 +- .../examples/{ => execution_monitoring}/tracing.rs | 54 +- .../{dataframe-to-s3.rs => dataframe_to_s3.rs} | 22 +- .../examples/external_dependency/main.rs | 84 + .../{query-aws-s3.rs => query_aws_s3.rs} | 13 +- .../ffi/ffi_example_table_provider/src/lib.rs | 7 +- .../examples/ffi/ffi_module_interface/Cargo.toml | 2 +- .../examples/ffi/ffi_module_interface/src/lib.rs | 7 +- .../examples/ffi/ffi_module_loader/Cargo.toml | 2 +- .../examples/ffi/ffi_module_loader/src/main.rs | 21 +- datafusion-examples/examples/flight/client.rs | 19 +- datafusion-examples/examples/flight/main.rs | 83 +- datafusion-examples/examples/flight/server.rs | 41 +- datafusion-examples/examples/flight/sql_server.rs | 31 +- .../{ => proto}/composed_extension_codec.rs | 18 +- datafusion-examples/examples/proto/main.rs | 82 + .../examples/{ => query_planning}/analyzer_rule.rs | 9 +- .../examples/{ => query_planning}/expr_api.rs | 49 +- .../examples/query_planning/main.rs | 108 + .../{ => query_planning}/optimizer_rule.rs | 7 +- .../{ => query_planning}/parse_sql_expr.rs | 80 +- .../examples/{ => query_planning}/plan_to_sql.rs | 102 +- .../examples/{ => query_planning}/planner_api.rs | 28 +- .../examples/{ => query_planning}/pruning.rs | 11 +- .../examples/{ => query_planning}/thread_pools.rs | 21 +- .../examples/relation_planner/main.rs | 121 + .../examples/relation_planner/match_recognize.rs | 408 +++ .../examples/relation_planner/pivot_unpivot.rs | 571 +++ .../examples/relation_planner/table_sample.rs | 820 +++++ datafusion-examples/examples/sql_dialect.rs | 134 - .../{sql_analysis.rs => sql_ops/analysis.rs} | 275 +- .../examples/sql_ops/custom_sql_parser.rs | 420 +++ .../{sql_frontend.rs => sql_ops/frontend.rs} | 6 +- datafusion-examples/examples/sql_ops/main.rs | 94 + .../examples/{sql_query.rs => sql_ops/query.rs} | 77 +- datafusion-examples/examples/udf/advanced_udaf.rs | 21 +- datafusion-examples/examples/udf/advanced_udf.rs | 8 +- datafusion-examples/examples/udf/advanced_udwf.rs | 20 +- datafusion-examples/examples/udf/async_udf.rs | 32 +- datafusion-examples/examples/udf/main.rs | 104 +- datafusion-examples/examples/udf/simple_udaf.rs | 2 + datafusion-examples/examples/udf/simple_udf.rs | 2 + datafusion-examples/examples/udf/simple_udtf.rs | 38 +- datafusion-examples/examples/udf/simple_udwf.rs | 50 +- .../mod.rs => datafusion-examples/src/lib.rs | 6 +- datafusion-examples/src/utils/csv_to_parquet.rs | 245 ++ .../src/utils/datasets/cars.rs | 16 +- datafusion-examples/src/utils/datasets/mod.rs | 139 + .../src/utils/datasets/regex.rs | 13 +- .../src/utils}/mod.rs | 7 +- datafusion/catalog-listing/Cargo.toml | 1 - datafusion/catalog-listing/src/config.rs | 88 +- datafusion/catalog-listing/src/helpers.rs | 587 +--- datafusion/catalog-listing/src/mod.rs | 4 +- datafusion/catalog-listing/src/options.rs | 4 +- datafusion/catalog-listing/src/table.rs | 591 +++- datafusion/catalog/src/async.rs | 12 +- datafusion/catalog/src/catalog.rs | 6 +- datafusion/catalog/src/cte_worktable.rs | 38 +- datafusion/catalog/src/default_table_source.rs | 2 +- datafusion/catalog/src/information_schema.rs | 59 +- datafusion/catalog/src/lib.rs | 4 +- datafusion/catalog/src/listing_schema.rs | 23 +- datafusion/catalog/src/memory/schema.rs | 2 +- datafusion/catalog/src/memory/table.rs | 351 +- datafusion/catalog/src/schema.rs | 6 +- datafusion/catalog/src/stream.rs | 2 +- datafusion/catalog/src/streaming.rs | 32 +- datafusion/catalog/src/table.rs | 35 +- datafusion/catalog/src/view.rs | 2 +- datafusion/common-runtime/src/common.rs | 8 +- datafusion/common-runtime/src/lib.rs | 4 +- datafusion/common-runtime/src/trace_utils.rs | 2 +- datafusion/common/Cargo.toml | 14 +- datafusion/common/benches/with_hashes.rs | 209 ++ datafusion/common/src/cast.rs | 8 +- datafusion/common/src/config.rs | 384 +- datafusion/common/src/cse.rs | 8 +- datafusion/common/src/datatype.rs | 114 +- datafusion/common/src/dfschema.rs | 163 +- datafusion/common/src/display/human_readable.rs | 139 + datafusion/common/src/display/mod.rs | 1 + datafusion/common/src/error.rs | 304 +- datafusion/common/src/file_options/csv_writer.rs | 17 + datafusion/common/src/file_options/json_writer.rs | 18 +- datafusion/common/src/file_options/mod.rs | 2 +- .../common/src/file_options/parquet_writer.rs | 74 +- datafusion/common/src/format.rs | 6 +- datafusion/common/src/hash_utils.rs | 909 ++++- datafusion/common/src/instant.rs | 2 +- datafusion/common/src/lib.rs | 38 +- datafusion/common/src/metadata.rs | 18 +- datafusion/common/src/nested_struct.rs | 408 ++- datafusion/common/src/param_value.rs | 2 +- datafusion/common/src/parquet_config.rs | 108 + datafusion/common/src/pruning.rs | 145 +- datafusion/common/src/pyarrow.rs | 169 - datafusion/common/src/rounding.rs | 2 +- datafusion/common/src/scalar/cache.rs | 6 +- datafusion/common/src/scalar/consts.rs | 12 + datafusion/common/src/scalar/mod.rs | 703 ++-- datafusion/common/src/stats.rs | 285 +- datafusion/common/src/test_util.rs | 54 +- datafusion/common/src/tree_node.rs | 54 +- datafusion/common/src/types/builtin.rs | 15 + datafusion/common/src/types/native.rs | 66 +- datafusion/common/src/utils/memory.rs | 200 +- datafusion/common/src/utils/mod.rs | 63 +- datafusion/common/src/utils/proxy.rs | 110 +- datafusion/core/Cargo.toml | 39 +- datafusion/core/benches/aggregate_query_sql.rs | 1 + datafusion/core/benches/csv_load.rs | 1 + datafusion/core/benches/data_utils/mod.rs | 3 +- datafusion/core/benches/dataframe.rs | 1 + datafusion/core/benches/distinct_query_sql.rs | 6 +- datafusion/core/benches/filter_query_sql.rs | 2 +- datafusion/core/benches/map_query_sql.rs | 14 +- datafusion/core/benches/math_query_sql.rs | 1 + datafusion/core/benches/parquet_query_sql.rs | 4 +- datafusion/core/benches/parquet_struct_query.rs | 312 ++ datafusion/core/benches/physical_plan.rs | 3 +- .../core/benches/preserve_file_partitioning.rs | 838 +++++ datafusion/core/benches/push_down_filter.rs | 6 +- .../core/benches/range_and_generate_series.rs | 90 + datafusion/core/benches/reset_plan_states.rs | 198 ++ datafusion/core/benches/scalar.rs | 2 +- datafusion/core/benches/sort.rs | 66 +- datafusion/core/benches/sort_limit_query_sql.rs | 4 +- datafusion/core/benches/spm.rs | 6 +- datafusion/core/benches/sql_planner.rs | 87 +- datafusion/core/benches/sql_planner_extended.rs | 2 +- datafusion/core/benches/sql_query_with_io.rs | 6 +- datafusion/core/benches/struct_query_sql.rs | 2 +- datafusion/core/benches/topk_aggregate.rs | 419 ++- datafusion/core/benches/window_query_sql.rs | 1 + datafusion/core/src/bin/print_functions_docs.rs | 11 +- datafusion/core/src/dataframe/mod.rs | 142 +- datafusion/core/src/dataframe/parquet.rs | 2 +- datafusion/core/src/datasource/dynamic_file.rs | 2 +- datafusion/core/src/datasource/empty.rs | 2 +- .../core/src/datasource/file_format/arrow.rs | 93 + datafusion/core/src/datasource/file_format/avro.rs | 44 +- datafusion/core/src/datasource/file_format/csv.rs | 249 +- datafusion/core/src/datasource/file_format/json.rs | 110 +- datafusion/core/src/datasource/file_format/mod.rs | 25 +- .../core/src/datasource/file_format/options.rs | 8 +- .../core/src/datasource/file_format/parquet.rs | 76 +- datafusion/core/src/datasource/listing/table.rs | 447 +-- .../core/src/datasource/listing_table_factory.rs | 255 +- datafusion/core/src/datasource/memory_test.rs | 11 +- datafusion/core/src/datasource/mod.rs | 195 +- .../core/src/datasource/physical_plan/avro.rs | 123 +- .../core/src/datasource/physical_plan/csv.rs | 337 +- .../core/src/datasource/physical_plan/json.rs | 56 +- .../core/src/datasource/physical_plan/mod.rs | 139 +- .../core/src/datasource/physical_plan/parquet.rs | 479 +-- datafusion/core/src/datasource/view_test.rs | 41 +- datafusion/core/src/execution/context/mod.rs | 300 +- datafusion/core/src/execution/context/parquet.rs | 6 +- datafusion/core/src/execution/session_state.rs | 303 +- .../core/src/execution/session_state_defaults.rs | 6 +- datafusion/core/src/lib.rs | 8 +- datafusion/core/src/physical_planner.rs | 739 +++- datafusion/core/src/prelude.rs | 2 +- datafusion/core/src/test/mod.rs | 34 +- datafusion/core/src/test/object_store.rs | 10 +- datafusion/core/src/test_util/mod.rs | 75 +- datafusion/core/src/test_util/parquet.rs | 45 +- datafusion/core/tests/catalog/memory.rs | 10 +- .../{schema_adapter => catalog_listing}/mod.rs | 2 +- .../tests/catalog_listing/pruned_partition_list.rs | 251 ++ datafusion/core/tests/config_from_env.rs | 60 +- datafusion/core/tests/core_integration.rs | 6 +- .../tests/custom_sources_cases/dml_planning.rs | 395 +++ datafusion/core/tests/custom_sources_cases/mod.rs | 10 +- .../provider_filter_pushdown.rs | 14 +- .../core/tests/custom_sources_cases/statistics.rs | 2 + .../part=123/data.arrow | Bin 0 -> 1608 bytes .../part=456/data.arrow | Bin 0 -> 1608 bytes .../core/tests/data/recursive_cte/closure.csv | 6 + .../core/tests/dataframe/dataframe_functions.rs | 12 +- datafusion/core/tests/dataframe/describe.rs | 2 +- datafusion/core/tests/dataframe/mod.rs | 1211 ++++--- .../core/tests/datasource/object_store_access.rs | 160 +- datafusion/core/tests/execution/coop.rs | 88 +- .../core/tests/execution/datasource_split.rs | 2 +- datafusion/core/tests/execution/logical_plan.rs | 2 +- datafusion/core/tests/execution/mod.rs | 1 + datafusion/core/tests/execution/register_arrow.rs | 90 + datafusion/core/tests/expr_api/mod.rs | 11 +- datafusion/core/tests/expr_api/parse_sql_expr.rs | 2 +- datafusion/core/tests/expr_api/simplification.rs | 174 +- datafusion/core/tests/fifo/mod.rs | 8 +- datafusion/core/tests/fuzz.rs | 5 +- datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs | 37 +- .../aggregation_fuzzer/context_generator.rs | 2 +- .../aggregation_fuzzer/data_generator.rs | 6 +- .../tests/fuzz_cases/aggregation_fuzzer/fuzzer.rs | 4 +- .../fuzz_cases/aggregation_fuzzer/query_builder.rs | 2 +- .../tests/fuzz_cases/distinct_count_string_fuzz.rs | 2 +- .../core/tests/fuzz_cases/equivalence/ordering.rs | 10 +- .../tests/fuzz_cases/equivalence/projection.rs | 8 +- .../tests/fuzz_cases/equivalence/properties.rs | 6 +- .../core/tests/fuzz_cases/equivalence/utils.rs | 8 +- datafusion/core/tests/fuzz_cases/join_fuzz.rs | 782 +++-- datafusion/core/tests/fuzz_cases/limit_fuzz.rs | 2 +- datafusion/core/tests/fuzz_cases/merge_fuzz.rs | 2 +- datafusion/core/tests/fuzz_cases/mod.rs | 6 + datafusion/core/tests/fuzz_cases/pruning.rs | 9 +- .../tests/fuzz_cases/record_batch_generator.rs | 20 +- datafusion/core/tests/fuzz_cases/sort_fuzz.rs | 4 +- .../fuzz_cases/sort_preserving_repartition_fuzz.rs | 11 +- .../core/tests/fuzz_cases/sort_query_fuzz.rs | 32 +- .../spilling_fuzz_in_memory_constrained_env.rs | 42 +- datafusion/core/tests/fuzz_cases/window_fuzz.rs | 53 +- datafusion/core/tests/macro_hygiene/mod.rs | 1 + .../memory_limit/memory_limit_validation/utils.rs | 12 +- datafusion/core/tests/memory_limit/mod.rs | 28 +- .../tests/memory_limit/repartition_mem_limit.rs | 15 +- datafusion/core/tests/optimizer/mod.rs | 32 +- datafusion/core/tests/parquet/custom_reader.rs | 19 +- datafusion/core/tests/parquet/encryption.rs | 5 +- datafusion/core/tests/parquet/expr_adapter.rs | 466 +++ .../core/tests/parquet/external_access_plan.rs | 21 +- datafusion/core/tests/parquet/file_statistics.rs | 28 +- datafusion/core/tests/parquet/filter_pushdown.rs | 39 +- datafusion/core/tests/parquet/mod.rs | 145 +- datafusion/core/tests/parquet/ordering.rs | 103 + datafusion/core/tests/parquet/page_pruning.rs | 21 +- datafusion/core/tests/parquet/row_group_pruning.rs | 339 +- datafusion/core/tests/parquet/schema_adapter.rs | 553 --- datafusion/core/tests/parquet/schema_coercion.rs | 22 +- datafusion/core/tests/parquet/utils.rs | 11 +- .../physical_optimizer/aggregate_statistics.rs | 92 +- .../combine_partial_final_agg.rs | 12 +- .../physical_optimizer/enforce_distribution.rs | 1280 ++++--- .../tests/physical_optimizer/enforce_sorting.rs | 175 +- .../enforce_sorting_monotonicity.rs | 2 +- .../physical_optimizer/filter_pushdown/mod.rs | 2119 ++++++++++-- .../physical_optimizer/filter_pushdown/util.rs | 135 +- .../tests/physical_optimizer/join_selection.rs | 35 +- .../tests/physical_optimizer/limit_pushdown.rs | 179 +- .../limited_distinct_aggregation.rs | 6 +- datafusion/core/tests/physical_optimizer/mod.rs | 6 + .../physical_optimizer/partition_statistics.rs | 484 ++- .../physical_optimizer/projection_pushdown.rs | 337 +- .../core/tests/physical_optimizer/pushdown_sort.rs | 998 ++++++ .../replace_with_order_preserving_variants.rs | 315 +- .../tests/physical_optimizer/sanity_checker.rs | 22 +- .../core/tests/physical_optimizer/test_utils.rs | 324 +- .../tests/physical_optimizer/window_optimize.rs | 4 +- .../schema_adapter_integration_tests.rs | 363 -- datafusion/core/tests/set_comparison.rs | 193 ++ datafusion/core/tests/sql/aggregates/basic.rs | 8 +- datafusion/core/tests/sql/aggregates/dict_nulls.rs | 24 +- datafusion/core/tests/sql/aggregates/mod.rs | 12 +- datafusion/core/tests/sql/explain_analyze.rs | 193 +- datafusion/core/tests/sql/joins.rs | 85 +- datafusion/core/tests/sql/mod.rs | 26 +- datafusion/core/tests/sql/path_partition.rs | 38 +- datafusion/core/tests/sql/runtime_config.rs | 92 + datafusion/core/tests/sql/select.rs | 52 +- datafusion/core/tests/sql/unparser.rs | 466 +++ datafusion/core/tests/tpc-ds/30.sql | 4 +- datafusion/core/tests/tpcds_planning.rs | 5 +- datafusion/core/tests/tracing/asserting_tracer.rs | 2 +- .../core/tests/tracing/traceable_object_store.rs | 4 +- datafusion/core/tests/user_defined/expr_planner.rs | 18 +- .../core/tests/user_defined/insert_operation.rs | 4 +- datafusion/core/tests/user_defined/mod.rs | 6 + .../core/tests/user_defined/relation_planner.rs | 531 +++ .../tests/user_defined/user_defined_aggregates.rs | 81 +- .../user_defined_async_scalar_functions.rs | 171 + .../core/tests/user_defined/user_defined_plan.rs | 88 +- .../user_defined/user_defined_scalar_functions.rs | 303 +- .../user_defined/user_defined_table_functions.rs | 41 +- .../user_defined/user_defined_window_functions.rs | 303 +- datafusion/datasource-arrow/NOTICE.txt | 2 +- datafusion/datasource-arrow/src/file_format.rs | 460 ++- datafusion/datasource-arrow/src/mod.rs | 4 + datafusion/datasource-arrow/src/source.rs | 587 +++- .../tests/data/example_stream.arrow | Bin 0 -> 1480 bytes .../example_stream_corrupted_metadata_length.arrow | Bin 0 -> 1480 bytes .../tests/data/example_stream_empty.arrow | Bin 0 -> 776 bytes .../src/avro_to_arrow/arrow_array_reader.rs | 156 +- .../datasource-avro/src/avro_to_arrow/reader.rs | 8 +- .../datasource-avro/src/avro_to_arrow/schema.rs | 20 +- datafusion/datasource-avro/src/file_format.rs | 18 +- datafusion/datasource-avro/src/mod.rs | 2 + datafusion/datasource-avro/src/source.rs | 105 +- datafusion/datasource-csv/Cargo.toml | 2 +- datafusion/datasource-csv/src/file_format.rs | 99 +- datafusion/datasource-csv/src/mod.rs | 15 +- datafusion/datasource-csv/src/source.rs | 245 +- datafusion/datasource-json/src/file_format.rs | 16 +- datafusion/datasource-json/src/mod.rs | 2 + datafusion/datasource-json/src/source.rs | 97 +- datafusion/datasource-parquet/Cargo.toml | 7 + .../benches/parquet_nested_filter_pushdown.rs | 238 ++ datafusion/datasource-parquet/src/access_plan.rs | 37 +- datafusion/datasource-parquet/src/file_format.rs | 135 +- datafusion/datasource-parquet/src/metadata.rs | 371 +- datafusion/datasource-parquet/src/metrics.rs | 50 +- datafusion/datasource-parquet/src/mod.rs | 4 + datafusion/datasource-parquet/src/opener.rs | 1464 +++++--- datafusion/datasource-parquet/src/page_filter.rs | 19 +- datafusion/datasource-parquet/src/reader.rs | 63 +- datafusion/datasource-parquet/src/row_filter.rs | 755 +++- .../datasource-parquet/src/row_group_filter.rs | 239 +- datafusion/datasource-parquet/src/sort.rs | 1022 ++++++ datafusion/datasource-parquet/src/source.rs | 432 ++- .../datasource-parquet/src/supported_predicates.rs | 144 + datafusion/datasource/Cargo.toml | 16 +- .../benches/split_groups_by_statistics.rs | 2 +- datafusion/datasource/src/decoder.rs | 28 +- datafusion/datasource/src/display.rs | 11 +- datafusion/datasource/src/file.rs | 151 +- datafusion/datasource/src/file_compression_type.rs | 64 +- datafusion/datasource/src/file_format.rs | 83 +- datafusion/datasource/src/file_groups.rs | 379 +- datafusion/datasource/src/file_scan_config.rs | 1725 ++++----- datafusion/datasource/src/file_sink_config.rs | 4 +- datafusion/datasource/src/file_stream.rs | 147 +- datafusion/datasource/src/memory.rs | 54 +- datafusion/datasource/src/mod.rs | 228 +- datafusion/datasource/src/projection.rs | 630 ++++ datafusion/datasource/src/schema_adapter.rs | 1076 +----- datafusion/datasource/src/sink.rs | 16 +- datafusion/datasource/src/source.rs | 78 +- datafusion/datasource/src/statistics.rs | 51 +- datafusion/datasource/src/table_schema.rs | 113 +- datafusion/datasource/src/test_util.rs | 87 +- datafusion/datasource/src/url.rs | 408 ++- datafusion/datasource/src/write/demux.rs | 43 +- datafusion/datasource/src/write/mod.rs | 25 +- datafusion/datasource/src/write/orchestration.rs | 24 +- datafusion/doc/src/lib.rs | 10 +- datafusion/doc/src/udf.rs | 4 +- datafusion/execution/Cargo.toml | 3 +- datafusion/execution/src/cache/cache_manager.rs | 391 ++- datafusion/execution/src/cache/cache_unit.rs | 1084 ++---- .../{cache_unit.rs => file_metadata_cache.rs} | 670 ++-- datafusion/execution/src/cache/list_files_cache.rs | 1218 +++++++ datafusion/execution/src/cache/mod.rs | 62 +- datafusion/execution/src/config.rs | 9 +- datafusion/execution/src/disk_manager.rs | 75 +- datafusion/execution/src/lib.rs | 4 +- datafusion/execution/src/memory_pool/mod.rs | 39 +- datafusion/execution/src/memory_pool/pool.rs | 65 +- datafusion/execution/src/object_store.rs | 30 +- datafusion/execution/src/parquet_encryption.rs | 2 +- datafusion/execution/src/runtime_env.rs | 194 +- datafusion/execution/src/task.rs | 7 +- datafusion/expr-common/Cargo.toml | 2 +- datafusion/expr-common/src/accumulator.rs | 25 +- datafusion/expr-common/src/casts.rs | 9 +- datafusion/expr-common/src/columnar_value.rs | 341 +- datafusion/expr-common/src/dyn_eq.rs | 4 +- datafusion/expr-common/src/groups_accumulator.rs | 5 +- datafusion/expr-common/src/interval_arithmetic.rs | 1131 ++++-- datafusion/expr-common/src/lib.rs | 2 + datafusion/expr-common/src/signature.rs | 264 +- datafusion/expr-common/src/statistics.rs | 166 +- .../expr-common/src/type_coercion/aggregates.rs | 2 +- datafusion/expr-common/src/type_coercion/binary.rs | 339 +- .../src/type_coercion/binary/tests/arithmetic.rs | 21 +- .../src/type_coercion/binary/tests/comparison.rs | 68 +- .../src/type_coercion/binary/tests/mod.rs | 1 + .../type_coercion/binary/tests/run_end_encoded.rs | 99 + datafusion/expr/Cargo.toml | 2 +- datafusion/expr/src/arguments.rs | 475 ++- datafusion/expr/src/async_udf.rs | 2 +- datafusion/expr/src/conditional_expressions.rs | 4 +- datafusion/expr/src/execution_props.rs | 22 +- datafusion/expr/src/expr.rs | 159 +- datafusion/expr/src/expr_fn.rs | 9 +- datafusion/expr/src/expr_rewriter/guarantees.rs | 668 ++++ datafusion/expr/src/expr_rewriter/mod.rs | 17 +- datafusion/expr/src/expr_rewriter/order_by.rs | 12 +- datafusion/expr/src/expr_schema.rs | 447 ++- datafusion/expr/src/function.rs | 8 +- datafusion/expr/src/lib.rs | 22 +- datafusion/expr/src/literal.rs | 5 +- datafusion/expr/src/logical_plan/builder.rs | 199 +- datafusion/expr/src/logical_plan/ddl.rs | 159 +- datafusion/expr/src/logical_plan/display.rs | 51 +- datafusion/expr/src/logical_plan/dml.rs | 11 +- datafusion/expr/src/logical_plan/invariants.rs | 52 +- datafusion/expr/src/logical_plan/mod.rs | 20 +- datafusion/expr/src/logical_plan/plan.rs | 345 +- datafusion/expr/src/logical_plan/statement.rs | 14 +- datafusion/expr/src/logical_plan/tree_node.rs | 35 +- datafusion/expr/src/partition_evaluator.rs | 10 +- datafusion/expr/src/planner.rs | 124 +- datafusion/expr/src/predicate_bounds.rs | 681 ++++ .../collection/mod.rs => expr/src/preimage.rs} | 14 +- datafusion/expr/src/ptr_eq.rs | 2 +- datafusion/expr/src/registry.rs | 2 +- datafusion/expr/src/select_expr.rs | 2 +- datafusion/expr/src/simplify.rs | 122 +- datafusion/expr/src/test/function_stub.rs | 12 +- datafusion/expr/src/tree_node.rs | 24 +- datafusion/expr/src/type_coercion/functions.rs | 391 ++- datafusion/expr/src/udaf.rs | 39 +- datafusion/expr/src/udf.rs | 91 +- datafusion/expr/src/udf_eq.rs | 2 +- datafusion/expr/src/udwf.rs | 14 +- datafusion/expr/src/utils.rs | 75 +- datafusion/expr/src/window_frame.rs | 54 +- datafusion/expr/src/window_state.rs | 17 +- datafusion/ffi/Cargo.toml | 28 +- datafusion/ffi/README.md | 92 + datafusion/ffi/src/arrow_wrappers.rs | 42 +- datafusion/ffi/src/catalog_provider.rs | 286 +- datafusion/ffi/src/catalog_provider_list.rs | 389 +++ .../joins/hash_join => ffi/src/execution}/mod.rs | 10 +- datafusion/ffi/src/execution/task_ctx.rs | 287 ++ datafusion/ffi/src/execution/task_ctx_provider.rs | 228 ++ datafusion/ffi/src/execution_plan.rs | 233 +- datafusion/ffi/src/expr/columnar_value.rs | 84 + datafusion/ffi/src/expr/distribution.rs | 210 ++ datafusion/ffi/src/expr/expr_properties.rs | 113 + datafusion/ffi/src/expr/interval.rs | 58 + .../{core/tests/execution => ffi/src/expr}/mod.rs | 7 +- datafusion/ffi/src/insert_op.rs | 3 +- datafusion/ffi/src/lib.rs | 38 +- datafusion/ffi/src/physical_expr/mod.rs | 988 ++++++ datafusion/ffi/src/physical_expr/partitioning.rs | 99 + datafusion/ffi/src/physical_expr/sort.rs | 76 + datafusion/ffi/src/plan_properties.rs | 223 +- .../ffi/src/proto/logical_extension_codec.rs | 723 ++++ .../tests/schema_adapter => ffi/src/proto}/mod.rs | 3 +- .../ffi/src/proto/physical_extension_codec.rs | 677 ++++ datafusion/ffi/src/record_batch_stream.rs | 151 +- datafusion/ffi/src/schema_provider.rs | 304 +- .../src/{session_config.rs => session/config.rs} | 107 +- datafusion/ffi/src/session/mod.rs | 621 ++++ datafusion/ffi/src/table_provider.rs | 433 ++- datafusion/ffi/src/table_source.rs | 7 +- datafusion/ffi/src/tests/async_provider.rs | 68 +- datafusion/ffi/src/tests/catalog.rs | 94 +- datafusion/ffi/src/tests/mod.rs | 61 +- datafusion/ffi/src/tests/sync_provider.rs | 11 +- datafusion/ffi/src/tests/udf_udaf_udwf.rs | 70 +- datafusion/ffi/src/tests/utils.rs | 8 +- datafusion/ffi/src/udaf/accumulator.rs | 239 +- datafusion/ffi/src/udaf/accumulator_args.rs | 122 +- datafusion/ffi/src/udaf/groups_accumulator.rs | 258 +- datafusion/ffi/src/udaf/mod.rs | 403 ++- datafusion/ffi/src/udf/mod.rs | 288 +- datafusion/ffi/src/udf/return_type_args.rs | 18 +- datafusion/ffi/src/udtf.rs | 211 +- datafusion/ffi/src/udwf/mod.rs | 297 +- datafusion/ffi/src/udwf/partition_evaluator.rs | 248 +- .../ffi/src/udwf/partition_evaluator_args.rs | 108 +- datafusion/ffi/src/udwf/range.rs | 1 - datafusion/ffi/src/util.rs | 59 +- datafusion/ffi/src/volatility.rs | 3 +- datafusion/ffi/tests/ffi_catalog.rs | 84 + datafusion/ffi/tests/ffi_integration.rs | 46 +- datafusion/ffi/tests/ffi_udaf.rs | 83 +- datafusion/ffi/tests/ffi_udf.rs | 15 +- datafusion/ffi/tests/ffi_udtf.rs | 18 +- datafusion/ffi/tests/ffi_udwf.rs | 11 +- datafusion/ffi/tests/utils/mod.rs | 43 + .../benches/accumulate.rs | 2 +- .../src/aggregate/avg_distinct/decimal.rs | 4 +- .../src/aggregate/count_distinct/bytes.rs | 10 +- .../src/aggregate/count_distinct/dict.rs | 4 +- .../src/aggregate/count_distinct/native.rs | 80 +- .../src/aggregate/groups_accumulator.rs | 2 +- .../src/aggregate/groups_accumulator/accumulate.rs | 280 +- .../src/aggregate/groups_accumulator/bool_op.rs | 2 +- .../src/aggregate/groups_accumulator/nulls.rs | 2 +- .../src/aggregate/groups_accumulator/prim_op.rs | 7 +- .../src/aggregate/sum_distinct/numeric.rs | 78 +- datafusion/functions-aggregate-common/src/lib.rs | 4 + .../functions-aggregate-common/src/merge_arrays.rs | 2 +- .../functions-aggregate-common/src/min_max.rs | 12 +- .../src/noop_accumulator.rs | 70 + .../functions-aggregate-common/src/tdigest.rs | 63 +- datafusion/functions-aggregate-common/src/utils.rs | 101 +- datafusion/functions-aggregate/Cargo.toml | 2 +- .../functions-aggregate/benches/array_agg.rs | 7 +- datafusion/functions-aggregate/benches/count.rs | 3 +- .../functions-aggregate/benches/min_max_bytes.rs | 4 +- datafusion/functions-aggregate/benches/sum.rs | 5 +- .../functions-aggregate/src/approx_distinct.rs | 72 +- .../functions-aggregate/src/approx_median.rs | 90 +- .../src/approx_percentile_cont.rs | 112 +- .../src/approx_percentile_cont_with_weight.rs | 27 +- datafusion/functions-aggregate/src/array_agg.rs | 58 +- datafusion/functions-aggregate/src/average.rs | 102 +- .../functions-aggregate/src/bit_and_or_xor.rs | 77 +- datafusion/functions-aggregate/src/bool_and_or.rs | 44 +- datafusion/functions-aggregate/src/correlation.rs | 207 +- datafusion/functions-aggregate/src/count.rs | 114 +- datafusion/functions-aggregate/src/covariance.rs | 125 +- datafusion/functions-aggregate/src/first_last.rs | 277 +- datafusion/functions-aggregate/src/grouping.rs | 28 +- datafusion/functions-aggregate/src/lib.rs | 3 + datafusion/functions-aggregate/src/median.rs | 184 +- datafusion/functions-aggregate/src/min_max.rs | 28 +- .../src/min_max/min_max_bytes.rs | 2 +- .../src/min_max/min_max_struct.rs | 3 +- datafusion/functions-aggregate/src/nth_value.rs | 32 +- .../functions-aggregate/src/percentile_cont.rs | 590 ++-- datafusion/functions-aggregate/src/planner.rs | 2 +- datafusion/functions-aggregate/src/regr.rs | 106 +- datafusion/functions-aggregate/src/stddev.rs | 63 +- datafusion/functions-aggregate/src/string_agg.rs | 36 +- datafusion/functions-aggregate/src/sum.rs | 166 +- datafusion/functions-aggregate/src/utils.rs | 4 +- datafusion/functions-aggregate/src/variance.rs | 208 +- datafusion/functions-nested/Cargo.toml | 10 +- datafusion/functions-nested/benches/array_has.rs | 377 ++ .../functions-nested/benches/array_reverse.rs | 76 +- datafusion/functions-nested/benches/array_slice.rs | 230 ++ datafusion/functions-nested/benches/map.rs | 6 +- datafusion/functions-nested/src/array_has.rs | 80 +- datafusion/functions-nested/src/cardinality.rs | 7 +- datafusion/functions-nested/src/concat.rs | 15 +- datafusion/functions-nested/src/dimension.rs | 8 +- datafusion/functions-nested/src/distance.rs | 6 +- datafusion/functions-nested/src/empty.rs | 5 +- datafusion/functions-nested/src/except.rs | 60 +- datafusion/functions-nested/src/extract.rs | 14 +- datafusion/functions-nested/src/flatten.rs | 19 +- datafusion/functions-nested/src/length.rs | 5 +- datafusion/functions-nested/src/lib.rs | 3 + datafusion/functions-nested/src/make_array.rs | 45 +- datafusion/functions-nested/src/map.rs | 414 ++- datafusion/functions-nested/src/map_entries.rs | 2 +- datafusion/functions-nested/src/map_extract.rs | 4 +- datafusion/functions-nested/src/map_keys.rs | 2 +- datafusion/functions-nested/src/map_values.rs | 2 +- datafusion/functions-nested/src/min_max.rs | 15 +- datafusion/functions-nested/src/planner.rs | 25 +- datafusion/functions-nested/src/position.rs | 31 +- datafusion/functions-nested/src/range.rs | 16 +- datafusion/functions-nested/src/remove.rs | 501 ++- datafusion/functions-nested/src/repeat.rs | 9 +- datafusion/functions-nested/src/replace.rs | 26 +- datafusion/functions-nested/src/resize.rs | 11 +- datafusion/functions-nested/src/reverse.rs | 119 +- datafusion/functions-nested/src/set_ops.rs | 134 +- datafusion/functions-nested/src/sort.rs | 10 +- datafusion/functions-nested/src/string.rs | 127 +- datafusion/functions-nested/src/utils.rs | 4 +- datafusion/functions-table/Cargo.toml | 2 +- datafusion/functions-table/src/generate_series.rs | 87 +- datafusion/functions-table/src/lib.rs | 3 + datafusion/functions-window-common/src/lib.rs | 3 + datafusion/functions-window/Cargo.toml | 10 +- datafusion/functions-window/benches/nth_value.rs | 263 ++ datafusion/functions-window/src/cume_dist.rs | 4 +- datafusion/functions-window/src/lead_lag.rs | 6 +- datafusion/functions-window/src/lib.rs | 5 +- datafusion/functions-window/src/nth_value.rs | 223 +- datafusion/functions-window/src/ntile.rs | 6 +- datafusion/functions-window/src/planner.rs | 2 +- datafusion/functions-window/src/rank.rs | 7 +- datafusion/functions-window/src/utils.rs | 6 +- datafusion/functions/Cargo.toml | 79 +- datafusion/functions/benches/ascii.rs | 33 +- datafusion/functions/benches/character_length.rs | 2 +- datafusion/functions/benches/chr.rs | 2 +- datafusion/functions/benches/concat.rs | 104 +- .../functions/benches/{concat.rs => concat_ws.rs} | 68 +- datafusion/functions/benches/contains.rs | 185 + datafusion/functions/benches/cot.rs | 49 +- .../functions/benches/{upper.rs => crypto.rs} | 51 +- datafusion/functions/benches/date_bin.rs | 6 +- datafusion/functions/benches/date_trunc.rs | 6 +- datafusion/functions/benches/encoding.rs | 2 +- datafusion/functions/benches/ends_with.rs | 185 + .../functions/benches/{uuid.rs => factorial.rs} | 37 +- datafusion/functions/benches/find_in_set.rs | 4 +- datafusion/functions/benches/floor_ceil.rs | 135 + datafusion/functions/benches/gcd.rs | 4 +- datafusion/functions/benches/helper.rs | 2 +- datafusion/functions/benches/initcap.rs | 125 +- datafusion/functions/benches/isnan.rs | 2 +- datafusion/functions/benches/iszero.rs | 47 +- datafusion/functions/benches/left.rs | 111 + datafusion/functions/benches/levenshtein.rs | 87 + datafusion/functions/benches/lower.rs | 2 +- datafusion/functions/benches/make_date.rs | 6 +- datafusion/functions/benches/nullif.rs | 4 +- datafusion/functions/benches/pad.rs | 315 +- datafusion/functions/benches/random.rs | 2 +- datafusion/functions/benches/regexp_count.rs | 118 + datafusion/functions/benches/regx.rs | 4 +- datafusion/functions/benches/repeat.rs | 4 +- .../functions/benches/{repeat.rs => replace.rs} | 112 +- datafusion/functions/benches/reverse.rs | 2 +- datafusion/functions/benches/round.rs | 154 + datafusion/functions/benches/signum.rs | 48 +- datafusion/functions/benches/split_part.rs | 382 ++ datafusion/functions/benches/starts_with.rs | 185 + datafusion/functions/benches/strpos.rs | 2 +- datafusion/functions/benches/substr.rs | 5 +- datafusion/functions/benches/substr_index.rs | 128 +- datafusion/functions/benches/to_char.rs | 8 +- datafusion/functions/benches/to_hex.rs | 120 +- datafusion/functions/benches/to_timestamp.rs | 26 +- datafusion/functions/benches/translate.rs | 90 + datafusion/functions/benches/{ltrim.rs => trim.rs} | 208 +- datafusion/functions/benches/trunc.rs | 50 +- datafusion/functions/benches/upper.rs | 2 +- datafusion/functions/benches/uuid.rs | 2 +- datafusion/functions/src/core/arrow_cast.rs | 23 +- datafusion/functions/src/core/arrow_metadata.rs | 160 + datafusion/functions/src/core/arrowtypeof.rs | 2 +- datafusion/functions/src/core/coalesce.rs | 6 +- datafusion/functions/src/core/getfield.rs | 601 +++- datafusion/functions/src/core/greatest.rs | 22 +- .../functions/src/core/greatest_least_utils.rs | 25 +- datafusion/functions/src/core/least.rs | 22 +- datafusion/functions/src/core/mod.rs | 16 + datafusion/functions/src/core/named_struct.rs | 2 +- datafusion/functions/src/core/nullif.rs | 2 +- datafusion/functions/src/core/nvl.rs | 4 +- datafusion/functions/src/core/nvl2.rs | 10 +- datafusion/functions/src/core/overlay.rs | 6 +- datafusion/functions/src/core/planner.rs | 2 +- datafusion/functions/src/core/struct.rs | 2 +- datafusion/functions/src/core/union_extract.rs | 20 +- datafusion/functions/src/core/union_tag.rs | 6 +- datafusion/functions/src/core/version.rs | 4 +- datafusion/functions/src/crypto/basic.rs | 186 +- datafusion/functions/src/crypto/digest.rs | 48 +- datafusion/functions/src/crypto/md5.rs | 52 +- datafusion/functions/src/crypto/mod.rs | 13 +- datafusion/functions/src/crypto/sha.rs | 175 + datafusion/functions/src/crypto/sha224.rs | 104 - datafusion/functions/src/crypto/sha256.rs | 102 - datafusion/functions/src/crypto/sha384.rs | 102 - datafusion/functions/src/crypto/sha512.rs | 102 - datafusion/functions/src/datetime/common.rs | 183 +- datafusion/functions/src/datetime/current_date.rs | 25 +- datafusion/functions/src/datetime/current_time.rs | 72 +- datafusion/functions/src/datetime/date_bin.rs | 330 +- datafusion/functions/src/datetime/date_part.rs | 43 +- datafusion/functions/src/datetime/date_trunc.rs | 481 ++- datafusion/functions/src/datetime/from_unixtime.rs | 4 +- datafusion/functions/src/datetime/make_date.rs | 337 +- datafusion/functions/src/datetime/make_time.rs | 267 ++ datafusion/functions/src/datetime/mod.rs | 62 +- datafusion/functions/src/datetime/now.rs | 22 +- datafusion/functions/src/datetime/planner.rs | 2 +- datafusion/functions/src/datetime/to_char.rs | 28 +- datafusion/functions/src/datetime/to_date.rs | 59 +- datafusion/functions/src/datetime/to_local_time.rs | 347 +- datafusion/functions/src/datetime/to_time.rs | 252 ++ datafusion/functions/src/datetime/to_timestamp.rs | 1038 ++++-- datafusion/functions/src/datetime/to_unixtime.rs | 40 +- datafusion/functions/src/encoding/inner.rs | 653 ++-- datafusion/functions/src/lib.rs | 3 + datafusion/functions/src/macros.rs | 55 +- datafusion/functions/src/math/abs.rs | 24 +- datafusion/functions/src/math/ceil.rs | 206 ++ datafusion/functions/src/math/cot.rs | 301 +- datafusion/functions/src/math/decimal.rs | 111 + datafusion/functions/src/math/factorial.rs | 122 +- datafusion/functions/src/math/floor.rs | 206 ++ datafusion/functions/src/math/gcd.rs | 35 +- datafusion/functions/src/math/iszero.rs | 133 +- datafusion/functions/src/math/lcm.rs | 2 +- datafusion/functions/src/math/log.rs | 241 +- datafusion/functions/src/math/mod.rs | 21 +- datafusion/functions/src/math/monotonicity.rs | 72 +- datafusion/functions/src/math/nans.rs | 34 +- datafusion/functions/src/math/nanvl.rs | 41 +- datafusion/functions/src/math/pi.rs | 10 +- datafusion/functions/src/math/power.rs | 667 +++- datafusion/functions/src/math/random.rs | 12 +- datafusion/functions/src/math/round.rs | 429 ++- datafusion/functions/src/math/signum.rs | 90 +- datafusion/functions/src/math/trunc.rs | 73 +- datafusion/functions/src/planner.rs | 2 +- datafusion/functions/src/regex/mod.rs | 2 +- datafusion/functions/src/regex/regexpcount.rs | 66 +- datafusion/functions/src/regex/regexpinstr.rs | 55 +- datafusion/functions/src/regex/regexplike.rs | 55 +- datafusion/functions/src/regex/regexpmatch.rs | 33 +- datafusion/functions/src/regex/regexpreplace.rs | 68 +- datafusion/functions/src/string/ascii.rs | 38 +- datafusion/functions/src/string/btrim.rs | 4 +- datafusion/functions/src/string/chr.rs | 14 +- datafusion/functions/src/string/common.rs | 245 +- datafusion/functions/src/string/concat.rs | 48 +- datafusion/functions/src/string/concat_ws.rs | 97 +- datafusion/functions/src/string/contains.rs | 91 +- datafusion/functions/src/string/ends_with.rs | 292 +- datafusion/functions/src/string/levenshtein.rs | 34 +- datafusion/functions/src/string/lower.rs | 2 +- datafusion/functions/src/string/ltrim.rs | 12 +- datafusion/functions/src/string/octet_length.rs | 2 +- datafusion/functions/src/string/repeat.rs | 47 +- datafusion/functions/src/string/replace.rs | 91 +- datafusion/functions/src/string/rtrim.rs | 12 +- datafusion/functions/src/string/split_part.rs | 109 +- datafusion/functions/src/string/starts_with.rs | 285 +- datafusion/functions/src/string/to_hex.rs | 227 +- datafusion/functions/src/string/upper.rs | 2 +- datafusion/functions/src/string/uuid.rs | 14 +- datafusion/functions/src/strings.rs | 4 +- .../functions/src/unicode/character_length.rs | 12 +- datafusion/functions/src/unicode/find_in_set.rs | 48 +- datafusion/functions/src/unicode/initcap.rs | 35 +- datafusion/functions/src/unicode/left.rs | 19 +- datafusion/functions/src/unicode/lpad.rs | 59 +- datafusion/functions/src/unicode/planner.rs | 2 +- datafusion/functions/src/unicode/reverse.rs | 12 +- datafusion/functions/src/unicode/right.rs | 6 +- datafusion/functions/src/unicode/rpad.rs | 60 +- datafusion/functions/src/unicode/strpos.rs | 63 +- datafusion/functions/src/unicode/substr.rs | 116 +- datafusion/functions/src/unicode/substrindex.rs | 74 +- datafusion/functions/src/unicode/translate.rs | 71 +- datafusion/functions/src/utils.rs | 196 +- datafusion/macros/Cargo.toml | 2 +- datafusion/macros/src/user_doc.rs | 3 +- .../optimizer/benches/projection_unnecessary.rs | 4 +- .../optimizer/src/analyzer/function_rewrite.rs | 2 +- datafusion/optimizer/src/analyzer/mod.rs | 2 +- .../src/analyzer/resolve_grouping_function.rs | 8 +- datafusion/optimizer/src/analyzer/type_coercion.rs | 273 +- .../optimizer/src/common_subexpr_eliminate.rs | 14 +- datafusion/optimizer/src/decorrelate.rs | 16 +- .../optimizer/src/decorrelate_lateral_join.rs | 6 +- .../src/decorrelate_predicate_subquery.rs | 99 +- datafusion/optimizer/src/eliminate_cross_join.rs | 36 +- .../optimizer/src/eliminate_duplicated_expr.rs | 6 +- datafusion/optimizer/src/eliminate_filter.rs | 6 +- .../optimizer/src/eliminate_group_by_constant.rs | 8 +- datafusion/optimizer/src/eliminate_join.rs | 4 +- datafusion/optimizer/src/eliminate_limit.rs | 10 +- datafusion/optimizer/src/eliminate_one_union.rs | 121 - datafusion/optimizer/src/eliminate_outer_join.rs | 7 +- .../optimizer/src/extract_equijoin_predicate.rs | 23 +- datafusion/optimizer/src/filter_null_join_keys.rs | 6 +- datafusion/optimizer/src/join_key_set.rs | 2 +- datafusion/optimizer/src/lib.rs | 13 +- .../optimizer/src/optimize_projections/mod.rs | 66 +- ...liminate_nested_union.rs => optimize_unions.rs} | 104 +- datafusion/optimizer/src/optimizer.rs | 53 +- datafusion/optimizer/src/plan_signature.rs | 2 +- .../optimizer/src/propagate_empty_relation.rs | 22 +- datafusion/optimizer/src/push_down_filter.rs | 45 +- datafusion/optimizer/src/push_down_limit.rs | 20 +- .../optimizer/src/replace_distinct_aggregate.rs | 6 +- datafusion/optimizer/src/rewrite_set_comparison.rs | 171 + .../optimizer/src/scalar_subquery_to_join.rs | 64 +- .../src/simplify_expressions/expr_simplifier.rs | 522 ++- .../src/simplify_expressions/guarantees.rs | 476 --- .../src/simplify_expressions/inlist_simplifier.rs | 92 +- .../optimizer/src/simplify_expressions/mod.rs | 7 +- .../optimizer/src/simplify_expressions/regex.rs | 20 +- .../src/simplify_expressions/simplify_exprs.rs | 40 +- .../src/simplify_expressions/simplify_literal.rs | 148 + .../src/simplify_expressions/udf_preimage.rs | 364 ++ .../src/simplify_expressions/unwrap_cast.rs | 26 +- .../optimizer/src/simplify_expressions/utils.rs | 52 +- .../optimizer/src/single_distinct_to_groupby.rs | 41 +- datafusion/optimizer/src/test/mod.rs | 4 +- datafusion/optimizer/src/test/user_defined.rs | 2 +- datafusion/optimizer/src/utils.rs | 6 +- .../optimizer/tests/optimizer_integration.rs | 369 +- datafusion/physical-expr-adapter/LICENSE.txt | 1 + datafusion/physical-expr-adapter/NOTICE.txt | 1 + datafusion/physical-expr-adapter/src/lib.rs | 8 +- .../physical-expr-adapter/src/schema_rewriter.rs | 793 ++++- datafusion/physical-expr-common/Cargo.toml | 3 + datafusion/physical-expr-common/src/binary_map.rs | 8 +- .../physical-expr-common/src/binary_view_map.rs | 5 +- datafusion/physical-expr-common/src/datum.rs | 86 +- datafusion/physical-expr-common/src/lib.rs | 4 + .../src/metrics/baseline.rs | 24 +- .../src/metrics/builder.rs | 24 +- .../src/metrics/custom.rs | 2 +- .../physical-expr-common/src/metrics/expression.rs | 88 + .../src/metrics/mod.rs | 39 +- .../src/metrics/value.rs | 342 +- .../physical-expr-common/src/physical_expr.rs | 75 +- datafusion/physical-expr-common/src/sort_expr.rs | 126 +- datafusion/physical-expr-common/src/tree_node.rs | 4 +- datafusion/physical-expr-common/src/utils.rs | 40 +- datafusion/physical-expr/Cargo.toml | 12 +- datafusion/physical-expr/benches/binary_op.rs | 7 +- datafusion/physical-expr/benches/case_when.rs | 12 +- datafusion/physical-expr/benches/in_list.rs | 280 +- datafusion/physical-expr/benches/is_null.rs | 4 +- datafusion/physical-expr/src/aggregate.rs | 42 +- datafusion/physical-expr/src/analysis.rs | 91 +- .../physical-expr/src/async_scalar_function.rs | 38 +- datafusion/physical-expr/src/equivalence/class.rs | 43 +- datafusion/physical-expr/src/equivalence/mod.rs | 6 +- .../physical-expr/src/equivalence/ordering.rs | 15 +- .../src/equivalence/properties/dependency.rs | 6 +- .../src/equivalence/properties/joins.rs | 2 +- .../src/equivalence/properties/mod.rs | 12 +- .../src/equivalence/properties/union.rs | 4 +- datafusion/physical-expr/src/expressions/binary.rs | 307 +- .../src/expressions/binary/kernels.rs | 49 +- datafusion/physical-expr/src/expressions/case.rs | 1387 ++++++-- .../literal_lookup_table/boolean_lookup_table.rs | 122 + .../bytes_like_lookup_table.rs | 223 ++ .../expressions/case/literal_lookup_table/mod.rs | 327 ++ .../literal_lookup_table/primitive_lookup_table.rs | 229 ++ datafusion/physical-expr/src/expressions/cast.rs | 69 +- .../physical-expr/src/expressions/cast_column.rs | 6 +- datafusion/physical-expr/src/expressions/column.rs | 8 +- .../src/expressions/dynamic_filters.rs | 379 +- .../physical-expr/src/expressions/in_list.rs | 3659 +++++++++++++++----- datafusion/physical-expr/src/expressions/like.rs | 22 +- .../physical-expr/src/expressions/literal.rs | 1 + datafusion/physical-expr/src/expressions/mod.rs | 26 +- .../physical-expr/src/expressions/negative.rs | 20 +- datafusion/physical-expr/src/expressions/no_op.rs | 2 +- datafusion/physical-expr/src/expressions/not.rs | 49 +- .../physical-expr/src/expressions/try_cast.rs | 12 +- .../src/expressions/unknown_column.rs | 2 +- .../physical-expr/src/intervals/cp_solver.rs | 91 +- .../physical-expr/src/intervals/test_utils.rs | 6 +- datafusion/physical-expr/src/intervals/utils.rs | 6 +- datafusion/physical-expr/src/lib.rs | 7 +- datafusion/physical-expr/src/partitioning.rs | 590 +++- datafusion/physical-expr/src/physical_expr.rs | 4 +- datafusion/physical-expr/src/planner.rs | 41 +- datafusion/physical-expr/src/projection.rs | 727 +++- datafusion/physical-expr/src/scalar_function.rs | 45 +- .../src/simplifier/const_evaluator.rs | 103 + datafusion/physical-expr/src/simplifier/mod.rs | 538 ++- datafusion/physical-expr/src/simplifier/not.rs | 124 + .../physical-expr/src/simplifier/unwrap_cast.rs | 53 +- .../physical-expr/src/statistics/stats_solver.rs | 8 +- datafusion/physical-expr/src/utils/guarantee.rs | 8 +- datafusion/physical-expr/src/utils/mod.rs | 10 +- datafusion/physical-expr/src/window/aggregate.rs | 4 +- .../physical-expr/src/window/sliding_aggregate.rs | 4 +- datafusion/physical-expr/src/window/standard.rs | 6 +- .../src/window/standard_window_function_expr.rs | 12 +- datafusion/physical-expr/src/window/window_expr.rs | 18 +- .../physical-optimizer/src/aggregate_statistics.rs | 45 +- .../src/coalesce_async_exec_input.rs | 71 - .../physical-optimizer/src/coalesce_batches.rs | 94 - .../src/combine_partial_final_agg.rs | 10 +- .../physical-optimizer/src/enforce_distribution.rs | 352 +- .../physical-optimizer/src/enforce_sorting/mod.rs | 29 +- .../replace_with_order_preserving_variants.rs | 25 +- .../src/enforce_sorting/sort_pushdown.rs | 10 +- datafusion/physical-optimizer/src/ensure_coop.rs | 340 +- .../physical-optimizer/src/filter_pushdown.rs | 60 +- .../physical-optimizer/src/join_selection.rs | 56 +- datafusion/physical-optimizer/src/lib.rs | 6 +- .../physical-optimizer/src/limit_pushdown.rs | 29 +- .../src/limit_pushdown_past_window.rs | 10 +- .../src/limited_distinct_aggregation.rs | 24 +- datafusion/physical-optimizer/src/optimizer.rs | 11 +- .../physical-optimizer/src/output_requirements.rs | 2 +- .../physical-optimizer/src/projection_pushdown.rs | 14 +- datafusion/physical-optimizer/src/pushdown_sort.rs | 129 + .../physical-optimizer/src/sanity_checker.rs | 20 +- .../physical-optimizer/src/topk_aggregation.rs | 63 +- .../physical-optimizer/src/update_aggr_exprs.rs | 6 +- datafusion/physical-plan/Cargo.toml | 5 +- .../physical-plan/benches/aggregate_vectorized.rs | 5 +- .../physical-plan/benches/partial_ordering.rs | 2 +- .../physical-plan/benches/sort_preserving_merge.rs | 4 +- datafusion/physical-plan/benches/spill_io.rs | 7 +- .../src/aggregates/group_values/metrics.rs | 2 +- .../src/aggregates/group_values/mod.rs | 4 +- .../group_values/multi_group_by/boolean.rs | 2 +- .../group_values/multi_group_by/bytes.rs | 10 +- .../group_values/multi_group_by/bytes_view.rs | 163 +- .../aggregates/group_values/multi_group_by/mod.rs | 38 +- .../group_values/multi_group_by/primitive.rs | 118 +- .../src/aggregates/group_values/null_builder.rs | 8 + .../src/aggregates/group_values/row.rs | 25 +- .../group_values/single_group_by/boolean.rs | 3 +- .../group_values/single_group_by/bytes.rs | 4 +- .../group_values/single_group_by/bytes_view.rs | 4 +- .../group_values/single_group_by/primitive.rs | 18 +- datafusion/physical-plan/src/aggregates/mod.rs | 1172 +++++-- .../physical-plan/src/aggregates/no_grouping.rs | 284 +- .../physical-plan/src/aggregates/row_hash.rs | 715 ++-- .../src/aggregates/topk/hash_table.rs | 334 +- .../physical-plan/src/aggregates/topk/heap.rs | 343 +- .../src/aggregates/topk/priority_map.rs | 267 +- .../physical-plan/src/aggregates/topk_stream.rs | 86 +- datafusion/physical-plan/src/analyze.rs | 24 +- datafusion/physical-plan/src/async_func.rs | 153 +- datafusion/physical-plan/src/coalesce/mod.rs | 15 +- datafusion/physical-plan/src/coalesce_batches.rs | 36 + .../physical-plan/src/coalesce_partitions.rs | 65 +- datafusion/physical-plan/src/column_rewriter.rs | 383 ++ datafusion/physical-plan/src/common.rs | 5 +- datafusion/physical-plan/src/coop.rs | 56 +- datafusion/physical-plan/src/display.rs | 4 +- datafusion/physical-plan/src/empty.rs | 39 +- datafusion/physical-plan/src/execution_plan.rs | 134 +- datafusion/physical-plan/src/explain.rs | 23 +- datafusion/physical-plan/src/filter.rs | 775 ++++- datafusion/physical-plan/src/filter_pushdown.rs | 77 +- datafusion/physical-plan/src/joins/array_map.rs | 547 +++ datafusion/physical-plan/src/joins/chain.rs | 69 + datafusion/physical-plan/src/joins/cross_join.rs | 77 +- .../physical-plan/src/joins/hash_join/exec.rs | 1864 ++++++++-- .../src/joins/hash_join/inlist_builder.rs | 133 + .../physical-plan/src/joins/hash_join/mod.rs | 3 + .../src/joins/hash_join/partitioned_hash_eval.rs | 602 ++++ .../src/joins/hash_join/shared_bounds.rs | 593 +++- .../physical-plan/src/joins/hash_join/stream.rs | 471 ++- .../physical-plan/src/joins/join_hash_map.rs | 188 +- datafusion/physical-plan/src/joins/mod.rs | 35 +- .../physical-plan/src/joins/nested_loop_join.rs | 630 +++- .../src/joins/piecewise_merge_join/classic_join.rs | 167 +- .../src/joins/piecewise_merge_join/exec.rs | 23 +- .../src/joins/sort_merge_join/exec.rs | 49 +- .../src/joins/sort_merge_join/metrics.rs | 10 - .../src/joins/sort_merge_join/stream.rs | 777 +++-- .../src/joins/sort_merge_join/tests.rs | 1059 ++++-- .../physical-plan/src/joins/stream_join_utils.rs | 134 +- .../physical-plan/src/joins/symmetric_hash_join.rs | 81 +- datafusion/physical-plan/src/joins/test_utils.rs | 13 +- datafusion/physical-plan/src/joins/utils.rs | 198 +- datafusion/physical-plan/src/lib.rs | 20 +- datafusion/physical-plan/src/limit.rs | 56 +- datafusion/physical-plan/src/memory.rs | 102 +- .../src/{joins/hash_join/mod.rs => metrics.rs} | 9 +- datafusion/physical-plan/src/placeholder_row.rs | 27 +- datafusion/physical-plan/src/projection.rs | 701 +++- datafusion/physical-plan/src/recursive_query.rs | 141 +- datafusion/physical-plan/src/render_tree.rs | 3 +- .../src/repartition/distributor_channels.rs | 4 +- datafusion/physical-plan/src/repartition/mod.rs | 515 ++- datafusion/physical-plan/src/sort_pushdown.rs | 120 + datafusion/physical-plan/src/sorts/cursor.rs | 4 +- datafusion/physical-plan/src/sorts/merge.rs | 4 +- .../physical-plan/src/sorts/multi_level_merge.rs | 20 +- datafusion/physical-plan/src/sorts/partial_sort.rs | 159 +- datafusion/physical-plan/src/sorts/sort.rs | 596 +++- .../src/sorts/sort_preserving_merge.rs | 213 +- datafusion/physical-plan/src/sorts/stream.rs | 19 +- .../physical-plan/src/sorts/streaming_merge.rs | 15 +- .../src/spill/in_progress_spill_file.rs | 31 +- datafusion/physical-plan/src/spill/mod.rs | 289 +- .../physical-plan/src/spill/spill_manager.rs | 9 +- datafusion/physical-plan/src/spill/spill_pool.rs | 77 +- datafusion/physical-plan/src/stream.rs | 191 +- datafusion/physical-plan/src/streaming.rs | 8 +- datafusion/physical-plan/src/test.rs | 31 +- datafusion/physical-plan/src/test/exec.rs | 12 +- datafusion/physical-plan/src/topk/mod.rs | 88 +- datafusion/physical-plan/src/tree_node.rs | 4 +- datafusion/physical-plan/src/union.rs | 157 +- datafusion/physical-plan/src/unnest.rs | 79 +- .../src/windows/bounded_window_agg_exec.rs | 144 +- datafusion/physical-plan/src/windows/mod.rs | 58 +- .../physical-plan/src/windows/window_agg_exec.rs | 56 +- datafusion/physical-plan/src/work_table.rs | 116 +- datafusion/proto-common/gen/Cargo.toml | 4 +- .../proto-common/proto/datafusion_common.proto | 6 + datafusion/proto-common/src/common.rs | 2 +- datafusion/proto-common/src/from_proto/mod.rs | 45 +- datafusion/proto-common/src/generated/mod.rs | 1 + datafusion/proto-common/src/generated/pbjson.rs | 208 +- datafusion/proto-common/src/generated/prost.rs | 11 + datafusion/proto-common/src/lib.rs | 2 + datafusion/proto-common/src/to_proto/mod.rs | 16 +- datafusion/proto/Cargo.toml | 4 +- datafusion/proto/gen/Cargo.toml | 4 +- datafusion/proto/proto/datafusion.proto | 35 + datafusion/proto/src/bytes/mod.rs | 8 +- datafusion/proto/src/bytes/registry.rs | 29 +- datafusion/proto/src/common.rs | 12 +- datafusion/proto/src/generated/datafusion.rs | 1 + .../proto/src/generated/datafusion_proto_common.rs | 11 + datafusion/proto/src/generated/mod.rs | 2 + datafusion/proto/src/generated/pbjson.rs | 1007 +++++- datafusion/proto/src/generated/prost.rs | 58 +- datafusion/proto/src/lib.rs | 2 + datafusion/proto/src/logical_plan/file_formats.rs | 30 +- datafusion/proto/src/logical_plan/from_proto.rs | 20 +- datafusion/proto/src/logical_plan/mod.rs | 92 +- datafusion/proto/src/logical_plan/to_proto.rs | 47 +- datafusion/proto/src/physical_plan/from_proto.rs | 175 +- datafusion/proto/src/physical_plan/mod.rs | 216 +- datafusion/proto/src/physical_plan/to_proto.rs | 80 +- .../proto/tests/cases/roundtrip_logical_plan.rs | 60 +- .../proto/tests/cases/roundtrip_physical_plan.rs | 383 +- datafusion/proto/tests/cases/serialize.rs | 6 +- datafusion/pruning/LICENSE.txt | 1 + datafusion/pruning/NOTICE.txt | 1 + datafusion/pruning/src/file_pruner.rs | 131 +- datafusion/pruning/src/lib.rs | 7 +- datafusion/pruning/src/pruning_predicate.rs | 390 ++- datafusion/session/src/lib.rs | 3 + datafusion/session/src/session.rs | 4 +- datafusion/spark/Cargo.toml | 24 + datafusion/spark/benches/char.rs | 2 +- datafusion/spark/benches/hex.rs | 152 + datafusion/spark/benches/slice.rs | 187 + datafusion/spark/benches/{char.rs => space.rs} | 27 +- .../substr.rs => spark/benches/substring.rs} | 9 +- datafusion/spark/benches/unhex.rs | 148 + datafusion/spark/src/function/aggregate/avg.rs | 35 +- datafusion/spark/src/function/aggregate/collect.rs | 200 ++ datafusion/spark/src/function/aggregate/mod.rs | 29 +- datafusion/spark/src/function/aggregate/try_sum.rs | 660 ++++ datafusion/spark/src/function/array/mod.rs | 16 +- datafusion/spark/src/function/array/repeat.rs | 128 + datafusion/spark/src/function/array/shuffle.rs | 66 +- datafusion/spark/src/function/array/slice.rs | 172 + datafusion/spark/src/function/array/spark_array.rs | 135 +- .../spark/src/function/bitmap/bitmap_count.rs | 73 +- datafusion/spark/src/function/bitwise/bit_count.rs | 85 +- datafusion/spark/src/function/bitwise/bit_get.rs | 261 +- datafusion/spark/src/function/bitwise/bit_shift.rs | 775 ++--- .../spark/src/function/bitwise/bitwise_not.rs | 88 +- datafusion/spark/src/function/bitwise/mod.rs | 18 +- datafusion/spark/src/function/collection/mod.rs | 13 +- datafusion/spark/src/function/collection/size.rs | 162 + datafusion/spark/src/function/conditional/if.rs | 8 +- .../spark/src/function/datetime/add_months.rs | 95 + datafusion/spark/src/function/datetime/date_add.rs | 98 +- .../spark/src/function/datetime/date_diff.rs | 119 + .../spark/src/function/datetime/date_part.rs | 139 + datafusion/spark/src/function/datetime/date_sub.rs | 85 +- .../spark/src/function/datetime/date_trunc.rs | 172 + datafusion/spark/src/function/datetime/extract.rs | 268 ++ datafusion/spark/src/function/datetime/last_day.rs | 92 +- .../src/function/datetime/make_dt_interval.rs | 175 +- .../spark/src/function/datetime/make_interval.rs | 130 +- datafusion/spark/src/function/datetime/mod.rs | 66 + datafusion/spark/src/function/datetime/next_day.rs | 89 +- .../spark/src/function/datetime/time_trunc.rs | 122 + datafusion/spark/src/function/datetime/trunc.rs | 143 + datafusion/spark/src/function/error_utils.rs | 6 +- datafusion/spark/src/function/hash/crc32.rs | 95 +- datafusion/spark/src/function/hash/sha1.rs | 106 +- datafusion/spark/src/function/hash/sha2.rs | 35 +- .../spark/src/function/map/map_from_arrays.rs | 88 +- .../spark/src/function/map/map_from_entries.rs | 103 +- datafusion/spark/src/function/map/utils.rs | 11 +- datafusion/spark/src/function/math/abs.rs | 448 +++ datafusion/spark/src/function/math/expm1.rs | 63 +- datafusion/spark/src/function/math/factorial.rs | 12 +- datafusion/spark/src/function/math/hex.rs | 434 ++- datafusion/spark/src/function/math/mod.rs | 21 +- datafusion/spark/src/function/math/modulus.rs | 34 +- datafusion/spark/src/function/math/rint.rs | 8 +- datafusion/spark/src/function/math/trigonometry.rs | 167 + datafusion/spark/src/function/math/unhex.rs | 214 ++ datafusion/spark/src/function/math/width_bucket.rs | 149 +- datafusion/spark/src/function/mod.rs | 1 + datafusion/spark/src/function/null_utils.rs | 122 + datafusion/spark/src/function/string/ascii.rs | 77 +- datafusion/spark/src/function/string/char.rs | 63 +- datafusion/spark/src/function/string/concat.rs | 177 +- datafusion/spark/src/function/string/elt.rs | 2 +- .../spark/src/function/string/format_string.rs | 108 +- datafusion/spark/src/function/string/ilike.rs | 90 +- datafusion/spark/src/function/string/length.rs | 70 +- datafusion/spark/src/function/string/like.rs | 92 +- datafusion/spark/src/function/string/luhn_check.rs | 2 +- datafusion/spark/src/function/string/mod.rs | 12 + datafusion/spark/src/function/string/space.rs | 232 ++ datafusion/spark/src/function/string/substring.rs | 258 ++ datafusion/spark/src/function/url/mod.rs | 29 +- datafusion/spark/src/function/url/parse_url.rs | 24 +- datafusion/spark/src/function/url/try_parse_url.rs | 2 +- .../url/{try_parse_url.rs => try_url_decode.rs} | 64 +- datafusion/spark/src/function/url/url_decode.rs | 261 ++ datafusion/spark/src/function/url/url_encode.rs | 131 + datafusion/spark/src/lib.rs | 25 +- .../types.rs => spark/src/planner.rs} | 42 +- datafusion/sql/Cargo.toml | 3 +- datafusion/sql/examples/sql.rs | 6 +- datafusion/sql/src/cte.rs | 27 +- datafusion/sql/src/expr/binary_op.rs | 6 +- datafusion/sql/src/expr/function.rs | 198 +- datafusion/sql/src/expr/identifier.rs | 91 +- datafusion/sql/src/expr/mod.rs | 104 +- datafusion/sql/src/expr/order_by.rs | 2 +- datafusion/sql/src/expr/subquery.rs | 77 +- datafusion/sql/src/expr/substring.rs | 10 +- datafusion/sql/src/expr/unary_op.rs | 11 +- datafusion/sql/src/expr/value.rs | 187 +- datafusion/sql/src/lib.rs | 2 + datafusion/sql/src/parser.rs | 214 +- datafusion/sql/src/planner.rs | 24 +- datafusion/sql/src/query.rs | 34 +- datafusion/sql/src/relation/join.rs | 2 +- datafusion/sql/src/relation/mod.rs | 123 +- datafusion/sql/src/resolve.rs | 163 +- datafusion/sql/src/select.rs | 208 +- datafusion/sql/src/set_expr.rs | 2 +- datafusion/sql/src/statement.rs | 415 ++- datafusion/sql/src/unparser/ast.rs | 66 +- datafusion/sql/src/unparser/dialect.rs | 49 +- datafusion/sql/src/unparser/expr.rs | 137 +- datafusion/sql/src/unparser/extension_unparser.rs | 4 +- datafusion/sql/src/unparser/plan.rs | 204 +- datafusion/sql/src/unparser/rewrite.rs | 13 +- datafusion/sql/src/unparser/utils.rs | 70 +- datafusion/sql/src/utils.rs | 101 +- datafusion/sql/src/values.rs | 8 +- datafusion/sql/tests/cases/collection.rs | 10 +- datafusion/sql/tests/cases/diagnostic.rs | 3 +- datafusion/sql/tests/cases/params.rs | 87 +- datafusion/sql/tests/cases/plan_to_sql.rs | 284 +- datafusion/sql/tests/common/mod.rs | 14 +- datafusion/sql/tests/sql_integration.rs | 2346 ++++++------- datafusion/sqllogictest/Cargo.toml | 15 +- datafusion/sqllogictest/README.md | 11 + datafusion/sqllogictest/bin/postgres_container.rs | 14 +- datafusion/sqllogictest/bin/sqllogictests.rs | 121 +- datafusion/sqllogictest/src/engines/conversion.rs | 11 +- .../src/engines/currently_executed_sql.rs | 85 + .../src/engines/datafusion_engine/runner.rs | 27 +- .../runner.rs | 23 +- datafusion/sqllogictest/src/engines/mod.rs | 7 +- .../src/engines/postgres_engine/mod.rs | 166 +- datafusion/sqllogictest/src/filters.rs | 19 +- datafusion/sqllogictest/src/lib.rs | 5 +- datafusion/sqllogictest/src/test_context.rs | 133 +- datafusion/sqllogictest/src/util.rs | 51 +- .../test_files/agg_func_substitute.slt | 27 +- datafusion/sqllogictest/test_files/aggregate.slt | 958 ++++- .../test_files/aggregate_repartition.slt | 20 +- .../test_files/aggregate_skip_partial.slt | 17 +- .../sqllogictest/test_files/aggregates_topk.slt | 262 +- datafusion/sqllogictest/test_files/array.slt | 209 +- datafusion/sqllogictest/test_files/arrow_files.slt | 260 ++ .../sqllogictest/test_files/arrow_typeof.slt | 31 +- datafusion/sqllogictest/test_files/async_udf.slt | 30 +- datafusion/sqllogictest/test_files/binary.slt | 10 + datafusion/sqllogictest/test_files/case.slt | 169 +- datafusion/sqllogictest/test_files/cast.slt | 36 + datafusion/sqllogictest/test_files/coalesce.slt | 6 +- .../sqllogictest/test_files/count_star_rule.slt | 23 +- .../test_files/create_external_table.slt | 2 +- datafusion/sqllogictest/test_files/cte.slt | 263 +- .../test_files/datetime/arith_date_date.slt | 15 + .../test_files/datetime/arith_date_integer.slt | 89 + .../test_files/datetime/arith_date_interval.slt | 37 + .../test_files/datetime/arith_date_time.slt | 116 + .../test_files/datetime/arith_interval_double.slt | 41 + .../datetime/arith_interval_interval.slt | 27 + .../test_files/datetime/arith_negate_interval.slt | 13 + .../test_files/datetime/arith_time_interval.slt | 70 + .../test_files/datetime/arith_time_time.slt | 47 + .../datetime/arith_timestamp_duration.slt | 147 + .../datetime/arith_timestamp_interval.slt | 36 + .../datetime/arith_timestamp_timestamp.slt | 13 + .../{ => datetime}/current_date_timezone.slt | 0 .../{ => datetime}/current_time_timezone.slt | 0 .../test_files/{expr => datetime}/date_part.slt | 132 +- .../test_files/{ => datetime}/dates.slt | 116 +- .../test_files/{ => datetime}/interval.slt | 0 .../test_files/{ => datetime}/interval_mysql.slt | 0 .../test_files/{ => datetime}/timestamps.slt | 2831 +++++++++++---- datafusion/sqllogictest/test_files/decimal.slt | 324 +- datafusion/sqllogictest/test_files/delete.slt | 16 +- datafusion/sqllogictest/test_files/dictionary.slt | 19 +- datafusion/sqllogictest/test_files/distinct_on.slt | 9 +- datafusion/sqllogictest/test_files/dml_delete.slt | 202 ++ datafusion/sqllogictest/test_files/dml_update.slt | 286 ++ .../test_files/dynamic_filter_pushdown_config.slt | 176 +- datafusion/sqllogictest/test_files/encoding.slt | 144 +- datafusion/sqllogictest/test_files/errors.slt | 9 +- datafusion/sqllogictest/test_files/explain.slt | 73 +- .../sqllogictest/test_files/explain_analyze.slt | 68 + .../sqllogictest/test_files/explain_tree.slt | 1207 +++---- datafusion/sqllogictest/test_files/expr.slt | 216 +- .../test_files/filter_without_sort_exec.slt | 42 +- datafusion/sqllogictest/test_files/functions.slt | 19 +- datafusion/sqllogictest/test_files/group_by.slt | 228 +- datafusion/sqllogictest/test_files/grouping.slt | 12 + .../test_files/grouping_set_repartition.slt | 246 ++ .../sqllogictest/test_files/information_schema.slt | 91 +- datafusion/sqllogictest/test_files/insert.slt | 18 +- .../sqllogictest/test_files/insert_to_external.slt | 12 +- datafusion/sqllogictest/test_files/join.slt.part | 104 +- .../test_files/join_disable_repartition_joins.slt | 21 +- .../test_files/join_is_not_distinct_from.slt | 94 +- datafusion/sqllogictest/test_files/joins.slt | 929 ++--- datafusion/sqllogictest/test_files/limit.slt | 26 +- .../sqllogictest/test_files/limit_pruning.slt | 81 + .../test_files/limit_single_row_batches.slt | 22 + .../test_files/listing_table_statistics.slt | 2 +- datafusion/sqllogictest/test_files/map.slt | 49 +- datafusion/sqllogictest/test_files/math.slt | 103 +- datafusion/sqllogictest/test_files/metadata.slt | 93 + .../test_files/monotonic_projection_test.slt | 18 +- .../sqllogictest/test_files/named_arguments.slt | 5 +- .../test_files/null_aware_anti_join.slt | 453 +++ datafusion/sqllogictest/test_files/operator.slt | 40 +- datafusion/sqllogictest/test_files/options.slt | 11 +- datafusion/sqllogictest/test_files/order.slt | 98 +- datafusion/sqllogictest/test_files/parquet.slt | 32 +- .../test_files/parquet_filter_pushdown.slt | 153 +- .../test_files/parquet_sorted_statistics.slt | 3 +- .../sqllogictest/test_files/parquet_statistics.slt | 21 +- datafusion/sqllogictest/test_files/predicates.slt | 62 +- datafusion/sqllogictest/test_files/prepare.slt | 22 +- .../test_files/preserve_file_partitioning.slt | 596 ++++ datafusion/sqllogictest/test_files/projection.slt | 7 +- .../test_files/projection_pushdown.slt | 1054 ++++++ .../sqllogictest/test_files/push_down_filter.slt | 148 +- datafusion/sqllogictest/test_files/pwmj.slt | 46 +- datafusion/sqllogictest/test_files/qualify.slt | 68 +- .../sqllogictest/test_files/regexp/regexp_like.slt | 5 +- datafusion/sqllogictest/test_files/repartition.slt | 22 +- .../sqllogictest/test_files/repartition_scan.slt | 34 +- .../test_files/repartition_subset_satisfaction.slt | 526 +++ .../sqllogictest/test_files/run_end_encoded.slt | 57 + datafusion/sqllogictest/test_files/scalar.slt | 152 +- .../sqllogictest/test_files/schema_evolution.slt | 144 + datafusion/sqllogictest/test_files/select.slt | 61 +- .../sqllogictest/test_files/set_variable.slt | 203 ++ .../sqllogictest/test_files/simplify_expr.slt | 37 +- .../{spark/math/hex.slt => slt_features.slt} | 64 +- .../sqllogictest/test_files/sort_merge_join.slt | 51 +- .../sqllogictest/test_files/sort_pushdown.slt | 886 +++++ .../test_files/spark/aggregate/collect.slt | 93 + .../test_files/spark/aggregate/try_sum.slt | 140 + .../test_files/spark/array/array_repeat.slt | 77 +- .../sqllogictest/test_files/spark/array/slice.slt | 101 + .../test_files/spark/bitwise/bit_count.slt | 15 + .../test_files/spark/bitwise/bit_get.slt | 25 + .../test_files/spark/bitwise/shiftright.slt | 115 + .../spark/bitwise/shiftrightunsigned.slt | 103 + .../test_files/spark/collection/size.slt | 132 + .../test_files/spark/datetime/add_months.slt | 52 +- .../test_files/spark/datetime/date_add.slt | 19 +- .../test_files/spark/datetime/date_diff.slt | 150 +- .../test_files/spark/datetime/date_part.slt | 304 +- .../test_files/spark/datetime/date_sub.slt | 10 + .../test_files/spark/datetime/date_trunc.slt | 177 +- .../test_files/spark/datetime/datediff.slt | 32 - .../test_files/spark/datetime/datepart.slt | 62 - .../test_files/spark/datetime/hour.slt | 23 +- .../test_files/spark/datetime/make_dt_interval.slt | 3 +- .../test_files/spark/datetime/minute.slt | 23 +- .../test_files/spark/datetime/second.slt | 23 +- .../test_files/spark/datetime/time_trunc.slt | 74 + .../test_files/spark/datetime/trunc.slt | 101 +- .../sqllogictest/test_files/spark/hash/crc32.slt | 31 +- .../sqllogictest/test_files/spark/hash/sha1.slt | 25 +- .../sqllogictest/test_files/spark/math/abs.slt | 69 +- .../sqllogictest/test_files/spark/math/csc.slt | 23 +- .../sqllogictest/test_files/spark/math/expm1.slt | 5 + .../sqllogictest/test_files/spark/math/hex.slt | 35 + .../sqllogictest/test_files/spark/math/sec.slt | 22 +- .../test_files/spark/math/shiftleft.slt | 100 + .../sqllogictest/test_files/spark/math/unhex.slt | 98 + .../test_files/spark/string/format_string.slt | 32 +- .../spark/{math/expm1.slt => string/space.slt} | 29 +- .../test_files/spark/string/substr.slt | 37 - .../test_files/spark/string/substring.slt | 206 +- .../spark/{math/hex.slt => url/try_url_decode.slt} | 51 +- .../test_files/spark/url/url_decode.slt | 60 +- .../test_files/spark/url/url_encode.slt | 24 +- .../test_files/string/string_literal.slt | 43 +- .../test_files/string/string_query.slt.part | 10 +- .../sqllogictest/test_files/string/string_view.slt | 2 +- datafusion/sqllogictest/test_files/struct.slt | 1096 +++++- datafusion/sqllogictest/test_files/subquery.slt | 164 +- .../sqllogictest/test_files/subquery_sort.slt | 9 +- .../sqllogictest/test_files/table_functions.slt | 24 + .../test_files/to_timestamp_timezone.slt | 204 ++ datafusion/sqllogictest/test_files/topk.slt | 9 +- .../sqllogictest/test_files/tpch/plans/q1.slt.part | 12 +- .../test_files/tpch/plans/q10.slt.part | 48 +- .../test_files/tpch/plans/q11.slt.part | 73 +- .../test_files/tpch/plans/q12.slt.part | 21 +- .../test_files/tpch/plans/q13.slt.part | 26 +- .../test_files/tpch/plans/q14.slt.part | 17 +- .../test_files/tpch/plans/q15.slt.part | 46 +- .../test_files/tpch/plans/q16.slt.part | 45 +- .../test_files/tpch/plans/q17.slt.part | 32 +- .../test_files/tpch/plans/q18.slt.part | 45 +- .../test_files/tpch/plans/q19.slt.part | 21 +- .../sqllogictest/test_files/tpch/plans/q2.slt.part | 120 +- .../test_files/tpch/plans/q20.slt.part | 66 +- .../test_files/tpch/plans/q21.slt.part | 74 +- .../test_files/tpch/plans/q22.slt.part | 46 +- .../sqllogictest/test_files/tpch/plans/q3.slt.part | 40 +- .../sqllogictest/test_files/tpch/plans/q4.slt.part | 24 +- .../sqllogictest/test_files/tpch/plans/q5.slt.part | 73 +- .../sqllogictest/test_files/tpch/plans/q6.slt.part | 5 +- .../sqllogictest/test_files/tpch/plans/q7.slt.part | 79 +- .../sqllogictest/test_files/tpch/plans/q8.slt.part | 103 +- .../sqllogictest/test_files/tpch/plans/q9.slt.part | 71 +- datafusion/sqllogictest/test_files/truncate.slt | 85 + datafusion/sqllogictest/test_files/union.slt | 163 +- datafusion/sqllogictest/test_files/unnest.slt | 64 +- datafusion/sqllogictest/test_files/update.slt | 20 +- datafusion/sqllogictest/test_files/window.slt | 313 +- .../sqllogictest/test_files/window_limits.slt | 12 +- datafusion/substrait/Cargo.toml | 7 +- datafusion/substrait/src/extensions.rs | 10 +- datafusion/substrait/src/lib.rs | 2 + .../consumer/expr/aggregate_function.rs | 6 +- .../src/logical_plan/consumer/expr/cast.rs | 4 +- .../logical_plan/consumer/expr/field_reference.rs | 4 +- .../consumer/expr/function_arguments.rs | 4 +- .../src/logical_plan/consumer/expr/literal.rs | 50 +- .../src/logical_plan/consumer/expr/mod.rs | 31 +- .../logical_plan/consumer/expr/scalar_function.rs | 133 +- .../logical_plan/consumer/expr/singular_or_list.rs | 4 +- .../src/logical_plan/consumer/expr/subquery.rs | 62 +- .../logical_plan/consumer/expr/window_function.rs | 10 +- .../substrait/src/logical_plan/consumer/plan.rs | 67 +- .../src/logical_plan/consumer/rel/aggregate_rel.rs | 8 +- .../src/logical_plan/consumer/rel/exchange_rel.rs | 4 +- .../src/logical_plan/consumer/rel/fetch_rel.rs | 8 +- .../src/logical_plan/consumer/rel/join_rel.rs | 6 +- .../substrait/src/logical_plan/consumer/rel/mod.rs | 6 +- .../src/logical_plan/consumer/rel/project_rel.rs | 4 +- .../src/logical_plan/consumer/rel/read_rel.rs | 124 +- .../src/logical_plan/consumer/rel/set_rel.rs | 5 +- .../src/logical_plan/consumer/rel/sort_rel.rs | 2 +- .../logical_plan/consumer/substrait_consumer.rs | 12 +- .../substrait/src/logical_plan/consumer/types.rs | 41 +- .../substrait/src/logical_plan/consumer/utils.rs | 37 +- .../producer/expr/aggregate_function.rs | 2 +- .../src/logical_plan/producer/expr/cast.rs | 6 +- .../logical_plan/producer/expr/field_reference.rs | 6 +- .../src/logical_plan/producer/expr/if_then.rs | 2 +- .../src/logical_plan/producer/expr/literal.rs | 8 +- .../src/logical_plan/producer/expr/mod.rs | 9 +- .../logical_plan/producer/expr/scalar_function.rs | 14 +- .../logical_plan/producer/expr/singular_or_list.rs | 2 +- .../src/logical_plan/producer/expr/subquery.rs | 58 +- .../logical_plan/producer/expr/window_function.rs | 10 +- .../substrait/src/logical_plan/producer/plan.rs | 6 +- .../src/logical_plan/producer/rel/aggregate_rel.rs | 40 +- .../src/logical_plan/producer/rel/exchange_rel.rs | 6 +- .../src/logical_plan/producer/rel/fetch_rel.rs | 2 +- .../src/logical_plan/producer/rel/join.rs | 6 +- .../src/logical_plan/producer/rel/project_rel.rs | 4 +- .../src/logical_plan/producer/rel/read_rel.rs | 169 +- .../src/logical_plan/producer/rel/set_rel.rs | 2 +- .../src/logical_plan/producer/rel/sort_rel.rs | 4 +- .../logical_plan/producer/substrait_producer.rs | 41 +- .../substrait/src/logical_plan/producer/types.rs | 10 +- .../substrait/src/logical_plan/producer/utils.rs | 4 +- datafusion/substrait/src/physical_plan/consumer.rs | 42 +- datafusion/substrait/src/physical_plan/producer.rs | 157 +- datafusion/substrait/src/serializer.rs | 1 + .../substrait/tests/cases/aggregation_tests.rs | 16 +- .../substrait/tests/cases/consumer_integration.rs | 444 +-- .../substrait/tests/cases/emit_kind_tests.rs | 44 +- datafusion/substrait/tests/cases/function_test.rs | 8 +- datafusion/substrait/tests/cases/logical_plans.rs | 65 +- .../tests/cases/roundtrip_logical_plan.rs | 390 ++- .../tests/cases/roundtrip_physical_plan.rs | 36 +- datafusion/substrait/tests/cases/serialize.rs | 20 +- .../substrait/tests/cases/substrait_validations.rs | 24 +- datafusion/substrait/tests/utils.rs | 23 +- datafusion/wasmtest/Cargo.toml | 6 +- .../wasmtest/datafusion-wasm-app/package-lock.json | 102 +- datafusion/wasmtest/src/lib.rs | 71 +- dev/changelog/51.0.0.md | 717 ++++ dev/changelog/52.0.0.md | 745 ++++ dev/depcheck/Cargo.toml | 3 +- .../depcheck/rust-toolchain.toml | 2 +- dev/depcheck/src/main.rs | 2 +- dev/release/README.md | 340 +- dev/release/add-branch-protection.sh | 160 + dev/rust_lint.sh | 123 +- dev/update_function_docs.sh | 30 + docs/.gitignore | 4 + docs/README.md | 5 + docs/build.sh | 9 +- docs/requirements.txt | 8 +- docs/scripts/generate_dependency_graph.sh | 97 + docs/source/_static/theme_overrides.css | 18 + docs/source/contributor-guide/architecture.md | 2 +- .../architecture/dependency-graph.md | 180 + docs/source/contributor-guide/communication.md | 68 +- .../contributor-guide/development_environment.md | 2 +- docs/source/contributor-guide/governance.md | 5 +- docs/source/contributor-guide/howtos.md | 8 +- docs/source/contributor-guide/index.md | 8 +- docs/source/contributor-guide/inviting.md | 2 +- docs/source/contributor-guide/testing.md | 32 +- docs/source/download.md | 26 +- docs/source/index.rst | 5 +- docs/source/library-user-guide/catalogs.md | 2 +- .../library-user-guide/custom-table-providers.md | 2 +- .../library-user-guide/extending-operators.md | 7 +- docs/source/library-user-guide/extending-sql.md | 344 ++ .../library-user-guide/functions/adding-udfs.md | 32 +- docs/source/library-user-guide/query-optimizer.md | 32 +- docs/source/library-user-guide/upgrading.md | 792 ++++- .../library-user-guide/using-the-dataframe-api.md | 2 +- .../library-user-guide/working-with-exprs.md | 8 +- docs/source/user-guide/cli/functions.md | 82 + docs/source/user-guide/concepts-readings-events.md | 28 + docs/source/user-guide/configs.md | 248 +- docs/source/user-guide/crate-configuration.md | 30 + docs/source/user-guide/explain-usage.md | 1 + docs/source/user-guide/expressions.md | 6 +- docs/source/user-guide/introduction.md | 8 +- docs/source/user-guide/metrics.md | 11 +- docs/source/user-guide/sql/aggregate_functions.md | 30 + docs/source/user-guide/sql/data_types.md | 5 + docs/source/user-guide/sql/ddl.md | 2 +- docs/source/user-guide/sql/explain.md | 18 +- docs/source/user-guide/sql/format_options.md | 102 +- docs/source/user-guide/sql/index.rst | 1 + docs/source/user-guide/sql/scalar_functions.md | 370 +- docs/source/user-guide/sql/struct_coercion.md | 354 ++ rust-toolchain.toml | 2 +- rustfmt.toml | 2 +- test-utils/src/array_gen/binary.rs | 2 +- test-utils/src/array_gen/boolean.rs | 2 +- test-utils/src/array_gen/decimal.rs | 2 +- test-utils/src/array_gen/primitive.rs | 4 +- test-utils/src/array_gen/random_data.rs | 16 +- test-utils/src/array_gen/string.rs | 2 +- test-utils/src/string_gen.rs | 2 +- test-utils/src/tpcds.rs | 2 +- typos.toml | 6 +- 1547 files changed, 131273 insertions(+), 46310 deletions(-) diff --cc .github/workflows/rust.yml index cb1cb054be,cd0a991126..1afc3fc0b8 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@@ -379,11 -315,10 +379,11 @@@ jobs # datafusion-cli tests linux-test-datafusion-cli: name: cargo test datafusion-cli (amd64) - needs: linux-build-lib + needs: [linux-build-lib, check-files] + if: needs.check-files.outputs.should_skip != 'true' runs-on: ubuntu-latest steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 with: submodules: true fetch-depth: 1 @@@ -643,35 -559,8 +630,10 @@@ shell: bash run: cargo test --profile ci --exclude datafusion-cli --workspace --lib --tests --bins --features avro,json,backtrace,integration-tests - test-datafusion-pyarrow: - name: cargo test pyarrow (amd64) - needs: [linux-build-lib, check-files] - if: needs.check-files.outputs.should_skip != 'true' - runs-on: ubuntu-latest - container: - image: amd64/rust:bullseye # Use the bullseye tag image which comes with python3.9 - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - submodules: true - fetch-depth: 1 - - name: Install PyArrow - run: | - echo "LIBRARY_PATH=$LD_LIBRARY_PATH" >> $GITHUB_ENV - apt-get update - apt-get install python3-pip -y - python3 -m pip install pyarrow - - name: Setup Rust toolchain - uses: ./.github/actions/setup-builder - with: - rust-version: stable - - name: Run datafusion-common tests - run: cargo test --profile ci -p datafusion-common --features=pyarrow,sql - vendor: name: Verify Vendored Code + needs: check-files + if: needs.check-files.outputs.should_skip != 'true' runs-on: ubuntu-latest container: image: amd64/rust @@@ -825,6 -709,23 +787,24 @@@ ./dev/update_function_docs.sh git diff --exit-code + examples-docs-check: + name: check example README is up-to-date - needs: linux-build-lib ++ needs: [linux-build-lib, check-files] ++ if: needs.check-files.outputs.should_skip != 'true' + runs-on: ubuntu-latest + container: + image: amd64/rust + + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + submodules: true + fetch-depth: 1 + + - name: Run examples docs check script + run: | + bash ci/scripts/check_examples_docs.sh + # Verify MSRV for the crates which are directly used by other projects: # - datafusion # - datafusion-substrait @@@ -879,46 -778,4 +859,36 @@@ run: cargo msrv --output-format json --log-target stdout verify - name: Check datafusion-proto working-directory: datafusion/proto - run: cargo msrv --output-format json --log-target stdout verify + run: cargo msrv --output-format json --log-target stdout verify - typos: - name: Spell Check with Typos - needs: [check-files] - if: needs.check-files.outputs.should_skip != 'true' - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - with: - persist-credentials: false - - uses: crate-ci/typos@07d900b8fa1097806b8adb6391b0d3e0ac2fdea7 # v1.39.0 + + rust-required-checks: + name: Validate Rust Required Checks + needs: [ linux-rustdoc, + linux-wasm-pack, + linux-build-lib, + macos-aarch64, + vendor, + check-fmt, + msrv, + linux-datafusion-common-features, + linux-datafusion-substrait-features, + linux-datafusion-proto-features, + linux-cargo-check-datafusion, + linux-cargo-check-datafusion-functions, + linux-test, + linux-test-example, + linux-test-doc, + verify-benchmark-results, + sqllogictest-postgres, + sqllogictest-substrait, - test-datafusion-pyarrow, + clippy, + cargo-toml-formatting-checks, + linux-test-datafusion-cli, - config-docs-check ++ config-docs-check, ++ examples-docs-check + ] + if: always() + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0 - - run: echo '${{ toJSON(needs) }}' | jq -e 'all(.[]; .result == "success" or .result == "skipped")' ++ - run: echo '${{ toJSON(needs) }}' | jq -e 'all(.[]; .result == "success" or .result == "skipped")' --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
