This is an automated email from the ASF dual-hosted git repository. goldmedal pushed a commit to branch feature/13793-unnest-unparse in repository https://gitbox.apache.org/repos/asf/datafusion.git
commit 91ec33836ac7d4335718fe34207674a6c12460a2 Merge: 5d5e55b452 ede665b212 Author: Jax Liu <[email protected]> AuthorDate: Sat Dec 21 16:31:48 2024 +0800 Merge branch 'main' into feature/13793-unnest-unparse .github/workflows/rust.yml | 36 +- Cargo.toml | 2 +- datafusion-cli/Cargo.lock | 74 +-- datafusion-cli/Cargo.toml | 2 + datafusion-cli/src/exec.rs | 14 +- datafusion-cli/src/object_storage.rs | 9 +- datafusion-examples/README.md | 1 + datafusion-examples/examples/custom_datasource.rs | 8 +- datafusion-examples/examples/remote_catalog.rs | 369 +++++++++++++++ datafusion/catalog/src/catalog.rs | 14 +- datafusion/common/Cargo.toml | 1 + datafusion/common/src/config.rs | 83 +++- datafusion/common/src/cse.rs | 150 ++++-- datafusion/common/src/join_type.rs | 25 +- datafusion/common/src/scalar/mod.rs | 221 ++++++++- datafusion/common/src/utils/mod.rs | 5 +- datafusion/core/Cargo.toml | 3 +- datafusion/core/benches/csv_load.rs | 10 + .../core/src/catalog_common/information_schema.rs | 67 ++- datafusion/core/src/dataframe/mod.rs | 24 + datafusion/core/src/datasource/file_format/csv.rs | 119 +++-- datafusion/core/src/datasource/file_format/json.rs | 8 +- .../core/src/datasource/file_format/options.rs | 12 +- .../core/src/datasource/file_format/parquet.rs | 57 ++- datafusion/core/src/datasource/listing/helpers.rs | 112 ++++- datafusion/core/src/datasource/listing/table.rs | 9 +- .../src/datasource/physical_plan/arrow_file.rs | 6 +- .../core/src/datasource/physical_plan/avro.rs | 8 +- .../core/src/datasource/physical_plan/csv.rs | 8 +- .../core/src/datasource/physical_plan/json.rs | 8 +- .../src/datasource/physical_plan/parquet/mod.rs | 13 +- .../physical_plan/parquet/row_group_filter.rs | 153 +++--- datafusion/core/src/datasource/stream.rs | 2 +- datafusion/core/src/execution/session_state.rs | 2 +- .../src/physical_optimizer/enforce_distribution.rs | 15 +- .../core/src/physical_optimizer/enforce_sorting.rs | 2 +- .../core/src/physical_optimizer/join_selection.rs | 28 +- .../replace_with_order_preserving_variants.rs | 6 +- .../core/src/physical_optimizer/sanity_checker.rs | 19 +- datafusion/core/src/physical_planner.rs | 11 +- datafusion/core/src/test/mod.rs | 46 +- datafusion/core/src/test_util/mod.rs | 23 +- datafusion/core/tests/config_from_env.rs | 17 +- datafusion/core/tests/custom_sources_cases/mod.rs | 12 +- .../provider_filter_pushdown.rs | 13 +- .../core/tests/custom_sources_cases/statistics.rs | 14 +- .../data/csv/aggregate_test_100_with_nulls.csv | 101 ++++ datafusion/core/tests/dataframe/mod.rs | 43 ++ datafusion/core/tests/fuzz_cases/window_fuzz.rs | 8 +- datafusion/core/tests/parquet/mod.rs | 68 ++- datafusion/core/tests/parquet/row_group_pruning.rs | 212 +++++++++ .../core/tests/user_defined/insert_operation.rs | 18 +- .../core/tests/user_defined/user_defined_plan.rs | 19 +- .../user_defined/user_defined_scalar_functions.rs | 10 +- datafusion/execution/src/runtime_env.rs | 3 +- datafusion/expr-common/src/signature.rs | 107 +++-- datafusion/expr-common/src/sort_properties.rs | 21 +- datafusion/expr-common/src/type_coercion/binary.rs | 22 +- datafusion/expr/src/expr.rs | 391 +++++++++++++++- datafusion/expr/src/expr_fn.rs | 8 +- datafusion/expr/src/logical_plan/plan.rs | 20 + datafusion/expr/src/type_coercion/functions.rs | 23 +- datafusion/expr/src/udf.rs | 36 +- datafusion/ffi/src/execution_plan.rs | 11 +- datafusion/ffi/src/plan_properties.rs | 107 ++++- datafusion/functions-aggregate/src/count.rs | 2 +- datafusion/functions-nested/src/distance.rs | 7 +- datafusion/functions-nested/src/extract.rs | 83 ++++ datafusion/functions-nested/src/flatten.rs | 11 +- datafusion/functions-nested/src/length.rs | 7 +- datafusion/functions-nested/src/make_array.rs | 2 +- datafusion/functions-nested/src/set_ops.rs | 3 + datafusion/functions-nested/src/string.rs | 9 +- datafusion/functions-nested/src/utils.rs | 19 +- datafusion/functions/src/core/greatest.rs | 183 ++------ .../functions/src/core/greatest_least_utils.rs | 133 ++++++ datafusion/functions/src/core/least.rs | 200 ++++++++ datafusion/functions/src/core/mod.rs | 8 + datafusion/functions/src/macros.rs | 21 +- datafusion/functions/src/math/abs.rs | 8 +- datafusion/functions/src/math/factorial.rs | 6 +- datafusion/functions/src/math/gcd.rs | 8 +- datafusion/functions/src/math/lcm.rs | 8 +- datafusion/functions/src/math/power.rs | 8 +- datafusion/functions/src/string/concat.rs | 5 + datafusion/functions/src/strings.rs | 37 +- .../optimizer/src/analyzer/inline_table_scan.rs | 8 +- datafusion/optimizer/src/analyzer/mod.rs | 11 +- .../optimizer/src/common_subexpr_eliminate.rs | 263 ++++++++++- datafusion/physical-expr/Cargo.toml | 1 + datafusion/physical-expr/src/equivalence/class.rs | 211 ++++++++- .../physical-expr/src/equivalence/properties.rs | 521 +++++++++++++++++++-- datafusion/physical-expr/src/expressions/binary.rs | 8 + .../physical-expr/src/expressions/literal.rs | 1 + .../physical-expr/src/expressions/negative.rs | 1 + datafusion/physical-expr/src/scalar_function.rs | 2 + .../physical-optimizer/src/output_requirements.rs | 3 +- datafusion/physical-optimizer/src/pruning.rs | 325 +++++++------ datafusion/physical-plan/src/aggregates/mod.rs | 36 +- .../physical-plan/src/aggregates/row_hash.rs | 46 +- datafusion/physical-plan/src/analyze.rs | 10 +- datafusion/physical-plan/src/coalesce_batches.rs | 3 +- .../physical-plan/src/coalesce_partitions.rs | 3 +- datafusion/physical-plan/src/empty.rs | 20 +- datafusion/physical-plan/src/execution_plan.rs | 253 ++++++---- datafusion/physical-plan/src/explain.rs | 9 +- datafusion/physical-plan/src/filter.rs | 4 +- datafusion/physical-plan/src/insert.rs | 8 +- datafusion/physical-plan/src/joins/cross_join.rs | 20 +- datafusion/physical-plan/src/joins/hash_join.rs | 56 ++- .../physical-plan/src/joins/nested_loop_join.rs | 37 +- .../physical-plan/src/joins/sort_merge_join.rs | 53 ++- .../physical-plan/src/joins/symmetric_hash_join.rs | 12 +- datafusion/physical-plan/src/lib.rs | 3 +- datafusion/physical-plan/src/limit.rs | 12 +- datafusion/physical-plan/src/memory.rs | 16 +- datafusion/physical-plan/src/placeholder_row.rs | 17 +- datafusion/physical-plan/src/projection.rs | 3 +- datafusion/physical-plan/src/recursive_query.rs | 6 +- datafusion/physical-plan/src/repartition/mod.rs | 10 +- datafusion/physical-plan/src/sorts/merge.rs | 28 +- datafusion/physical-plan/src/sorts/partial_sort.rs | 107 +++-- datafusion/physical-plan/src/sorts/sort.rs | 68 ++- .../src/sorts/sort_preserving_merge.rs | 61 ++- .../physical-plan/src/sorts/streaming_merge.rs | 4 + datafusion/physical-plan/src/streaming.rs | 23 +- datafusion/physical-plan/src/test/exec.rs | 51 +- datafusion/physical-plan/src/topk/mod.rs | 36 +- datafusion/physical-plan/src/union.rs | 19 +- datafusion/physical-plan/src/unnest.rs | 71 +-- datafusion/physical-plan/src/values.rs | 13 +- .../src/windows/bounded_window_agg_exec.rs | 31 +- datafusion/physical-plan/src/windows/mod.rs | 2 +- .../physical-plan/src/windows/window_agg_exec.rs | 51 +- datafusion/physical-plan/src/work_table.rs | 10 +- .../proto-common/proto/datafusion_common.proto | 9 +- datafusion/proto-common/src/from_proto/mod.rs | 2 + datafusion/proto-common/src/generated/pbjson.rs | 18 + datafusion/proto-common/src/generated/prost.rs | 98 ++-- datafusion/proto-common/src/to_proto/mod.rs | 1 + datafusion/proto/proto/datafusion.proto | 1 + .../proto/src/generated/datafusion_proto_common.rs | 11 +- datafusion/proto/src/generated/pbjson.rs | 17 + datafusion/proto/src/generated/prost.rs | 2 + datafusion/proto/src/logical_plan/file_formats.rs | 6 + datafusion/proto/src/logical_plan/from_proto.rs | 2 +- datafusion/proto/src/physical_plan/from_proto.rs | 15 +- datafusion/proto/src/physical_plan/to_proto.rs | 1 + .../proto/tests/cases/roundtrip_logical_plan.rs | 12 +- .../proto/tests/cases/roundtrip_physical_plan.rs | 43 +- datafusion/sql/Cargo.toml | 1 + datafusion/sql/src/expr/function.rs | 24 +- datafusion/sql/src/expr/mod.rs | 8 +- datafusion/sql/src/expr/value.rs | 194 ++++++-- datafusion/sql/src/parser.rs | 14 +- datafusion/sql/src/planner.rs | 49 +- datafusion/sql/src/select.rs | 1 + datafusion/sql/src/statement.rs | 294 ++++++++++-- datafusion/sql/src/unparser/ast.rs | 3 + datafusion/sql/src/unparser/dialect.rs | 6 +- datafusion/sql/src/unparser/expr.rs | 32 +- datafusion/sql/src/unparser/plan.rs | 9 +- datafusion/sql/src/unparser/utils.rs | 12 +- datafusion/sql/tests/sql_integration.rs | 69 +-- datafusion/sqllogictest/test_files/aggregate.slt | 57 ++- datafusion/sqllogictest/test_files/array.slt | 16 +- .../test_files/create_external_table.slt | 14 + datafusion/sqllogictest/test_files/functions.slt | 198 ++++++++ .../sqllogictest/test_files/information_schema.slt | 180 ++++--- datafusion/sqllogictest/test_files/options.slt | 96 +++- .../test_files/parquet_filter_pushdown.slt | 8 +- .../sqllogictest/test_files/push_down_filter.slt | 73 +++ .../sqllogictest/test_files/repartition_scan.slt | 8 +- .../sqllogictest/test_files/set_variable.slt | 8 +- datafusion/sqllogictest/test_files/window.slt | 2 +- datafusion/substrait/Cargo.toml | 3 +- datafusion/substrait/src/logical_plan/consumer.rs | 31 +- datafusion/substrait/src/logical_plan/producer.rs | 39 +- .../tests/cases/roundtrip_logical_plan.rs | 9 +- docs/source/contributor-guide/index.md | 38 +- docs/source/user-guide/configs.md | 2 +- docs/source/user-guide/sql/scalar_functions.md | 24 + 182 files changed, 6431 insertions(+), 1733 deletions(-) --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
