This is an automated email from the ASF dual-hosted git repository.
ozankabak pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/datafusion.git
from 8ebed674dd Script and documentation for regenerating sqlite test files
(#14290)
add 5e1e693b6e Introduce unified `DataSourceExec` for provided
datasources, remove `ParquetExec`, `CsvExec`, etc (#14224)
No new revisions were added by this update.
Summary of changes:
datafusion-cli/src/functions.rs | 20 +-
.../examples/advanced_parquet_index.rs | 92 +--
datafusion-examples/examples/csv_json_opener.rs | 61 +-
datafusion-examples/examples/custom_file_format.rs | 5 +
.../examples/parquet_exec_visitor.rs | 28 +-
datafusion-examples/examples/parquet_index.rs | 21 +-
datafusion-examples/examples/remote_catalog.rs | 7 +-
datafusion-examples/examples/simple_udtf.rs | 6 +-
datafusion/core/benches/physical_plan.rs | 6 +-
datafusion/core/benches/sort.rs | 21 +-
datafusion/core/src/datasource/data_source.rs | 65 ++
.../core/src/datasource/file_format/arrow.rs | 15 +-
datafusion/core/src/datasource/file_format/avro.rs | 31 +-
datafusion/core/src/datasource/file_format/csv.rs | 39 +-
datafusion/core/src/datasource/file_format/json.rs | 20 +-
datafusion/core/src/datasource/file_format/mod.rs | 18 +-
.../core/src/datasource/file_format/parquet.rs | 35 +-
datafusion/core/src/datasource/listing/table.rs | 20 +-
datafusion/core/src/datasource/memory.rs | 19 +-
datafusion/core/src/datasource/mod.rs | 1 +
.../src/datasource/physical_plan/arrow_file.rs | 180 +++--
.../core/src/datasource/physical_plan/avro.rs | 276 ++++---
.../core/src/datasource/physical_plan/csv.rs | 601 +++++++---------
.../datasource/physical_plan/file_scan_config.rs | 256 ++++++-
.../src/datasource/physical_plan/file_stream.rs | 27 +-
.../core/src/datasource/physical_plan/json.rs | 251 ++++---
.../core/src/datasource/physical_plan/mod.rs | 23 +-
.../physical_plan/parquet/access_plan.rs | 2 +-
.../src/datasource/physical_plan/parquet/mod.rs | 790 ++++++---------------
.../src/datasource/physical_plan/parquet/opener.rs | 2 +-
.../datasource/physical_plan/parquet/row_filter.rs | 4 +-
.../src/datasource/physical_plan/parquet/source.rs | 589 +++++++++++++++
datafusion/core/src/datasource/schema_adapter.rs | 33 +-
datafusion/core/src/datasource/view.rs | 5 +-
datafusion/core/src/lib.rs | 64 +-
datafusion/core/src/physical_planner.rs | 13 +-
datafusion/core/src/test/mod.rs | 30 +-
datafusion/core/src/test_util/parquet.rs | 70 +-
datafusion/core/tests/dataframe/mod.rs | 8 +-
datafusion/core/tests/fuzz_cases/aggregate_fuzz.rs | 32 +-
datafusion/core/tests/fuzz_cases/join_fuzz.rs | 34 +-
datafusion/core/tests/fuzz_cases/merge_fuzz.rs | 7 +-
datafusion/core/tests/fuzz_cases/pruning.rs | 39 +-
datafusion/core/tests/fuzz_cases/sort_fuzz.rs | 10 +-
.../fuzz_cases/sort_preserving_repartition_fuzz.rs | 18 +-
datafusion/core/tests/fuzz_cases/window_fuzz.rs | 42 +-
datafusion/core/tests/memory_limit/mod.rs | 52 +-
datafusion/core/tests/parquet/custom_reader.rs | 31 +-
.../core/tests/parquet/external_access_plan.rs | 35 +-
datafusion/core/tests/parquet/file_statistics.rs | 25 +-
datafusion/core/tests/parquet/page_pruning.rs | 22 +-
datafusion/core/tests/parquet/schema_coercion.rs | 29 +-
datafusion/core/tests/parquet/utils.rs | 19 +-
.../tests/{parquet_exec.rs => parquet_config.rs} | 2 +-
.../physical_optimizer/aggregate_statistics.rs | 13 +-
.../combine_partial_final_agg.rs | 12 +-
.../physical_optimizer/enforce_distribution.rs | 430 ++++++-----
.../tests/physical_optimizer/enforce_sorting.rs | 346 +++++----
.../limited_distinct_aggregation.rs | 32 +-
.../physical_optimizer/projection_pushdown.rs | 134 ++--
.../replace_with_order_preserving_variants.rs | 77 +-
.../tests/physical_optimizer/sanity_checker.rs | 22 +-
.../core/tests/physical_optimizer/test_utils.rs | 41 +-
datafusion/core/tests/sql/explain_analyze.rs | 6 +-
datafusion/core/tests/sql/joins.rs | 12 +-
datafusion/core/tests/sql/path_partition.rs | 18 +-
.../user_defined/user_defined_table_functions.rs | 19 +-
datafusion/execution/src/memory_pool/mod.rs | 2 +-
.../physical-optimizer/src/enforce_distribution.rs | 8 +-
datafusion/physical-plan/benches/spm.rs | 6 +-
datafusion/physical-plan/src/aggregates/mod.rs | 42 +-
datafusion/physical-plan/src/display.rs | 2 +-
datafusion/physical-plan/src/execution_plan.rs | 10 +-
datafusion/physical-plan/src/joins/hash_join.rs | 78 +-
.../physical-plan/src/joins/nested_loop_join.rs | 13 +-
.../physical-plan/src/joins/sort_merge_join.rs | 14 +-
datafusion/physical-plan/src/joins/test_utils.rs | 12 +-
datafusion/physical-plan/src/lib.rs | 1 +
datafusion/physical-plan/src/memory.rs | 583 +++++++++++----
datafusion/physical-plan/src/repartition/mod.rs | 43 +-
datafusion/physical-plan/src/sorts/partial_sort.rs | 26 +-
datafusion/physical-plan/src/sorts/sort.rs | 16 +-
.../src/sorts/sort_preserving_merge.rs | 55 +-
datafusion/physical-plan/src/source.rs | 208 ++++++
datafusion/physical-plan/src/test.rs | 15 +-
datafusion/physical-plan/src/union.rs | 15 +-
datafusion/physical-plan/src/values.rs | 14 +-
datafusion/physical-plan/src/visitor.rs | 6 +-
.../src/windows/bounded_window_agg_exec.rs | 13 +-
datafusion/proto/src/physical_plan/from_proto.rs | 29 +-
datafusion/proto/src/physical_plan/mod.rs | 243 ++++---
.../proto/tests/cases/roundtrip_physical_plan.rs | 78 +-
.../src/engines/datafusion_engine/normalize.rs | 4 +-
.../test_files/agg_func_substitute.slt | 6 +-
datafusion/sqllogictest/test_files/aggregate.slt | 28 +-
.../sqllogictest/test_files/aggregates_topk.slt | 10 +-
datafusion/sqllogictest/test_files/arrow_files.slt | 4 +-
datafusion/sqllogictest/test_files/avro.slt | 2 +-
datafusion/sqllogictest/test_files/copy.slt | 4 +-
.../sqllogictest/test_files/count_star_rule.slt | 6 +-
.../test_files/create_external_table.slt | 6 +-
datafusion/sqllogictest/test_files/cse.slt | 22 +-
datafusion/sqllogictest/test_files/csv_files.slt | 2 +-
datafusion/sqllogictest/test_files/cte.slt | 42 +-
datafusion/sqllogictest/test_files/ddl.slt | 2 +-
datafusion/sqllogictest/test_files/dictionary.slt | 6 +-
datafusion/sqllogictest/test_files/distinct_on.slt | 2 +-
datafusion/sqllogictest/test_files/explain.slt | 64 +-
datafusion/sqllogictest/test_files/expr.slt | 2 +-
datafusion/sqllogictest/test_files/group_by.slt | 120 ++--
datafusion/sqllogictest/test_files/insert.slt | 8 +-
.../sqllogictest/test_files/insert_to_external.slt | 8 +-
datafusion/sqllogictest/test_files/join.slt.part | 30 +-
.../test_files/join_disable_repartition_joins.slt | 8 +-
datafusion/sqllogictest/test_files/joins.slt | 208 +++---
datafusion/sqllogictest/test_files/json.slt | 4 +-
datafusion/sqllogictest/test_files/limit.slt | 24 +-
datafusion/sqllogictest/test_files/map.slt | 2 +-
.../test_files/monotonic_projection_test.slt | 16 +-
datafusion/sqllogictest/test_files/operator.slt | 16 +-
datafusion/sqllogictest/test_files/options.slt | 6 +-
datafusion/sqllogictest/test_files/order.slt | 50 +-
datafusion/sqllogictest/test_files/parquet.slt | 20 +-
.../test_files/parquet_filter_pushdown.slt | 14 +-
.../test_files/parquet_sorted_statistics.slt | 14 +-
datafusion/sqllogictest/test_files/predicates.slt | 14 +-
datafusion/sqllogictest/test_files/projection.slt | 4 +-
datafusion/sqllogictest/test_files/references.slt | 2 +-
datafusion/sqllogictest/test_files/regexp.slt | 4 +-
datafusion/sqllogictest/test_files/repartition.slt | 4 +-
.../sqllogictest/test_files/repartition_scan.slt | 16 +-
datafusion/sqllogictest/test_files/scalar.slt | 4 +-
datafusion/sqllogictest/test_files/select.slt | 48 +-
.../sqllogictest/test_files/sort_merge_join.slt | 4 +-
datafusion/sqllogictest/test_files/struct.slt | 2 +-
datafusion/sqllogictest/test_files/subquery.slt | 20 +-
.../sqllogictest/test_files/subquery_sort.slt | 10 +-
datafusion/sqllogictest/test_files/topk.slt | 2 +-
.../sqllogictest/test_files/tpch/plans/q1.slt.part | 2 +-
.../test_files/tpch/plans/q10.slt.part | 8 +-
.../test_files/tpch/plans/q11.slt.part | 12 +-
.../test_files/tpch/plans/q12.slt.part | 4 +-
.../test_files/tpch/plans/q13.slt.part | 4 +-
.../test_files/tpch/plans/q14.slt.part | 4 +-
.../test_files/tpch/plans/q15.slt.part | 6 +-
.../test_files/tpch/plans/q16.slt.part | 6 +-
.../test_files/tpch/plans/q17.slt.part | 6 +-
.../test_files/tpch/plans/q18.slt.part | 8 +-
.../test_files/tpch/plans/q19.slt.part | 4 +-
.../sqllogictest/test_files/tpch/plans/q2.slt.part | 18 +-
.../test_files/tpch/plans/q20.slt.part | 10 +-
.../test_files/tpch/plans/q21.slt.part | 12 +-
.../test_files/tpch/plans/q22.slt.part | 6 +-
.../sqllogictest/test_files/tpch/plans/q3.slt.part | 6 +-
.../sqllogictest/test_files/tpch/plans/q4.slt.part | 4 +-
.../sqllogictest/test_files/tpch/plans/q5.slt.part | 12 +-
.../sqllogictest/test_files/tpch/plans/q6.slt.part | 2 +-
.../sqllogictest/test_files/tpch/plans/q7.slt.part | 12 +-
.../sqllogictest/test_files/tpch/plans/q8.slt.part | 16 +-
.../sqllogictest/test_files/tpch/plans/q9.slt.part | 12 +-
datafusion/sqllogictest/test_files/union.slt | 46 +-
datafusion/sqllogictest/test_files/unnest.slt | 4 +-
datafusion/sqllogictest/test_files/window.slt | 122 ++--
datafusion/substrait/src/physical_plan/consumer.rs | 7 +-
datafusion/substrait/src/physical_plan/producer.rs | 172 ++---
.../tests/cases/roundtrip_physical_plan.rs | 8 +-
.../library-user-guide/building-logical-plans.md | 2 +-
.../library-user-guide/custom-table-providers.md | 2 +-
docs/source/user-guide/explain-usage.md | 48 +-
docs/source/user-guide/sql/explain.md | 4 +-
170 files changed, 4868 insertions(+), 3460 deletions(-)
create mode 100644 datafusion/core/src/datasource/data_source.rs
create mode 100644
datafusion/core/src/datasource/physical_plan/parquet/source.rs
rename datafusion/core/tests/{parquet_exec.rs => parquet_config.rs} (93%)
create mode 100644 datafusion/physical-plan/src/source.rs
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
