This is an automated email from the ASF dual-hosted git repository.
alamb pushed a change to branch 6171-simplify-with-guarantee
in repository https://gitbox.apache.org/repos/asf/arrow-datafusion.git
from f4e868052d pr feedback
add fc1fd9b63f ScalarFunctionExpr Maintaining Order (#7417)
add 3ea870cf09 Bug-fix/find_orderings_of_exprs (#7457)
add b34e7cfd55 Update prost-derive requirement from 0.11 to 0.12 (#7468)
add 716e197de8 Revert "Update prost-derive requirement from 0.11 to 0.12
(#7468)" (#7476)
add a2caad0cb4 Return error if spill file does not exist in ExternalSorter
(#7479)
add 03bbc0da83 Remove unused file_type.rs (#7478)
add c4a61852d7 Minor: more flexible pool size setting for datafusion-cli
(#7454)
add 47fd9bf5b7 Bump actions/checkout from 3 to 4 (#7480)
add 87eb126eba Support Write Options in DataFrame::write_* methods (#7435)
add e83b75f65b cp_solver, Duration vs Interval cases (#7475)
add 3bbd1f0486 feat: Allow creating a ValuesExec from record batches
(#7444)
add 18aef7c165 Make `LogicalPlan::with_new_exprs,` deprecate `from_plan`
(#7396)
add dfad0afc34 refactor: change file type logic for create table (#7477)
add 4e33be5e12 minor: do not fail analyzer if subquery plan contains
extension (#7455)
add dfd685187d Make in_list work with multiple items (#7449)
add 81d7f886e3 Minor: Add doc comments and example for
`ScalarVaue::to_scalar` (#7491)
add 0b3a0545e3 minor: Add ARROW to `CREATE EXTERNAL TABLE` docs and add
example of `COMPRESSION TYPE` (#7489)
add 2ffda2a9a8 Add backtrace to error messages (#7434)
add 7c6d731281 Make sqllogictest platform-independent for the sign of NaN
(#7462)
add ec1c8b8931 Support Configuring Parquet Column Specific Options via SQL
Statement Options (#7466)
add 63e452ace0 Minor: improve error message (#7498)
add 3a52ee1dd8 paralellize parquet (#7483)
add 4d44512946 `PrimitiveGroupsAccumulator` should propagate timestamp
timezone information properly (#7494)
add 93c209f1b5 Minor: Add `ScalarValue::data_type()` for consistency with
other APIs (#7492)
add 73d6d5f50a feat: explain with statistics (#7459)
add fda561e803 fix: incorrect nullability calculation of InListExpr (#7496)
add 270dc8ba76 Prepare 31.0.0 release (#7508)
add c7a93fcd02 Minor(proto): Implement `TryFrom<&DFSchema>` for
`protobuf::DfSchema` (#7505)
add 495c25f7d8 fix: inconsistent behaviors when dividing floating numbers
by zero (#7503)
add abea8938b5 Parallelize Stateless (CSV/JSON) File Write Serialization
(#7452)
add cc8ae61a79 Remove stray comment markings from encoding error message
(#7512)
add 7b9bb08c67 Remove implicit interval type coercion from ScalarValue
comparison (#7514)
add 0cd30886cd Minor: deprecate ScalarValue::get_datatype() (#7507)
add c3898cf9fd Propagate error from spawned task reading spills (#7510)
add 60e00b3a65 Refactor the EnforceDistribution Rule (#7488)
add 63ccd4ab8b Remove get_scan_files and ExecutionPlan::file_scan_config
(#7357) (#7487)
add 4abae3b4fa Simplify ScalarValue::distance (#7517) (#7519)
add 9b5733fe61 typo: change delimeter to delimiter (#7521)
add 87527c4308 Fix some simplification rules for floating-point arithmetic
operations (#7515)
add 561e0d7e87 Documentation Updates for New Write Related Features (#7520)
add cf229b82cb Move tests from repartition to enforce_distribution (#7539)
add 3580558305 Update Cargo.toml (#7541)
add 4fac0e1f7a Fix flaky test_sort_fetch_memory_calculation (#7534)
add 11b72d6765 Move common code to utils (#7545)
add 8e2a0e6e8f Minor: Add comments and clearer constructors to `Interval`
(#7526)
add a28d5eb49f Merge remote-tracking branch 'apache/main' into
6171-simplify-with-guarantee
add b50df8064c Fix clippy
add 2452957954 fix doc links
No new revisions were added by this update.
Summary of changes:
.github/workflows/dev.yml | 4 +-
.github/workflows/dev_pr.yml | 2 +-
.github/workflows/docs.yaml | 4 +-
.github/workflows/docs_pr.yaml | 2 +-
.github/workflows/rust.yml | 38 +-
Cargo.toml | 2 +-
benchmarks/Cargo.toml | 8 +-
datafusion-cli/Cargo.lock | 176 +-
datafusion-cli/Cargo.toml | 5 +-
datafusion-cli/src/helper.rs | 8 +-
datafusion-cli/src/main.rs | 138 +-
datafusion-examples/examples/dataframe-to-s3.rs | 12 +-
datafusion/CHANGELOG.md | 1 +
datafusion/common/src/column.rs | 4 +-
datafusion/common/src/config.rs | 4 +
datafusion/common/src/dfschema.rs | 16 +-
datafusion/common/src/error.rs | 70 +-
datafusion/common/src/file_options/csv_writer.rs | 17 +-
datafusion/common/src/file_options/json_writer.rs | 6 +
datafusion/common/src/file_options/mod.rs | 122 +-
.../common/src/file_options/parquet_writer.rs | 93 +-
datafusion/common/src/file_options/parse_utils.rs | 17 +-
datafusion/common/src/file_type.rs | 410 --
datafusion/common/src/scalar.rs | 331 +-
datafusion/common/src/stats.rs | 21 +
datafusion/core/Cargo.toml | 14 +-
datafusion/core/src/catalog/mod.rs | 2 +-
datafusion/core/src/dataframe.rs | 161 +-
datafusion/core/src/datasource/file_format/csv.rs | 9 +
datafusion/core/src/datasource/file_format/json.rs | 5 +
.../core/src/datasource/file_format/parquet.rs | 78 +-
.../core/src/datasource/file_format/write.rs | 278 +-
datafusion/core/src/datasource/listing/table.rs | 4 +-
.../core/src/datasource/listing_table_factory.rs | 15 +-
datafusion/core/src/datasource/memory.rs | 22 +-
.../core/src/datasource/physical_plan/avro.rs | 4 -
.../core/src/datasource/physical_plan/csv.rs | 14 +-
.../datasource/physical_plan/file_scan_config.rs | 40 +-
.../core/src/datasource/physical_plan/json.rs | 12 +-
.../core/src/datasource/physical_plan/mod.rs | 5 +-
.../core/src/datasource/physical_plan/parquet.rs | 12 +-
datafusion/core/src/execution/context.rs | 8 +-
.../src/physical_optimizer/dist_enforcement.rs | 2193 ----------
.../src/physical_optimizer/enforce_distribution.rs | 4406 ++++++++++++++++++++
.../{sort_enforcement.rs => enforce_sorting.rs} | 146 +-
.../core/src/physical_optimizer/join_selection.rs | 2 +-
datafusion/core/src/physical_optimizer/mod.rs | 5 +-
.../core/src/physical_optimizer/optimizer.rs | 19 +-
.../src/physical_optimizer/pipeline_checker.rs | 2 +-
.../core/src/physical_optimizer/repartition.rs | 1353 ------
.../replace_with_order_preserving_variants.rs | 7 +-
.../core/src/physical_optimizer/sort_pushdown.rs | 19 +-
.../core/src/physical_optimizer/test_utils.rs | 16 +-
datafusion/core/src/physical_optimizer/utils.rs | 76 +-
datafusion/core/src/physical_plan/analyze.rs | 18 +-
.../core/src/physical_plan/coalesce_batches.rs | 4 +
datafusion/core/src/physical_plan/common.rs | 6 +-
datafusion/core/src/physical_plan/display.rs | 54 +-
datafusion/core/src/physical_plan/filter.rs | 10 +-
.../src/physical_plan/joins/hash_join_utils.rs | 2 +-
datafusion/core/src/physical_plan/mod.rs | 6 -
.../core/src/physical_plan/repartition/mod.rs | 9 +-
datafusion/core/src/physical_plan/sorts/sort.rs | 12 +-
.../physical_plan/sorts/sort_preserving_merge.rs | 6 +
datafusion/core/src/physical_plan/stream.rs | 29 +-
datafusion/core/src/physical_plan/values.rs | 69 +-
datafusion/core/src/physical_planner.rs | 52 +-
datafusion/core/src/test/exec.rs | 4 +
datafusion/core/tests/config_from_env.rs | 2 +-
datafusion/core/tests/dataframe/mod.rs | 10 +-
datafusion/core/tests/path_partition.rs | 4 +-
datafusion/core/tests/sql/explain_analyze.rs | 18 +
datafusion/core/tests/sql/joins.rs | 30 +-
datafusion/core/tests/sql/order.rs | 4 +-
datafusion/core/tests/sql/sql_api.rs | 10 +-
datafusion/core/tests/sql/timestamp.rs | 4 +-
datafusion/execution/Cargo.toml | 4 +-
datafusion/execution/src/disk_manager.rs | 2 +-
datafusion/execution/src/memory_pool/pool.rs | 10 +-
datafusion/execution/src/object_store.rs | 10 +-
datafusion/expr/Cargo.toml | 2 +-
datafusion/expr/src/columnar_value.rs | 2 +-
datafusion/expr/src/expr_fn.rs | 2 +-
datafusion/expr/src/expr_rewriter/mod.rs | 2 +-
datafusion/expr/src/expr_schema.rs | 2 +-
datafusion/expr/src/logical_plan/builder.rs | 15 +-
datafusion/expr/src/logical_plan/dml.rs | 45 +-
datafusion/expr/src/logical_plan/plan.rs | 442 +-
datafusion/expr/src/type_coercion/aggregates.rs | 6 +-
datafusion/expr/src/type_coercion/binary.rs | 14 +-
datafusion/expr/src/utils.rs | 340 +-
datafusion/expr/src/window_frame.rs | 10 +-
datafusion/optimizer/Cargo.toml | 8 +-
datafusion/optimizer/src/analyzer/subquery.rs | 55 +
datafusion/optimizer/src/analyzer/type_coercion.rs | 17 +-
datafusion/optimizer/src/optimizer.rs | 8 +-
datafusion/optimizer/src/push_down_filter.rs | 6 +-
.../src/simplify_expressions/expr_simplifier.rs | 48 +-
.../src/simplify_expressions/guarantees.rs | 10 +-
.../src/simplify_expressions/simplify_exprs.rs | 4 +-
.../optimizer/src/unwrap_cast_in_comparison.rs | 19 +-
datafusion/physical-expr/Cargo.toml | 4 +-
.../src/aggregate/approx_percentile_cont.rs | 4 +-
.../src/aggregate/array_agg_ordered.rs | 4 +-
datafusion/physical-expr/src/aggregate/average.rs | 12 +-
.../src/aggregate/groups_accumulator/prim_op.rs | 8 +-
datafusion/physical-expr/src/aggregate/min_max.rs | 57 +-
datafusion/physical-expr/src/aggregate/utils.rs | 24 -
datafusion/physical-expr/src/analysis.rs | 6 +-
datafusion/physical-expr/src/array_expressions.rs | 36 +-
.../physical-expr/src/datetime_expressions.rs | 22 +-
datafusion/physical-expr/src/expressions/binary.rs | 6 +-
datafusion/physical-expr/src/expressions/column.rs | 27 +-
.../physical-expr/src/expressions/in_list.rs | 144 +-
.../physical-expr/src/expressions/literal.rs | 2 +-
datafusion/physical-expr/src/expressions/mod.rs | 58 +-
datafusion/physical-expr/src/expressions/not.rs | 2 +-
datafusion/physical-expr/src/expressions/nullif.rs | 2 +-
datafusion/physical-expr/src/functions.rs | 109 +-
.../physical-expr/src/intervals/cp_solver.rs | 411 +-
.../src/intervals/interval_aritmetic.rs | 71 +-
datafusion/physical-expr/src/intervals/mod.rs | 5 +-
datafusion/physical-expr/src/intervals/utils.rs | 192 +
datafusion/physical-expr/src/regex_expressions.rs | 4 +-
datafusion/physical-expr/src/scalar_function.rs | 18 +
datafusion/physical-expr/src/udf.rs | 1 +
datafusion/physical-expr/src/utils.rs | 62 +-
datafusion/proto/Cargo.toml | 6 +-
datafusion/proto/src/logical_plan/from_proto.rs | 2 +-
datafusion/proto/src/logical_plan/mod.rs | 33 +-
datafusion/proto/src/logical_plan/to_proto.rs | 18 +-
datafusion/proto/src/physical_plan/from_proto.rs | 2 +
datafusion/proto/src/physical_plan/mod.rs | 2 +
datafusion/sql/Cargo.toml | 4 +-
datafusion/sql/src/expr/identifier.rs | 8 +-
datafusion/sql/src/expr/value.rs | 4 +-
datafusion/sql/src/parser.rs | 2 +-
datafusion/sql/src/query.rs | 6 +-
datafusion/sql/src/statement.rs | 13 +-
datafusion/sql/tests/sql_integration.rs | 205 +-
datafusion/sqllogictest/Cargo.toml | 4 +-
datafusion/sqllogictest/src/engines/conversion.rs | 24 +-
datafusion/sqllogictest/test_files/copy.slt | 12 +-
datafusion/sqllogictest/test_files/decimal.slt | 7 +-
datafusion/sqllogictest/test_files/explain.slt | 37 +-
datafusion/sqllogictest/test_files/groupby.slt | 28 +-
.../sqllogictest/test_files/information_schema.slt | 1 +
.../sqllogictest/test_files/intersection.slt | 1 -
datafusion/sqllogictest/test_files/joins.slt | 63 +-
datafusion/sqllogictest/test_files/math.slt | 331 +-
datafusion/sqllogictest/test_files/order.slt | 135 +-
datafusion/sqllogictest/test_files/predicates.slt | 93 +-
datafusion/sqllogictest/test_files/scalar.slt | 55 +-
datafusion/sqllogictest/test_files/select.slt | 6 +-
.../test_files/tpch/drop_tables.slt.part | 2 -
.../sqllogictest/test_files/tpch/q15.slt.part | 28 +-
datafusion/sqllogictest/test_files/union.slt | 15 +-
datafusion/sqllogictest/test_files/window.slt | 1 +
datafusion/substrait/Cargo.toml | 2 +-
dev/changelog/31.0.0.md | 123 +
docs/source/user-guide/configs.md | 3 +-
docs/source/user-guide/expressions.md | 2 +-
docs/source/user-guide/sql/ddl.md | 18 +-
docs/source/user-guide/sql/dml.md | 12 +-
docs/source/user-guide/sql/index.rst | 1 +
docs/source/user-guide/sql/scalar_functions.md | 4 +-
docs/source/user-guide/sql/write_options.md | 131 +
167 files changed, 8874 insertions(+), 5745 deletions(-)
delete mode 100644 datafusion/common/src/file_type.rs
delete mode 100644 datafusion/core/src/physical_optimizer/dist_enforcement.rs
create mode 100644
datafusion/core/src/physical_optimizer/enforce_distribution.rs
rename datafusion/core/src/physical_optimizer/{sort_enforcement.rs =>
enforce_sorting.rs} (96%)
delete mode 100644 datafusion/core/src/physical_optimizer/repartition.rs
create mode 100644 datafusion/physical-expr/src/intervals/utils.rs
create mode 100644 dev/changelog/31.0.0.md
create mode 100644 docs/source/user-guide/sql/write_options.md
