This is an automated email from the ASF dual-hosted git repository.
dependabot[bot] pushed a change to branch dependabot/cargo/main/proto-33bc8ff79f
in repository https://gitbox.apache.org/repos/asf/datafusion.git
omit aed15a7a6d chore(deps): bump pbjson-types in the proto group across 1
directory
add 591ae10c98 feat: Store only the last row in the previous cursor for
round robin tie-breaking purposes (#23619)
add 3985bd5b4a Enable more lints (#24066)
add 186f96fba6 refactor(hash-aggr): migrate ordered single aggregation
(#24259)
add ab12f5e4be fix(ffi): preserve TableProvider DML overrides (#23906)
add 1164d60221 Minor: avoid a few clones in sum accumulator setup (#24309)
add 6b051b0bfd chore(deps): bump the all-other-cargo-deps group across 1
directory with 3 updates (#24320)
add 1a770de07e perf(catalog): cut `datafusion-catalog` compile time ~6x
(#24325)
add ca1e25e235 perf(core): cut `datafusion` core compile time ~10x (#24329)
add 519687ca22 perf(catalog-listing): cut `datafusion-catalog-listing`
compile time ~4.4x (#24330)
add 8c099dc150 perf(session): cut `datafusion-session` compile time ~1.7x
(#24326)
add f17158738b perf(build): don't build the doc-generator binaries by
default (#24338)
add e76f1af093 perf(build): use line-tables-only debug info in the `dev`
profile (#24339)
add 0a429a37db fix: emit unmatched NULL-key right rows in RIGHT/FULL
PiecewiseMergeJoin (#24336)
add 6adb504363 bench: add parquet_row_filter_skip suite for the
fully-matched RowFilter skip (#24328)
add 3406067328 chore(deps): bump Swatinem/rust-cache from 2.8.1 to 2.9.2
in /.github/actions/setup-macos-aarch64-builder (#24307)
add 473ba171c0 chore(deps): bump taiki-e/install-action from 2.85.10 to
2.85.11 (#24306)
add 90164e1ae5 chore(deps-dev): bump webpack-cli from 5.1.4 to 7.2.2 in
/datafusion/wasmtest/datafusion-wasm-app (#24302)
add 50cdbde1ed fix: preserve NULL semantics in bitwise xor simplification
(#24248)
add ede32c170d chore(deps): bump pygithub from 2.8.1 to 2.9.1 (#24299)
add 91ead9f87b chore(deps): bump setuptools from 83.0.0 to 84.0.0 (#24301)
add 13552dd6e8 chore(deps): bump rich from 14.3.2 to 15.0.0 (#24305)
add c085cdfa9b chore(deps-dev): bump webpack from 5.105.0 to 5.109.2 in
/datafusion/wasmtest/datafusion-wasm-app in the all-npm-deps group across 1
directory (#24300)
add 5a5da23bdc chore(deps): bump typing-extensions from 4.15.0 to 4.16.0
(#24304)
add 6b7da9a9f3 chore(deps): bump requests from 2.33.0 to 2.34.2 (#24303)
add a635b18d3b docs: fix signum documentation to say zero returns 0
(#24324)
add 773ef226c6 docs: update proto-models README for the datafusion-common
dependency (#24279)
add 84a8e6db91 Reduce datafusion-spark dependency footprint (#24351)
add 1f0615a12b Fix ArrowBytesViewMap retained capacity accounting (#24257)
add 574fe67f9a fix(parquet): sync rg_plan to decoder frontier — fix wrong
TopK results from re-reading already-delivered row groups (#24352) (#24354)
add 9f377c4486 fix: correct list field inner type in array functions
(#24345)
add 00eba790b1 feat(pwmj): support LeftSemi/LeftAnti existence joins via
classic scan (#23870)
add 571477c4a7 feat(cli): add opt-in Spark function registration (#24179)
add 1b67f2e9d6 Docs: document manual catalog impls and how compilation got
faster (#24362)
add ec110cea18 fix(parquet): don't runtime-prune row groups while a
page-index RowSelection is live (#24355) (#24359)
add 7c079f744f fix: preserve the input list's inner field in
array_append/prepend/replace (#24365)
add cc7565be1e fix: don't report equi-key equivalence for
PiecewiseMergeJoin range joins (#24361)
add b9399dc228 perf: Update `TableProvider` to take scan projection as
`Option<&[usize]>` (#24376)
add f956fd1128 fix(proto): preserve Arrow IPC stream format (#24224)
add f3ef45a53c docs: Document `WindowFn` and add comments (#24118)
add a10d19374f Perf: remove extra allocations for hex operations (#24057)
add 67502a364c Document decimal AVG wrapping arithmetic (#23685)
add bb45fb8275 docs(upgrading): document RecursiveQuery schema field in
upgrade guide (#23991)
add 47794ff246 Improve planning speed: Fast path for `union_schema` when
all children share a schema (#24389)
add 4403c5d84d bench: pwmj left semi/anti join (#24160)
add d3c47b4a80 fix: drain PiecewiseMergeJoin output before state
transitions (#24298)
add 16b08db00d fix: preserve NULL semantics when simplifying col ~ '.*'
(#24380)
add e7e037dbeb docs: add an agent skill for auditing datafusion-spark
expressions (#23893)
add bf1c17ae6b feat: union nested Parquet leaves across projection
accesses (#24130)
add 2da78d67ce fix(spark): correct mod ANSI zero divisor and negative zero
handling (#23987)
add 4042812a86 fix: consider nulls_first when propagating ordered
SortProperties (#24206)
add c1b39bd503 Feat: add dictionaries as a supported group column type
(#23187)
add b551ba0375 fix: skip WindowTopN rewrite when fetch is 0 to avoid
PartitionedTopK panic (#24405)
add 3359eab09e feat: add implicit native coercion constructor (#24282)
add 9fdf1447ef feat: Add support for Map type in SQL (#24397)
add aff5850c0a Add per-aggregate argument evaluation metrics for grouped
hash aggregation (#24024)
add 110a90aeeb feat: add sphinx-sitemap dependency and configuration for
sitemap generation (#24332)
add bbf739a3cf feat: Add robots.txt to site (#24333)
add 710e74e676 Add focused benchmark for Parquet pruning setup cache
(#24323)
add 83be77c523 fix: translate SQL wildcards in SIMILAR TO patterns
(#22263) (#23188)
add 9a83691aad docs: add HashJoinExec metrics to user guide (#24055)
add bb038a6739 fix: ScalarSubqueryToJoin fails when CTE inside correlated
scalar subquery (#24363)
add c7be84a18c perf: Reduce record batch memory accounting overhead
(#24319)
add 708ee7ece4 refactor: CrossJoinStream (simplifying, less state, async
generator pattern) (#24291)
add 6eaca8bfe8 perf: Extend WindowTopN to `dense_rank` (#23869)
add e16d453516 docs: add Streamling to known users (#24439)
add 97ab69e303 chore(deps): bump pbjson-types in the proto group across 1
directory
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (aed15a7a6d)
\
N -- N -- N refs/heads/dependabot/cargo/main/proto-33bc8ff79f
(97ab69e303)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
.../audit-datafusion-spark-expression/SKILL.md | 996 +++++++++++++++++
.../setup-macos-aarch64-builder/action.yaml | 2 +-
.github/workflows/audit.yml | 2 +-
.github/workflows/breaking_changes_detector.yml | 2 +-
.github/workflows/dependencies.yml | 2 +-
.github/workflows/dev.yml | 6 +-
.github/workflows/docs.yaml | 2 +-
.github/workflows/docs_pr.yaml | 2 +-
.github/workflows/rust.yml | 8 +-
Cargo.lock | 34 +-
Cargo.toml | 67 +-
benchmarks/Cargo.toml | 6 +-
benchmarks/bench.sh | 32 +
benchmarks/benches/parquet_pruning_setup_cache.rs | 110 ++
benchmarks/sql_benchmarks/README.md | 4 +-
.../benchmarks/control/q10.benchmark | 7 +
.../benchmarks/skip/q01.benchmark | 7 +
.../benchmarks/skip/q02.benchmark | 7 +
.../benchmarks/skip/q03.benchmark | 7 +
.../init/cleanup.sql | 0
.../parquet_row_filter_skip/init/settings.sql | 9 +
.../parquet_row_filter_skip/load/clustered.sql | 44 +
.../parquet_row_filter_skip/load/scrambled.sql | 43 +
.../parquet_row_filter_skip.benchmark.template | 50 +
.../parquet_row_filter_skip.suite | 26 +
.../queries/control/q10.sql | 8 +
.../parquet_row_filter_skip/queries/skip/q01.sql | 7 +
.../parquet_row_filter_skip/queries/skip/q02.sql | 6 +
.../parquet_row_filter_skip/queries/skip/q03.sql | 8 +
.../parquet_row_filter_skip/scratch/.gitignore | 1 +
benchmarks/src/clickbench.rs | 4 +-
datafusion-cli/Cargo.toml | 1 +
datafusion-cli/src/functions.rs | 16 +-
datafusion-cli/src/lib.rs | 2 +-
datafusion-cli/src/main.rs | 24 +-
datafusion-cli/tests/cli_integration.rs | 31 +
.../custom_data_source/custom_datasource.rs | 8 +-
.../custom_data_source/default_column_values.rs | 4 +-
.../examples/data_io/parquet_advanced_index.rs | 4 +-
.../examples/data_io/parquet_embedded_index.rs | 2 +-
.../examples/data_io/parquet_index.rs | 4 +-
.../examples/data_io/remote_catalog.rs | 4 +-
datafusion-examples/examples/udf/simple_udtf.rs | 4 +-
datafusion-testing | 2 +-
datafusion/catalog-listing/src/table.rs | 161 ++-
datafusion/catalog/src/async.rs | 2 +-
datafusion/catalog/src/cte_worktable.rs | 89 +-
datafusion/catalog/src/default_table_source.rs | 2 +-
datafusion/catalog/src/empty.rs | 2 +-
datafusion/catalog/src/memory/table.rs | 155 ++-
datafusion/catalog/src/stream.rs | 124 ++-
datafusion/catalog/src/streaming.rs | 42 +-
datafusion/catalog/src/view.rs | 38 +-
datafusion/common/Cargo.toml | 4 +
datafusion/common/benches/record_batch_memory.rs | 190 ++++
datafusion/common/src/cse.rs | 2 +-
datafusion/common/src/utils/hex.rs | 2 +
datafusion/common/src/utils/memory.rs | 415 ++++++-
datafusion/common/src/utils/mod.rs | 4 +-
datafusion/common/src/utils/proxy.rs | 2 +-
datafusion/core/Cargo.toml | 26 +
datafusion/core/benches/parquet_query_sql.rs | 6 +-
datafusion/core/benches/pwmj_semi_anti_sql.rs | 289 +++++
datafusion/core/src/dataframe/mod.rs | 36 +-
datafusion/core/src/datasource/dynamic_file.rs | 26 +-
.../core/src/datasource/file_format/options.rs | 109 +-
datafusion/core/src/datasource/listing/table.rs | 58 +-
.../core/src/datasource/listing_table_factory.rs | 29 +-
datafusion/core/src/datasource/memory_test.rs | 2 +-
datafusion/core/src/datasource/provider.rs | 29 +-
datafusion/core/src/execution/session_state.rs | 62 +-
datafusion/core/src/lib.rs | 2 +-
datafusion/core/src/physical_planner.rs | 73 +-
datafusion/core/src/test_util/mod.rs | 34 +-
.../tests/custom_sources_cases/dml_planning.rs | 6 +-
datafusion/core/tests/custom_sources_cases/mod.rs | 6 +-
.../provider_filter_pushdown.rs | 2 +-
.../core/tests/custom_sources_cases/statistics.rs | 4 +-
datafusion/core/tests/fuzz_cases/join_fuzz.rs | 223 +++-
.../spilling_fuzz_in_memory_constrained_env.rs | 23 +-
datafusion/core/tests/memory_limit/mod.rs | 4 +-
.../tests/parquet/dynamic_row_group_pruning.rs | 179 ++-
.../physical_optimizer/projection_pushdown.rs | 2 +-
.../tests/physical_optimizer/pushdown_utils.rs | 2 +-
.../core/tests/physical_optimizer/test_utils.rs | 2 +-
.../core/tests/physical_optimizer/window_topn.rs | 103 +-
.../core/tests/user_defined/insert_operation.rs | 2 +-
.../core/tests/user_defined/statistics_requests.rs | 6 +-
.../user_defined/user_defined_table_functions.rs | 4 +-
datafusion/datasource-arrow/src/source.rs | 62 +-
datafusion/datasource-parquet/src/metadata.rs | 4 +-
.../src/nested_schema_pruning.rs | 109 ++
datafusion/datasource-parquet/src/opener/mod.rs | 58 +-
.../datasource-parquet/src/projection_read_plan.rs | 213 ++--
datafusion/datasource-parquet/src/push_decoder.rs | 89 +-
datafusion/expr-common/Cargo.toml | 1 +
datafusion/expr-common/src/signature.rs | 43 +-
datafusion/expr-common/src/sort_properties.rs | 267 ++++-
datafusion/expr/src/logical_plan/builder.rs | 2 +-
datafusion/ffi/src/table_provider.rs | 534 +++++++--
datafusion/ffi/src/tests/async_provider.rs | 2 +-
datafusion/ffi/src/tests/mod.rs | 70 +-
datafusion/ffi/src/tests/query_planner.rs | 7 +-
datafusion/ffi/src/tests/utils.rs | 1 +
datafusion/ffi/src/udtf.rs | 41 +-
datafusion/ffi/tests/ffi_integration.rs | 42 +
.../src/aggregate/avg_distinct/decimal.rs | 17 +-
.../src/aggregate/avg_distinct/numeric.rs | 2 +-
.../src/aggregate/sum_distinct/numeric.rs | 4 +-
.../src/approx_percentile_cont.rs | 10 +-
datafusion/functions-aggregate/src/average.rs | 37 +-
datafusion/functions-aggregate/src/sum.rs | 4 +-
datafusion/functions-nested/src/concat.rs | 140 ++-
datafusion/functions-nested/src/extract.rs | 33 +-
datafusion/functions-nested/src/range.rs | 8 +-
datafusion/functions-nested/src/replace.rs | 157 ++-
datafusion/functions-nested/src/utils.rs | 53 +-
datafusion/functions-table/src/generate_series.rs | 4 +-
datafusion/functions/src/crypto/md5.rs | 43 +-
datafusion/functions/src/encoding/inner.rs | 4 +-
datafusion/functions/src/math/signum.rs | 3 +-
datafusion/functions/src/unicode/reverse.rs | 7 +-
datafusion/optimizer/src/decorrelate.rs | 22 +-
.../src/simplify_expressions/expr_simplifier.rs | 88 +-
.../optimizer/src/simplify_expressions/regex.rs | 9 +-
.../src/simplify_expressions/simplify_exprs.rs | 4 +-
.../physical-expr-common/src/binary_view_map.rs | 86 +-
.../src/equivalence/properties/mod.rs | 2 +-
datafusion/physical-expr/src/expressions/binary.rs | 251 ++++-
datafusion/physical-expr/src/expressions/mod.rs | 3 +
.../src/expressions/similar_to_pattern.rs | 417 +++++++
datafusion/physical-expr/src/window/window_expr.rs | 7 +
datafusion/physical-optimizer/src/window_topn.rs | 96 +-
datafusion/physical-plan/Cargo.toml | 4 +
datafusion/physical-plan/benches/union_schema.rs | 159 +++
.../src/aggregates/aggregate_hash_table/common.rs | 29 +-
.../aggregate_hash_table/common_ordered.rs | 133 ++-
.../src/aggregates/aggregate_hash_table/mod.rs | 3 +-
.../aggregate_hash_table/ordered_final_table.rs | 15 +-
.../aggregate_hash_table/ordered_partial_table.rs | 15 +-
.../{single_table.rs => ordered_single_table.rs} | 59 +-
.../aggregate_hash_table/partial_table.rs | 1 +
.../src/aggregates/group_values/metrics.rs | 190 +++-
.../src/aggregates/group_values/mod.rs | 2 +-
.../group_values/multi_group_by/dictionary.rs | 841 ++++++++++++++
.../aggregates/group_values/multi_group_by/mod.rs | 113 +-
.../src/aggregates/grouped_hash_stream.rs | 61 +-
.../physical-plan/src/aggregates/hash_stream.rs | 12 +-
datafusion/physical-plan/src/aggregates/mod.rs | 116 +-
.../src/aggregates/ordered_final_stream.rs | 21 +-
...ed_final_stream.rs => ordered_single_stream.rs} | 503 ++++-----
.../physical-plan/src/aggregates/single_stream.rs | 13 +-
datafusion/physical-plan/src/analyze.rs | 2 +-
datafusion/physical-plan/src/filter.rs | 2 +-
datafusion/physical-plan/src/joins/cross_join.rs | 241 ++--
.../src/joins/piecewise_merge_join/classic_join.rs | 133 +--
.../src/joins/piecewise_merge_join/exec.rs | 86 +-
.../joins/piecewise_merge_join/existence_join.rs | 819 ++++++++++++++
.../src/joins/piecewise_merge_join/mod.rs | 1 +
.../src/joins/piecewise_merge_join/utils.rs | 14 +-
datafusion/physical-plan/src/joins/utils.rs | 9 +-
datafusion/physical-plan/src/sorts/cursor.rs | 115 +-
datafusion/physical-plan/src/sorts/merge.rs | 23 +-
.../physical-plan/src/sorts/partitioned_topk.rs | 68 +-
datafusion/physical-plan/src/streaming.rs | 6 +-
datafusion/physical-plan/src/topk/mod.rs | 1165 +++++++++++++++++++-
datafusion/physical-plan/src/union.rs | 46 +
datafusion/proto-models/README.md | 17 +-
datafusion/proto-models/proto/datafusion.proto | 15 +
datafusion/proto-models/src/generated/pbjson.rs | 195 ++++
datafusion/proto-models/src/generated/prost.rs | 43 +-
datafusion/proto/src/bytes/mod.rs | 33 +-
datafusion/proto/src/logical_plan/mod.rs | 5 +-
datafusion/proto/src/physical_plan/from_proto.rs | 6 +-
datafusion/proto/tests/cases/plans/exprs.rs | 22 +-
datafusion/proto/tests/cases/plans/sources.rs | 101 +-
datafusion/proto/tests/cases/serialize.rs | 61 +-
datafusion/session/Cargo.toml | 1 +
datafusion/session/src/table.rs | 96 +-
datafusion/spark/Cargo.toml | 5 +-
datafusion/spark/README.md | 12 +
datafusion/spark/src/function/hash/sha1.rs | 7 +-
datafusion/spark/src/function/hash/sha2.rs | 121 +-
datafusion/spark/src/function/math/modulus.rs | 265 ++++-
datafusion/spark/src/function/string/quote.rs | 6 +-
datafusion/spark/src/function/table/mod.rs | 2 +-
datafusion/spark/src/lib.rs | 6 +-
datafusion/sql/src/planner.rs | 11 +-
datafusion/sqllogictest/src/test_context.rs | 2 +-
.../sqllogictest/test_files/array/array_append.slt | 62 ++
.../sqllogictest/test_files/array/array_concat.slt | 19 +
.../sqllogictest/test_files/array/array_pop.slt | 33 +
.../test_files/array/array_prepend.slt | 62 ++
.../test_files/array/array_replace.slt | 87 ++
.../sqllogictest/test_files/array/array_slice.slt | 31 +
datafusion/sqllogictest/test_files/ddl.slt | 37 +
.../test_files/dynamic_row_group_pruning.slt | 177 +++
datafusion/sqllogictest/test_files/order.slt | 84 +-
.../test_files/parquet_nested_schema_pruning.slt | 44 +-
datafusion/sqllogictest/test_files/pwmj.slt | 913 ++++++++++++++-
datafusion/sqllogictest/test_files/scalar.slt | 11 +
.../sqllogictest/test_files/simplify_expr.slt | 22 +-
datafusion/sqllogictest/test_files/spark/README.md | 4 +
.../sqllogictest/test_files/spark/array/slice.slt | 6 +
.../sqllogictest/test_files/spark/math/mod.slt | 31 +
.../sqllogictest/test_files/spark/math/pmod.slt | 25 +
datafusion/sqllogictest/test_files/strings.slt | 256 ++++-
datafusion/sqllogictest/test_files/subquery.slt | 27 +
.../sqllogictest/test_files/type_coercion.slt | 4 +-
datafusion/sqllogictest/test_files/window_topn.slt | 476 ++++++++
.../wasmtest/datafusion-wasm-app/package-lock.json | 700 +++++-------
.../wasmtest/datafusion-wasm-app/package.json | 4 +-
dev/release/rat_exclude_files.txt | 1 +
dev/update_config_docs.sh | 4 +-
dev/update_function_docs.sh | 6 +-
docs/pyproject.toml | 3 +-
docs/source/conf.py | 19 +-
.../library-user-guide/custom-table-providers.md | 10 +-
docs/source/library-user-guide/upgrading/55.0.0.md | 117 ++
docs/source/robots.txt | 4 +
docs/source/user-guide/cli/usage.md | 14 +
docs/source/user-guide/introduction.md | 1 +
docs/source/user-guide/metrics.md | 19 +
docs/source/user-guide/sql/scalar_functions.md | 3 +-
uv.lock | 59 +-
225 files changed, 14536 insertions(+), 2145 deletions(-)
create mode 100644 .ai/skills/audit-datafusion-spark-expression/SKILL.md
create mode 100644 benchmarks/benches/parquet_pruning_setup_cache.rs
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/control/q10.benchmark
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q01.benchmark
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q02.benchmark
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/benchmarks/skip/q03.benchmark
copy benchmarks/sql_benchmarks/{predicate_eval =>
parquet_row_filter_skip}/init/cleanup.sql (100%)
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/init/settings.sql
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/load/clustered.sql
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/load/scrambled.sql
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.benchmark.template
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/parquet_row_filter_skip.suite
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/control/q10.sql
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q01.sql
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q02.sql
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/queries/skip/q03.sql
create mode 100644
benchmarks/sql_benchmarks/parquet_row_filter_skip/scratch/.gitignore
create mode 100644 datafusion/common/benches/record_batch_memory.rs
create mode 100644 datafusion/core/benches/pwmj_semi_anti_sql.rs
create mode 100644
datafusion/physical-expr/src/expressions/similar_to_pattern.rs
create mode 100644 datafusion/physical-plan/benches/union_schema.rs
copy
datafusion/physical-plan/src/aggregates/aggregate_hash_table/{single_table.rs
=> ordered_single_table.rs} (58%)
create mode 100644
datafusion/physical-plan/src/aggregates/group_values/multi_group_by/dictionary.rs
copy datafusion/physical-plan/src/aggregates/{ordered_final_stream.rs =>
ordered_single_stream.rs} (62%)
create mode 100644
datafusion/physical-plan/src/joins/piecewise_merge_join/existence_join.rs
create mode 100644 docs/source/robots.txt
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
