This is an automated email from the ASF dual-hosted git repository.
gabotechs pushed a change to branch gabotechs/port-iceberg-rust-history
in repository https://gitbox.apache.org/repos/asf/datafusion-iceberg.git
omit 4ef4679 deps: upgrade to DataFusion 55.0 and Arrow/Parquet 59.2
(#2958)
omit 40e94c0 refactor: parse table properties lazily (#3030)
omit 962d10c docs: fix references in public docs (#2937)
omit 08577a5 refactor: Migrate TableProperties consumers to getters (#2989)
omit 27a250d refactor: Add manifest reader (#2884)
omit a1763d5 feat(encryption) [15/N] write encrypted parquet data files
(#2701)
omit 61ec233 fix (datafusion) IcebergWriteExec emits incorrect schema
(#2975)
omit 572e831 feat(parquet): wire in additional parquet writer settings
(#2887)
omit c1f391d feat: add history metadata table (#2825)
omit 844c9a9 chore(lint) check unused_qualifications (#2839)
omit 65aab27 feat: Variant Support (#2188)
omit b576a89 chore(deps): Bump DataFusion to 54.0.0 (#2648)
omit f26e5ce chore: Update changelog for 0.10.0 (#2782)
omit 49dbf74 feat(writer): honor write.parquet.* via
ParquetWriterBuilder::from_table_properties (#2561)
omit dde4592 fix(datafusion): reject non-append insert operations (#2714)
omit d3b83ee fix(datafusion): return single row with count 0 for empty
inserts (#2712)
omit fd39d74 chore: Update changelog and dependency lists for 0.10.0
(#2745)
omit ef65c4c chore: Bump version to 0.10.0, update changelog (#2700)
omit 0064eaf chore: Move dependency list generation from a release task to
a PR author responsibility (#2676)
omit d27b5d4 feat(datafusion): push down isnan over NaN-preserving numeric
expressions (#2592)
omit e30ec0c chore: Bump MSRV to 1.94 (#2652)
omit bb34fd4 refactor!(writer): borrow TableMetadata in
DefaultLocationGenerator::new (#2604)
omit 8881c57 chore(refactor): add manifest list reader (#2541)
omit 377fa01 chore(ci): omit blanket/auto-trait/auto-derived impls from
public API (#2542)
omit 1830263 feat(ci): add cargo-public-api checks for public API
stability (#2525)
omit 6fdd1ca Use cargo publish to publish crates (#2489)
omit bffd0ad feat: HuggingFace Hub storage backend and CDC table
properties (#2375)
omit be4d23e feat!(runtime): Support custom Runtime in Catalog (#2308)
omit 8889f45 fix(tests): remove unnecessary `.to_string()` calls for
metadata_location (#2407)
omit a5b443b feat(datafusion): show pushed-down limit in IcebergTableScan
EXPLAIN output (#2360)
omit 7261ba6 Make `convert_filters_to_predicate` public (#2118)
omit 798b276 deps: upgrade DataFusion to 53.0, Arrow to 58 (#2206)
omit 417a98e chore: Bump version to 0.9.0, update dep list and changelog
(#2223)
omit ed944e8 chore: Update MSRV to 1.91.0 (#2204)
omit 8c4d341 feat!(storage): Integration Storage trait with FileIO (#2116)
omit 8ae961e feat(datafusion): support isnan predicate pushdown to Iceberg
(#2142)
omit 0d1ff2b feat(datafusion): Add Timestamp scalar value conversion for
predicate pushdown (#2069)
omit fb64081 feat(datafusion): Add Boolean predicate pushdown support
(#2082)
omit f0a66e8 feat(datafusion): Add LIKE predicate pushdown for StartsWith
patterns (#2014)
omit fca5c7f feat(datafusion): Add support for DROP TABLE (#2033)
omit a330a37 feat(datafusion): Add Binary scalar value conversion for
predicate pushdown (#2048)
omit 8de0b28 feat(datafusion): Support `CREATE TABLE` for DataFusion
(#1972)
omit 27db087 feat(datafusion): Add schema validation for partition
projection (#2008)
omit a4acdc0 feat(spec): add table_properties() method to TableMetadata
(#2002)
omit 0df6881 feat(datafusion): Add LIMIT pushdown support (#2006)
omit 232fe9d feat(datafusion): Apply SortExec when writing in clustered
mode (#2005)
omit 4943678 feat: make FanoutWriter writer configurable (#1962)
omit 23e6331 Bump to version 0.8.0 (#1938)
omit 478de96 deps: bump DataFusion to 51, Arrow to 57, pyo to 0.26 (#1899)
omit 9bbdd96 chore: bump MSRV to 1.88, fix warnings and clippy errors
(#1902)
omit ab5a61a feat(datafusion): Add `sort_by_partition` to sort the input
partitioned data (#1618)
omit dfed290 feat(datafusion): Split IcebergTableProvider into static and
non-static table provider (#1879)
omit 7d7b632 feat(datafusion): Support `INSERT INTO` partitioned tables
(#1827)
omit 3edd87c feat(datafusion): Add TaskWriter for DataFusion (#1769)
omit a483aaa feat(datafusion): implement the partitioning node for
DataFusion to define the partitioning (#1620)
omit b88d907 chore: inline format args (#1805)
omit e479278 feat: Update Datafusion to v49 (#1704)
omit 3368a84 refactor(arrow,datafusion): Reuse PartitionValueCalculator in
RecordBatchPartitionSplitter (#1781)
omit 9679cbc feat(datafusion): implement the project node to add the
partition columns (#1602)
omit 1b7ee92 feat(writer): Add clustered and fanout writer (#1735)
omit 73bbe34 feat(spec): add `table_properties.rs` to spec (#1733)
omit 0b4bcbd Improve `IcebergCommitExec` to correctly populate
properties/schema (#1721)
omit ca49792 fix: ensure CoalescePartitionsExec is enabled for
IcebergCommitExec (#1723)
omit 0d845f4 refactor(writer): Refactor writers for the future
partitioning writers (#1657)
omit db5afe6 chore: update changelog and dependencies for 0.7.0 release
(#1713)
omit 344e399 chore: Bump toolchain version to rust 1.87 and update
dependencies list (#1688)
omit 447b720 feat: bump datafusion to 48 (#1501)
omit 3135a84 feat(writer): Make LocationGenerator partition-aware (#1625)
omit d4b32af fix: await get catalog in datafusion integration test (#1635)
omit 96c2a76 feat(datafusion): Implement `insert_into` for
`IcebergTableProvider` (#1600)
omit b5200d3 feat(catalog): Implement catalog loader for in memory (#1623)
omit 7ed072a Include license and notice files in published crates (#1601)
omit b759121 feat(arrow): Allow ArrowArrayAccessor to use field.name to
match fields (#1561)
omit 9f4f308 feat(datafusion): Implement IcebergWriteExec for DataFusion
write support (#1585)
omit 179ca2f feat(datafusion): Add IcebergCommitExec to commit the written
data files (#1588)
omit 502e0f5 feat(iceberg-datafusion): make IcebergTableScan public (#1570)
omit fd9a4eb Preparing 0.6.0 release (update dep list) (#1525)
omit 118aeb2 refactor: Move memory catalog into core (#1480)
omit 4048b22 feat(datafusion): support metadata tables for Datafusion
(#879)
omit 16ab874 Bump iceberg-rust version to 0.5.1 (#1380)
omit 2387433 Regenerate the licenses and update the allowed list (#1363)
omit bca5f9c Bump iceberg-rust version to 0.5.0 (#1345)
omit 669e694 Bump iceberg-rust version to 0.5.0 (Round 1) (#1342)
omit a37bab9 Expose datafusion table provider as python binding (#1324)
omit b939bf7 chore: bump up arrow/parquet/datafusion (#1294)
omit 0a0da54 refactor: use the same MSRV for datafusion integration (#1197)
omit 6397a94 ci: Use taplo to replace cargo-sort (#1186)
omit 4c03505 refactor: Bump MSRV to 1.84 for preparing next release (#1185)
omit 040e8ea feat: nan_value_counts support (#907)
omit 6086ce9 fix: Do not extract expression from cast to date (#977)
omit ca5adec Add pyspark DataFusion integration test (#850)
omit 580b3fa Datafusion: Support cast operations (#821)
omit cdb1107 ci: check MSRV correctly (#849)
omit 5f9b1c5 chore: datafusion 44 upgrade (#867)
add f9fb4e8 Add crate for sqllogictest. (#827)
add 3aaa42e chore: datafusion 44 upgrade (#867)
add aaa4069 ci: check MSRV correctly (#849)
add 09ed98a Datafusion: Support cast operations (#821)
add 5c2966a Add pyspark DataFusion integration test (#850)
add 0ece710 fix: Do not extract expression from cast to date (#977)
add 7a95b27 feat: nan_value_counts support (#907)
add c14fae7 refactor: Bump MSRV to 1.84 for preparing next release (#1185)
add 7a4c58a ci: Use taplo to replace cargo-sort (#1186)
add 4126c1c refactor: use the same MSRV for datafusion integration (#1197)
add aa0f624 Introduce datafusion engine for sqllogictests. (#1215)
add b2a4070 chore: bump up arrow/parquet/datafusion (#1294)
add f5c22c2 Expose datafusion table provider as python binding (#1324)
add 53b2de8 Bump iceberg-rust version to 0.5.0 (Round 1) (#1342)
add 41cdc58 Bump iceberg-rust version to 0.5.0 (#1345)
add d081ca1 Regenerate the licenses and update the allowed list (#1363)
add b54e9b3 Make `dependencies.py` check all subdirectories for cargo
toml files (#1370)
add 6e821da Bump iceberg-rust version to 0.5.1 (#1380)
add 4e5be8b Rename iceberg-cli crate to iceberg-playground (#1427)
add f5a3200 feat(datafusion): support metadata tables for Datafusion
(#879)
add ea8297e refactor: Move memory catalog into core (#1480)
add c340da1 Preparing 0.6.0 release (update dep list) (#1525)
add 0357088 feat(iceberg-datafusion): make IcebergTableScan public (#1570)
add fe92810 feat(datafusion): Add IcebergCommitExec to commit the written
data files (#1588)
add 4f84224 feat(catalog): Add catalog loader and builder implementation
for rest catalog (#1580)
add c3dc228 feat(datafusion): Implement IcebergWriteExec for DataFusion
write support (#1585)
add fa4a5bc feat(arrow): Allow ArrowArrayAccessor to use field.name to
match fields (#1561)
add 759b949 Include license and notice files in published crates (#1601)
add 24d7b05 feat(catalog): Implement catalog loader for in memory (#1623)
add 313fecf feat(datafusion): Implement `insert_into` for
`IcebergTableProvider` (#1600)
add 1cb5e33 fix: await get catalog in datafusion integration test (#1635)
add 9fa0a7e feat(writer): Make LocationGenerator partition-aware (#1625)
add cc90cd1 feat(sqllogictest): Add sqllogictest schedule definition and
parsing (#1630)
add 648489b feat: bump datafusion to 48 (#1501)
add 1a7aee9 chore: Bump toolchain version to rust 1.87 and update
dependencies list (#1688)
add 1d73839 chore: update changelog and dependencies for 0.7.0 release
(#1713)
add 79428dd refactor(writer): Refactor writers for the future
partitioning writers (#1657)
add abd9092 fix: ensure CoalescePartitionsExec is enabled for
IcebergCommitExec (#1723)
add 73ca4b3 Improve `IcebergCommitExec` to correctly populate
properties/schema (#1721)
add ca1e7df feat(spec): add `table_properties.rs` to spec (#1733)
add 8dbc0cd feat(writer): Add clustered and fanout writer (#1735)
add faffe05 feat(datafusion): implement the project node to add the
partition columns (#1602)
add 4bc4b95 feat(sqllogictest): Add support for iceberg datafusion
sqllogictest integration (#1764)
add 1a151a2 fix(build): Pin home version after merging #1764 (#1783)
add 8755ffd refactor(arrow,datafusion): Reuse PartitionValueCalculator in
RecordBatchPartitionSplitter (#1781)
add 940faf8 feat: Update Datafusion to v49 (#1704)
add df30add chore: inline format args (#1805)
add f3630c8 feat(datafusion): implement the partitioning node for
DataFusion to define the partitioning (#1620)
add 1b1634d feat(datafusion): Add TaskWriter for DataFusion (#1769)
add 875de0c feat(datafusion): Support `INSERT INTO` partitioned tables
(#1827)
add 4afa220 feat(datafusion): Split IcebergTableProvider into static and
non-static table provider (#1879)
add 34987e7 feat(datafusion): Add `sort_by_partition` to sort the input
partitioned data (#1618)
add bd34f6a feat(datafusion): Add sqllogictest for DataFusion INSERT INTO
(#1887)
add 14ef92f chore: bump MSRV to 1.88, fix warnings and clippy errors
(#1902)
add fe7c516 deps: bump DataFusion to 51, Arrow to 57, pyo to 0.26 (#1899)
add 26f29f5 Bump to version 0.8.0 (#1938)
add 3e42b6a feat(sqllogictest): use serde derived structs for schedule
parsing (#1953)
add ef318f6 feat: make FanoutWriter writer configurable (#1962)
add f5d4712 feat(datafusion): Apply SortExec when writing in clustered
mode (#2005)
add 2987cb9 feat(datafusion): Add LIMIT pushdown support (#2006)
add ae97284 feat(spec): add table_properties() method to TableMetadata
(#2002)
add 0d67697 feat(datafusion): Add schema validation for partition
projection (#2008)
add 3e2bd46 feat(datafusion): Support `CREATE TABLE` for DataFusion
(#1972)
add edb680f feat(datafusion): Add Binary scalar value conversion for
predicate pushdown (#2048)
add ac28435 feat(datafusion): Add support for DROP TABLE (#2033)
add ef3d134 feat(datafusion): Add LIKE predicate pushdown for StartsWith
patterns (#2014)
add f943524 Cleanup integration tests (#2074)
add bb74aec Split sqllogictests into different schedule (#2085)
add 7de99de feat(datafusion): Add Boolean predicate pushdown support
(#2082)
add c3317d6 feat(datafusion): Add Timestamp scalar value conversion for
predicate pushdown (#2069)
add 51f2e72 feat(playground): add memory catalog support (#2121)
add 52a768c feat(datafusion): support isnan predicate pushdown to Iceberg
(#2142)
add eac138b chore(deps): upgrade to DataFusion 52 (#1997)
add 8e03379 feat!(storage): Integration Storage trait with FileIO (#2116)
add da36df4 chore: Update MSRV to 1.91.0 (#2204)
add 40296e2 chore: Bump version to 0.9.0, update dep list and changelog
(#2223)
add 8b2a218 deps: upgrade DataFusion to 53.0, Arrow to 58 (#2206)
add a595da8 Make `convert_filters_to_predicate` public (#2118)
add c14f6ad feat(datafusion): show pushed-down limit in IcebergTableScan
EXPLAIN output (#2360)
add 24e5efa fix(sqllogictest): add missing tokio `rt-multi-thread`
feature (#2386)
add 1716108 fix(tests): remove unnecessary `.to_string()` calls for
metadata_location (#2407)
add db407e1 feat!(runtime): Support custom Runtime in Catalog (#2308)
add fcef7a4 feat: HuggingFace Hub storage backend and CDC table
properties (#2375)
add 1e8c97c ci(toml): check toml fmt (#2504)
add 1e33270 Use cargo publish to publish crates (#2489)
add cf64ae4 feat(ci): add cargo-public-api checks for public API
stability (#2525)
add a891a6e chore(ci): omit blanket/auto-trait/auto-derived impls from
public API (#2542)
add 324a8a3 chore(refactor): add manifest list reader (#2541)
add 401ca48 refactor!(writer): borrow TableMetadata in
DefaultLocationGenerator::new (#2604)
add c7f0b96 chore: Bump MSRV to 1.94 (#2652)
add 927cbc3 feat(datafusion): push down isnan over NaN-preserving numeric
expressions (#2592)
add 96069a3 chore: Move dependency list generation from a release task to
a PR author responsibility (#2676)
add 23760e2 chore: Bump version to 0.10.0, update changelog (#2700)
add 8a9e85e chore: Update changelog and dependency lists for 0.10.0
(#2745)
add 34bc119 fix(datafusion): return single row with count 0 for empty
inserts (#2712)
add 6886622 fix(datafusion): reject non-append insert operations (#2714)
add f84b768 feat(writer): honor write.parquet.* via
ParquetWriterBuilder::from_table_properties (#2561)
add 6ed0625 chore: Update changelog for 0.10.0 (#2782)
add e5de23c chore(deps): Bump DataFusion to 54.0.0 (#2648)
add 6ad5362 feat: Variant Support (#2188)
add 26e9c04 chore(lint) check unused_qualifications (#2839)
add df357cc feat: add history metadata table (#2825)
add 1c126c1 feat(parquet): wire in additional parquet writer settings
(#2887)
add 967ca39 fix (datafusion) IcebergWriteExec emits incorrect schema
(#2975)
add 09615d6 feat(encryption) [15/N] write encrypted parquet data files
(#2701)
add 6af5807 refactor: Add manifest reader (#2884)
add c927fcb refactor: Migrate TableProperties consumers to getters (#2989)
add 700db67 feature(encryption): [17/N] Enable encrypted writes (#3018)
add 6af6a00 docs: fix references in public docs (#2937)
add aac2424 chore: Update MSRV to Rust 1.95 (#3083)
add 9c1a70a refactor: parse table properties lazily (#3030)
add fdd7e23 deps: upgrade to DataFusion 55.0 and Arrow/Parquet 59.2
(#2958)
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (4ef4679)
\
N -- N -- N refs/heads/gabotechs/port-iceberg-rust-history
(fdd7e23)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
No new revisions were added by this update.
Summary of changes:
Cargo.toml => playground/Cargo.toml | 46 +-
playground/DEPENDENCIES.rust.tsv | 517 +++++++++++++++++++++
README.md => playground/README.md | 7 +-
playground/src/catalog.rs | 204 ++++++++
{src => playground/src}/lib.rs | 14 +-
playground/src/main.rs | 112 +++++
Cargo.toml => sqllogictest/Cargo.toml | 39 +-
.../DEPENDENCIES.rust.tsv | 69 +++
README.md => sqllogictest/README.md | 17 +-
sqllogictest/src/engine/datafusion.rs | 204 ++++++++
sqllogictest/src/engine/mod.rs | 143 ++++++
{src => sqllogictest/src}/error.rs | 46 +-
{src => sqllogictest/src}/lib.rs | 17 +-
sqllogictest/src/schedule.rs | 255 ++++++++++
.../testdata/schedules/df_basic_queries.toml | 27 +-
.../schedules/df_binary_predicate_pushdown.toml | 27 +-
.../schedules/df_boolean_predicate_pushdown.toml | 27 +-
.../testdata/schedules/df_create_table.toml | 27 +-
.../testdata/schedules/df_drop_table.toml | 27 +-
.../schedules/df_encrypted_insert_round_trip.toml | 27 +-
.../testdata/schedules/df_insert_and_like.toml | 31 +-
.../testdata/schedules/df_scan_all_types.toml | 27 +-
.../testdata/schedules/df_show_tables.toml | 27 +-
.../schedules/df_timestamp_predicate_pushdown.toml | 27 +-
.../testdata/slts/df_test/basic_queries.slt | 224 +++++++++
.../slts/df_test/binary_predicate_pushdown.slt | 67 +++
.../slts/df_test/boolean_predicate_pushdown.slt | 124 +++++
.../testdata/slts/df_test/create_table.slt | 90 ++++
sqllogictest/testdata/slts/df_test/drop_table.slt | 63 +++
.../slts/df_test/encrypted_insert_round_trip.slt | 35 +-
sqllogictest/testdata/slts/df_test/insert_into.slt | 153 ++++++
.../slts/df_test/like_predicate_pushdown.slt | 109 +++++
.../testdata/slts/df_test/scan_all_types.slt | 128 +++++
sqllogictest/testdata/slts/df_test/show_tables.slt | 46 ++
.../slts/df_test/timestamp_predicate_pushdown.slt | 183 ++++++++
sqllogictest/tests/sqllogictests.rs | 87 ++++
36 files changed, 2966 insertions(+), 307 deletions(-)
copy Cargo.toml => playground/Cargo.toml (54%)
create mode 100644 playground/DEPENDENCIES.rust.tsv
copy README.md => playground/README.md (82%)
create mode 100644 playground/src/catalog.rs
copy {src => playground/src}/lib.rs (81%)
create mode 100644 playground/src/main.rs
copy Cargo.toml => sqllogictest/Cargo.toml (65%)
copy DEPENDENCIES.rust.tsv => sqllogictest/DEPENDENCIES.rust.tsv (85%)
copy README.md => sqllogictest/README.md (65%)
create mode 100644 sqllogictest/src/engine/datafusion.rs
create mode 100644 sqllogictest/src/engine/mod.rs
copy {src => sqllogictest/src}/error.rs (51%)
copy {src => sqllogictest/src}/lib.rs (77%)
create mode 100644 sqllogictest/src/schedule.rs
copy .asf.yml => sqllogictest/testdata/schedules/df_basic_queries.toml (57%)
copy .asf.yml =>
sqllogictest/testdata/schedules/df_binary_predicate_pushdown.toml (57%)
copy .asf.yml =>
sqllogictest/testdata/schedules/df_boolean_predicate_pushdown.toml (57%)
copy .asf.yml => sqllogictest/testdata/schedules/df_create_table.toml (57%)
copy .asf.yml => sqllogictest/testdata/schedules/df_drop_table.toml (57%)
copy .asf.yml =>
sqllogictest/testdata/schedules/df_encrypted_insert_round_trip.toml (57%)
copy .asf.yml => sqllogictest/testdata/schedules/df_insert_and_like.toml (57%)
copy .asf.yml => sqllogictest/testdata/schedules/df_scan_all_types.toml (57%)
copy .asf.yml => sqllogictest/testdata/schedules/df_show_tables.toml (57%)
copy .asf.yml =>
sqllogictest/testdata/schedules/df_timestamp_predicate_pushdown.toml (57%)
create mode 100644 sqllogictest/testdata/slts/df_test/basic_queries.slt
create mode 100644
sqllogictest/testdata/slts/df_test/binary_predicate_pushdown.slt
create mode 100644
sqllogictest/testdata/slts/df_test/boolean_predicate_pushdown.slt
create mode 100644 sqllogictest/testdata/slts/df_test/create_table.slt
create mode 100644 sqllogictest/testdata/slts/df_test/drop_table.slt
copy .asf.yml =>
sqllogictest/testdata/slts/df_test/encrypted_insert_round_trip.slt (57%)
create mode 100644 sqllogictest/testdata/slts/df_test/insert_into.slt
create mode 100644
sqllogictest/testdata/slts/df_test/like_predicate_pushdown.slt
create mode 100644 sqllogictest/testdata/slts/df_test/scan_all_types.slt
create mode 100644 sqllogictest/testdata/slts/df_test/show_tables.slt
create mode 100644
sqllogictest/testdata/slts/df_test/timestamp_predicate_pushdown.slt
create mode 100644 sqllogictest/tests/sqllogictests.rs
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
