This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a change to branch main
in repository https://gitbox.apache.org/repos/asf/hudi-rs.git
from d8196af chore: fix codecov upload issue (#318)
add 169671b feat: support reading avro data block for MOR tables (#309)
No new revisions were added by this update.
Summary of changes:
.github/workflows/ci.yml | 6 +-
Cargo.toml | 4 +
crates/core/Cargo.toml | 4 +
{demo => crates/core/src/avro_to_arrow}/README.md | 14 +-
.../core/src/avro_to_arrow/arrow_array_reader.rs | 970 +++++++++++++++++++++
crates/core/src/avro_to_arrow/mod.rs | 37 +
crates/core/src/avro_to_arrow/schema.rs | 287 ++++++
crates/core/src/error.rs | 3 +
crates/core/src/file_group/log_file/avro.rs | 64 ++
crates/core/src/file_group/log_file/content.rs | 251 ++++++
crates/core/src/file_group/log_file/log_block.rs | 59 --
crates/core/src/file_group/log_file/mod.rs | 2 +
crates/core/src/file_group/log_file/reader.rs | 70 +-
crates/core/src/lib.rs | 1 +
crates/core/src/table/mod.rs | 57 +-
...89a34c95d32f-0_20250316025816068.log.1_0-54-122 | Bin 0 -> 1172 bytes
.../mor/avro/v6_trips_8i1u.sql | 53 ++
.../mor/avro/v6_trips_8i1u.zip | Bin 0 -> 28861 bytes
.../cow/v6_complexkeygen_hivestyle.sql | 0
.../cow/v6_complexkeygen_hivestyle.zip | Bin
.../data/{tables => sample_table}/cow/v6_empty.sql | 0
.../data/{tables => sample_table}/cow/v6_empty.zip | Bin
.../cow/v6_nonpartitioned.sql | 0
.../cow/v6_nonpartitioned.zip | Bin
.../v6_simplekeygen_hivestyle_no_metafields.sql | 0
.../v6_simplekeygen_hivestyle_no_metafields.zip | Bin
.../cow/v6_simplekeygen_nonhivestyle.sql | 0
.../cow/v6_simplekeygen_nonhivestyle.zip | Bin
...v6_simplekeygen_nonhivestyle_overwritetable.sql | 0
...v6_simplekeygen_nonhivestyle_overwritetable.zip | Bin
.../cow/v6_timebasedkeygen_nonhivestyle.sql | 0
.../cow/v6_timebasedkeygen_nonhivestyle.zip | Bin
.../mor/parquet}/v6_complexkeygen_hivestyle.sql | 0
.../mor/parquet}/v6_complexkeygen_hivestyle.zip | Bin
.../mor => sample_table/mor/parquet}/v6_empty.sql | 0
.../mor => sample_table/mor/parquet}/v6_empty.zip | Bin
.../mor/parquet}/v6_nonpartitioned.sql | 0
.../mor/parquet}/v6_nonpartitioned.zip | Bin
.../mor/parquet}/v6_nonpartitioned_rollback.sql | 0
.../mor/parquet}/v6_nonpartitioned_rollback.zip | Bin
.../v6_simplekeygen_hivestyle_no_metafields.sql | 0
.../v6_simplekeygen_hivestyle_no_metafields.zip | Bin
.../mor/parquet}/v6_simplekeygen_nonhivestyle.sql | 0
.../mor/parquet}/v6_simplekeygen_nonhivestyle.zip | Bin
...v6_simplekeygen_nonhivestyle_overwritetable.sql | 0
...v6_simplekeygen_nonhivestyle_overwritetable.zip | Bin
crates/test/src/lib.rs | 103 ++-
demo/infra/mc/prepare_data.sh | 12 +-
demo/table-api-python/src/main.py | 2 +-
demo/table-api-rust/src/main.rs | 2 +-
50 files changed, 1886 insertions(+), 115 deletions(-)
copy {demo => crates/core/src/avro_to_arrow}/README.md (64%)
create mode 100644 crates/core/src/avro_to_arrow/arrow_array_reader.rs
create mode 100644 crates/core/src/avro_to_arrow/mod.rs
create mode 100644 crates/core/src/avro_to_arrow/schema.rs
create mode 100644 crates/core/src/file_group/log_file/avro.rs
create mode 100644 crates/core/src/file_group/log_file/content.rs
create mode 100644
crates/core/tests/data/log_files/valid_log_avro/.ff32ab89-5ad0-4968-83b4-89a34c95d32f-0_20250316025816068.log.1_0-54-122
create mode 100644
crates/test/data/quickstart_trips_table/mor/avro/v6_trips_8i1u.sql
create mode 100644
crates/test/data/quickstart_trips_table/mor/avro/v6_trips_8i1u.zip
rename crates/test/data/{tables =>
sample_table}/cow/v6_complexkeygen_hivestyle.sql (100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_complexkeygen_hivestyle.zip (100%)
rename crates/test/data/{tables => sample_table}/cow/v6_empty.sql (100%)
rename crates/test/data/{tables => sample_table}/cow/v6_empty.zip (100%)
rename crates/test/data/{tables => sample_table}/cow/v6_nonpartitioned.sql
(100%)
rename crates/test/data/{tables => sample_table}/cow/v6_nonpartitioned.zip
(100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_simplekeygen_hivestyle_no_metafields.sql (100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_simplekeygen_hivestyle_no_metafields.zip (100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_simplekeygen_nonhivestyle.sql (100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_simplekeygen_nonhivestyle.zip (100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_simplekeygen_nonhivestyle_overwritetable.sql (100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_simplekeygen_nonhivestyle_overwritetable.zip (100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_timebasedkeygen_nonhivestyle.sql (100%)
rename crates/test/data/{tables =>
sample_table}/cow/v6_timebasedkeygen_nonhivestyle.zip (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_complexkeygen_hivestyle.sql (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_complexkeygen_hivestyle.zip (100%)
rename crates/test/data/{tables/mor => sample_table/mor/parquet}/v6_empty.sql
(100%)
rename crates/test/data/{tables/mor => sample_table/mor/parquet}/v6_empty.zip
(100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_nonpartitioned.sql (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_nonpartitioned.zip (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_nonpartitioned_rollback.sql (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_nonpartitioned_rollback.zip (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_simplekeygen_hivestyle_no_metafields.sql (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_simplekeygen_hivestyle_no_metafields.zip (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_simplekeygen_nonhivestyle.sql (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_simplekeygen_nonhivestyle.zip (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_simplekeygen_nonhivestyle_overwritetable.sql (100%)
rename crates/test/data/{tables/mor =>
sample_table/mor/parquet}/v6_simplekeygen_nonhivestyle_overwritetable.zip (100%)
