This is an automated email from the ASF dual-hosted git repository.
abhishekrb pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/druid.git
from 6177f6efd73 Fixing formatting of Iceberg Catalog Object (#15748)
add 9f95a691f78 Extension to read and ingest Delta Lake tables (#15755)
No new revisions were added by this update.
Summary of changes:
distribution/pom.xml | 4 +
docs/development/extensions-contrib/delta-lake.md | 44 +++
docs/ingestion/input-sources.md | 39 ++-
.../druid-deltalake-extensions/pom.xml | 156 ++++++++++
.../druid/delta/common/DeltaLakeDruidModule.java | 70 +++++
.../apache/druid/delta/input/DeltaInputRow.java | 208 +++++++++++++
.../apache/druid/delta/input/DeltaInputSource.java | 261 +++++++++++++++++
.../druid/delta/input/DeltaInputSourceReader.java | 138 +++++++++
.../org/apache/druid/delta/input/DeltaSplit.java | 71 +++++
.../apache/druid/delta/input/DeltaTimeUtils.java | 59 ++++
.../org/apache/druid/delta/input/RowSerde.java | 158 ++++++++++
.../org.apache.druid.initialization.DruidModule | 16 +
.../druid/delta/input/DeltaInputRowTest.java | 80 +++++
.../druid/delta/input/DeltaInputSourceTest.java | 256 ++++++++++++++++
.../apache/druid/delta/input/DeltaTestUtils.java | 322 +++++++++++++++++++++
.../druid/delta/input/DeltaTimeUtilsTest.java | 80 +++++
.../org/apache/druid/delta/input/RowSerdeTest.java | 45 +++
.../src/test/resources/README.md | 68 +++++
.../src/test/resources/create_delta_table.py | 122 ++++++++
...-4e49-b8e7-46132a57d039-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-4dbc-90e5-de1d6a5b5ac6-c000.snappy.parquet.crc | Bin 0 -> 16 bytes
...-40f2-9185-45237f51b9bf-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-4189-8927-97fe1720df8d-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-4a5c-b504-62c7c27702c1-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-475a-afbb-44ae0b9cba30-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-4145-ac40-6ccbc04acc79-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-43b7-87db-448c67a315df-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-444c-8984-6baecf6987ee-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-415f-b28a-a85d960ccb04-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-477d-b5fd-d00a857f7e54-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-4092-8ea2-0ee576ca7327-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-45ab-92d4-24d725cc07e1-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-4f1c-8c07-3dac19b12e91-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-4c02-b256-bc890fadce7e-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
...-4f80-bd63-e369c6335699-c000.snappy.parquet.crc | Bin 0 -> 28 bytes
.../_delta_log/.00000000000000000000.json.crc | Bin 0 -> 72 bytes
.../_delta_log/.00000000000000000001.json.crc | Bin 0 -> 36 bytes
.../_delta_log/00000000000000000000.json | 13 +
.../_delta_log/00000000000000000001.json | 6 +
...0c50-4e49-b8e7-46132a57d039-c000.snappy.parquet | Bin 0 -> 2316 bytes
...c0df-4dbc-90e5-de1d6a5b5ac6-c000.snappy.parquet | Bin 0 -> 979 bytes
...db98-40f2-9185-45237f51b9bf-c000.snappy.parquet | Bin 0 -> 2316 bytes
...a2b2-4189-8927-97fe1720df8d-c000.snappy.parquet | Bin 0 -> 2301 bytes
...0232-4a5c-b504-62c7c27702c1-c000.snappy.parquet | Bin 0 -> 2455 bytes
...8ebd-475a-afbb-44ae0b9cba30-c000.snappy.parquet | Bin 0 -> 2317 bytes
...bb75-4145-ac40-6ccbc04acc79-c000.snappy.parquet | Bin 0 -> 2454 bytes
...54cb-43b7-87db-448c67a315df-c000.snappy.parquet | Bin 0 -> 2302 bytes
...c414-444c-8984-6baecf6987ee-c000.snappy.parquet | Bin 0 -> 2316 bytes
...5a9e-415f-b28a-a85d960ccb04-c000.snappy.parquet | Bin 0 -> 2302 bytes
...faaa-477d-b5fd-d00a857f7e54-c000.snappy.parquet | Bin 0 -> 2317 bytes
...e71e-4092-8ea2-0ee576ca7327-c000.snappy.parquet | Bin 0 -> 2455 bytes
...b81a-45ab-92d4-24d725cc07e1-c000.snappy.parquet | Bin 0 -> 2302 bytes
...2392-4f1c-8c07-3dac19b12e91-c000.snappy.parquet | Bin 0 -> 2455 bytes
...e177-4c02-b256-bc890fadce7e-c000.snappy.parquet | Bin 0 -> 2454 bytes
...aaec-4f80-bd63-e369c6335699-c000.snappy.parquet | Bin 0 -> 2324 bytes
.../src/test/resources/requirements.txt | 2 +
pom.xml | 2 +
web-console/assets/delta.png | Bin 0 -> 6527 bytes
.../druid-models/ingestion-spec/ingestion-spec.tsx | 20 +-
.../src/druid-models/input-source/input-source.tsx | 36 ++-
.../__snapshots__/load-data-view.spec.tsx.snap | 14 +
.../src/views/load-data-view/load-data-view.tsx | 10 +
.../input-source-step/input-source-info.tsx | 3 +
website/.spelling | 9 +-
64 files changed, 2303 insertions(+), 9 deletions(-)
create mode 100644 docs/development/extensions-contrib/delta-lake.md
create mode 100644 extensions-contrib/druid-deltalake-extensions/pom.xml
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/common/DeltaLakeDruidModule.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputRow.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputSource.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaInputSourceReader.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaSplit.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/DeltaTimeUtils.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/main/java/org/apache/druid/delta/input/RowSerde.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/main/resources/META-INF/services/org.apache.druid.initialization.DruidModule
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputRowTest.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaInputSourceTest.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaTestUtils.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/DeltaTimeUtilsTest.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/java/org/apache/druid/delta/input/RowSerdeTest.java
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/README.md
create mode 100755
extensions-contrib/druid-deltalake-extensions/src/test/resources/create_delta_table.py
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00000-b17c520a-0c50-4e49-b8e7-46132a57d039-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00000-f0224389-c0df-4dbc-90e5-de1d6a5b5ac6-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00001-e18dc7d0-db98-40f2-9185-45237f51b9bf-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00001-f5c4b19d-a2b2-4189-8927-97fe1720df8d-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00002-b2249397-0232-4a5c-b504-62c7c27702c1-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00003-648766cd-8ebd-475a-afbb-44ae0b9cba30-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00003-fa1d35b8-bb75-4145-ac40-6ccbc04acc79-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00004-d580234a-54cb-43b7-87db-448c67a315df-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00005-963e7ff5-c414-444c-8984-6baecf6987ee-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00005-9ff9b585-5a9e-415f-b28a-a85d960ccb04-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00006-78cd057e-faaa-477d-b5fd-d00a857f7e54-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00007-00eb0d30-e71e-4092-8ea2-0ee576ca7327-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00007-15147217-b81a-45ab-92d4-24d725cc07e1-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00008-9f526a56-2392-4f1c-8c07-3dac19b12e91-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00009-e21486a5-e177-4c02-b256-bc890fadce7e-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/.part-00009-ee9dd918-aaec-4f80-bd63-e369c6335699-c000.snappy.parquet.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/_delta_log/.00000000000000000000.json.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/_delta_log/.00000000000000000001.json.crc
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/_delta_log/00000000000000000000.json
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/_delta_log/00000000000000000001.json
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00000-b17c520a-0c50-4e49-b8e7-46132a57d039-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00000-f0224389-c0df-4dbc-90e5-de1d6a5b5ac6-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00001-e18dc7d0-db98-40f2-9185-45237f51b9bf-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00001-f5c4b19d-a2b2-4189-8927-97fe1720df8d-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00002-b2249397-0232-4a5c-b504-62c7c27702c1-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00003-648766cd-8ebd-475a-afbb-44ae0b9cba30-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00003-fa1d35b8-bb75-4145-ac40-6ccbc04acc79-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00004-d580234a-54cb-43b7-87db-448c67a315df-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00005-963e7ff5-c414-444c-8984-6baecf6987ee-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00005-9ff9b585-5a9e-415f-b28a-a85d960ccb04-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00006-78cd057e-faaa-477d-b5fd-d00a857f7e54-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00007-00eb0d30-e71e-4092-8ea2-0ee576ca7327-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00007-15147217-b81a-45ab-92d4-24d725cc07e1-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00008-9f526a56-2392-4f1c-8c07-3dac19b12e91-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00009-e21486a5-e177-4c02-b256-bc890fadce7e-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/employee-delta-table/part-00009-ee9dd918-aaec-4f80-bd63-e369c6335699-c000.snappy.parquet
create mode 100644
extensions-contrib/druid-deltalake-extensions/src/test/resources/requirements.txt
create mode 100644 web-console/assets/delta.png
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
