This is an automated email from the ASF dual-hosted git repository.
numinnex pushed a change to branch kafka_gateway_group
in repository https://gitbox.apache.org/repos/asf/iggy.git
from e782f9354 second round
add 2cacea5b1 fix(partitions): sync checkpoints through original writers
(#4253)
add 7bae8ec84 fix(connectors): tag sink batches with the payload's schema
(#4204)
add 2894368e1 feat(gateways): map Kafka records to Iggy messages and back
(#4229)
add 0568bc6cd merge master
No new revisions were added by this update.
Summary of changes:
.claude/skills/connector-runtime/SKILL.md | 5 +-
.claude/skills/connector-sdk/SKILL.md | 17 +-
.claude/skills/connector-sink/SKILL.md | 2 +-
Cargo.lock | 37 +-
Cargo.toml | 7 +-
core/connectors/runtime/Cargo.toml | 4 +
core/connectors/runtime/README.md | 5 +-
core/connectors/runtime/src/benchmark.rs | 5 +-
core/connectors/runtime/src/metrics.rs | 34 +-
core/connectors/runtime/src/sink.rs | 669 ++++++-
core/connectors/sdk/Cargo.toml | 2 +-
core/connectors/sdk/README.md | 15 +-
core/connectors/sdk/src/lib.rs | 334 +++-
core/connectors/sdk/src/sink.rs | 149 +-
.../sdk/src/transforms/flatbuffer_convert.rs | 23 +-
.../connectors/sdk/src/transforms/proto_convert.rs | 65 +-
core/connectors/sinks/README.md | 8 +
core/connectors/sinks/clickhouse_sink/README.md | 8 +-
core/connectors/sinks/clickhouse_sink/src/body.rs | 110 +-
core/connectors/sinks/delta_sink/Cargo.toml | 3 +
core/connectors/sinks/delta_sink/src/sink.rs | 81 +-
core/connectors/sinks/doris_sink/README.md | 3 +-
core/connectors/sinks/doris_sink/src/lib.rs | 87 +-
.../connectors/sinks/elasticsearch_sink/src/lib.rs | 80 +-
core/connectors/sinks/http_sink/README.md | 3 +-
core/connectors/sinks/http_sink/src/lib.rs | 37 +-
.../iceberg_sink/src/router/dynamic_router.rs | 58 +-
.../sinks/iceberg_sink/src/router/mod.rs | 43 +-
.../connectors/sinks/influxdb_sink/dependencies.md | 2 +-
core/connectors/sinks/meilisearch_sink/README.md | 13 +-
core/connectors/sinks/meilisearch_sink/src/lib.rs | 38 +-
core/connectors/sinks/quickwit_sink/README.md | 2 +-
core/connectors/sinks/quickwit_sink/src/lib.rs | 22 +-
core/connectors/sinks/s3_sink/README.md | 2 +-
core/connectors/sinks/s3_sink/src/formatter.rs | 28 +-
core/connectors/sinks/surrealdb_sink/README.md | 11 +-
core/connectors/sinks/surrealdb_sink/src/lib.rs | 17 +
.../sources/influxdb_source/dependencies.md | 2 +-
core/integration/Cargo.toml | 1 +
.../integration/tests/connectors/clickhouse/mod.rs | 2 +
.../tests/connectors/clickhouse/proto_text.rs | 113 ++
.../sink.toml => clickhouse/proto_text.toml} | 2 +-
.../proto_text_config}/clickhouse_sink.toml | 43 +-
core/integration/tests/connectors/doris/mod.rs | 1 +
.../meilisearch_sink.rs => doris/proto_text.rs} | 56 +-
.../{delta/sink.toml => doris/proto_text.toml} | 2 +-
.../proto_text_config/doris_sink.toml} | 35 +-
.../tests/connectors/elasticsearch/mod.rs | 1 +
.../tests/connectors/elasticsearch/proto_text.rs | 126 ++
.../sink.toml => elasticsearch/proto_text.toml} | 2 +-
.../proto_text_config/elasticsearch_sink.toml} | 23 +-
core/integration/tests/connectors/runtime/mod.rs | 1 +
.../tests/connectors/runtime/schema_tagging.rs | 134 ++
.../sink.toml => runtime/schema_tagging.toml} | 2 +-
.../stdout_sink.toml} | 15 +-
core/partitions/Cargo.toml | 9 +-
core/partitions/src/iggy_index_writer.rs | 4 +
core/partitions/src/iggy_partition.rs | 28 +-
core/partitions/src/lib.rs | 2 +
core/partitions/src/persistence.rs | 49 +-
core/simulator/src/storage/tests.rs | 93 +-
gateways/kafka/Cargo.toml | 16 +-
gateways/kafka/docs/BRIDGE_MAPPING.md | 195 ++-
gateways/kafka/src/lib.rs | 1 +
gateways/kafka/src/records.rs | 1830 ++++++++++++++++++++
65 files changed, 4470 insertions(+), 347 deletions(-)
create mode 100644 core/integration/tests/connectors/clickhouse/proto_text.rs
copy core/integration/tests/connectors/{delta/sink.toml =>
clickhouse/proto_text.toml} (93%)
copy core/{connectors/runtime/example_config/connectors =>
integration/tests/connectors/clickhouse/proto_text_config}/clickhouse_sink.toml
(54%)
copy core/integration/tests/connectors/{meilisearch/meilisearch_sink.rs =>
doris/proto_text.rs} (55%)
copy core/integration/tests/connectors/{delta/sink.toml =>
doris/proto_text.toml} (93%)
copy
core/integration/tests/connectors/{runtime/sink_transform_error_config/stdout.toml
=> doris/proto_text_config/doris_sink.toml} (60%)
create mode 100644
core/integration/tests/connectors/elasticsearch/proto_text.rs
copy core/integration/tests/connectors/{delta/sink.toml =>
elasticsearch/proto_text.toml} (92%)
copy core/{connectors/sinks/elasticsearch_sink/config.toml =>
integration/tests/connectors/elasticsearch/proto_text_config/elasticsearch_sink.toml}
(60%)
create mode 100644 core/integration/tests/connectors/runtime/schema_tagging.rs
copy core/integration/tests/connectors/{delta/sink.toml =>
runtime/schema_tagging.toml} (92%)
copy
core/integration/tests/connectors/runtime/{sink_invalid_config/stdout_valid.toml
=> schema_tagging_config/stdout_sink.toml} (76%)
create mode 100644 gateways/kafka/src/records.rs
