hubcio commented on code in PR #3658:
URL: https://github.com/apache/iggy/pull/3658#discussion_r3784974717
##########
core/connectors/runtime/Dockerfile:
##########
@@ -82,34 +106,121 @@ RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG
"linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
*) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
esac && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} -p iggy-connectors
$PROFILE_FLAG && \
+ cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors
/app/iggy-connectors
+
+#
+# Generate the third-party license manifest. Required by ASF release policy:
+# convenience binaries that statically link third-party code MUST include the
+# full license text of every bundled crate inside the artifact. This one covers
+# the runtime only, which is all the slim image ships.
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+ ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary
+
+#
+# Fat builder - runtime binary plus every connector plugin cdylib, and a
+# license manifest covering all of them.
+#
+FROM --platform=$BUILDPLATFORM toolchain AS fat-builder
+ARG PROFILE=release
+ARG TARGETPLATFORM
+ARG LIBC=glibc
+
+# Only this branch reads the derived plugin set, so adding a connector leaves
+# the slim branch's cooked dependency layer untouched.
+COPY --from=planner /app/connector-plugins.txt connector-plugins.txt
+COPY --from=planner /app/connector-manifests.txt connector-manifests.txt
+
+#
+# Cook dependencies
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/app/target,id=cargo-target-fat-${TARGETPLATFORM}-${LIBC}
\
+ case "$TARGETPLATFORM:$LIBC" in \
+ "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \
+ "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \
+ "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \
+ "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
+ *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
+ esac && \
+ PLUGIN_FLAGS="$(sed 's/^/-p /' connector-plugins.txt | tr '\n' ' ')" && \
if [ "$PROFILE" = "debug" ]; then \
- cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors && \
- cp /app/target/${RUST_TARGET}/debug/iggy-connectors /app/iggy-connectors; \
+ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET}
--zigbuild \
+ -p iggy-connectors $PLUGIN_FLAGS; \
else \
- cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors
--release && \
- cp /app/target/${RUST_TARGET}/release/iggy-connectors
/app/iggy-connectors; \
+ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET}
--zigbuild --release \
+ -p iggy-connectors $PLUGIN_FLAGS; \
fi
+COPY . .
+
#
-# Generate third-party license manifest. Required by ASF release policy:
-# convenience binaries that statically link third-party code MUST include
-# the full license text of every bundled crate inside the artifact.
+# Build
#
RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
- TARGET="$(uname -m)-unknown-linux-musl" && \
- curl -sSfL
"https://github.com/EmbarkStudios/cargo-about/releases/download/0.9.0/cargo-about-0.9.0-${TARGET}.tar.gz";
\
- | tar -xz -C /usr/local/bin --strip-components=1
"cargo-about-0.9.0-${TARGET}/cargo-about" && \
- ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary
+
--mount=type=cache,target=/app/target,id=cargo-target-fat-${TARGETPLATFORM}-${LIBC}
\
+ case "$TARGETPLATFORM:$LIBC" in \
+ "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \
+ "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \
+ "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \
+ "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
+ *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
+ esac && \
+ PLUGIN_FLAGS="$(sed 's/^/-p /' connector-plugins.txt | tr '\n' ' ')" && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} -p iggy-connectors
$PLUGIN_FLAGS $PROFILE_FLAG && \
+ cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors
/app/iggy-connectors && \
+ mkdir -p /app/plugins && \
+ while IFS= read -r name; do \
+ [ -z "$name" ] && continue; \
+ so_file="/app/target/${RUST_TARGET}/${PROFILE_DIR}/lib${name}.so"; \
+ if [ ! -f "$so_file" ]; then echo "expected plugin artifact missing:
$so_file" >&2 && exit 1; fi; \
+ cp "$so_file" /app/plugins/; \
+ done < connector-plugins.txt
#
-# Final runtime - Debian trixie Slim
+# Generate the third-party license manifest covering the runtime plus every
+# bundled plugin closure. Same ASF policy as the slim manifest above; each
+# image ships the manifest that matches what it actually bundles, so the slim
+# image never over-declares plugin dependencies it does not carry.
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+ MANIFEST_FLAGS="$(sed 's/^/--manifest /' connector-manifests.txt | tr '\n'
' ')" && \
+ ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml $MANIFEST_FLAGS --output
/app/LICENSE-binary-fat
Review Comment:
this LICENSE-binary is missing the runtime's own deps. the union root in
third-party-licenses.sh consumes each manifest as a path dependency, and
`iggy-connectors` has no lib target, so cargo silently drops it and its whole
closure (just a warning, exit 0). measured it: 113 crates linked into the fat
image (axum, compio, dlopen2, figment, mimalloc, opentelemetry, tonic, ...) are
absent from the manifest it ships. slim is fine - single manifest takes the
early-return path. fix in two parts: make the script die when a union manifest
lacks a lib target, then run cargo-about per manifest against the real
workspace and merge. that also keeps versions lock-exact - the synthetic root
re-resolves without the lockfile today, e.g. it lists arrow 58.4.0 while the
binaries link 58.3.0.
##########
.github/actions/utils/docker-buildx/action.yml:
##########
@@ -342,6 +346,7 @@ runs:
with:
context: ${{ steps.ctx.outputs.context }}
file: ${{ steps.config.outputs.dockerfile }}
+ target: ${{ inputs.target }}
Review Comment:
`target` is threaded through, but the cache refs aren't: both flavors write
`<image>:buildcache-<arch>` with mode=max at the same time, and since publish
runs with gha-cache off, the registry is the only cache - last writer wins and
the other flavor rebuilds its cook and build layers from scratch every run.
thread the flavor into `scope_arch`/`scope_os` and the buildcache ref. leave
the shared `apache/iggy:buildcache-*` ref alone, that one is cross-component on
purpose.
##########
.dockerignore:
##########
@@ -21,6 +21,7 @@
/scripts/ci/*
!/scripts/ci/third-party-licenses.sh
!/scripts/ci/render-node-licenses.mjs
+!/scripts/ci/connector-plugins.sh
Review Comment:
the use_latest_ci overlay copies `.github`, `scripts` and the dockerfiles
onto the release commit, but never `.dockerignore` - so publishing an older
commit with use_latest_ci (defaults to true on manual dispatch) gets master's
dockerfile calling connector-plugins.sh while the old dockerignore still
excludes it from the build context. the planner stage dies, and both flavors go
through it. add `.dockerignore` to the save lists in publish.yml.
##########
.github/workflows/publish.yml:
##########
@@ -1003,24 +1029,29 @@ jobs:
run: |
IMAGE="${{ steps.config.outputs.image }}"
VERSION="${{ steps.ver.outputs.version }}"
+ SUFFIX="${{ matrix.flavor_suffix }}"
# Auto-publish pushed :version only when should_tag was true; in
# every other auto-publish case (:edge-only) inspect :edge instead.
if [ "${{ inputs.create_edge_docker_tag }}" = "true" ] && [
"$SHOULD_TAG" != "true" ]; then
- echo "Inspecting :edge manifest (versioned manifest was skipped:
should_tag=false)"
- docker buildx imagetools inspect "${IMAGE}:edge"
+ echo "Inspecting :edge${SUFFIX} manifest (versioned manifest was
skipped: should_tag=false)"
+ docker buildx imagetools inspect "${IMAGE}:edge${SUFFIX}"
else
- docker buildx imagetools inspect "${IMAGE}:${VERSION}"
+ docker buildx imagetools inspect "${IMAGE}:${VERSION}${SUFFIX}"
fi
# Inline per-component tagging: tightly couple the git tag to the
# multi-arch manifest that just shipped. should_tag was computed in the
# version step above and already encodes the SNAPSHOT and auto-publish
# stable-Docker skip rules. dry_run is gated at the job level.
+ # The git release tag is flavor-independent (one tag per component
+ # version), so only the default flavor creates it; other flavors ship
+ # their manifests but must not race to create the same tag.
- name: Tag Docker release (${{ matrix.key }})
if: |
success() &&
inputs.skip_tag_creation == false &&
+ matrix.flavor_suffix == '' &&
Review Comment:
this gate creates a bad failure mode for the slim tag: if the slim manifest
push fails while fat succeeds, the tag gets created, and every later
auto-publish sees it on remote and flips should_tag to false - so `x.y.z-slim`
never ships, and even "re-run failed jobs" goes green without pushing it (the
re-run recomputes should_tag=false and only refreshes `:edge-slim`). only a
manual publish recovers. cleaner shape: make docker-manifests one job per
component that reads the flavors from config, downloads all
`docker-digest-<key>*` artifacts with `merge-multiple: false`, asserts the
digest dirs match the configured flavor set, pushes every flavor's manifest,
then tags once at the end. tag only exists when every flavor shipped, and this
guard goes away.
##########
core/connectors/runtime/Dockerfile:
##########
@@ -82,34 +106,121 @@ RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG
"linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
*) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
esac && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} -p iggy-connectors
$PROFILE_FLAG && \
Review Comment:
switching `--bin iggy-connectors` to `-p iggy-connectors` changes feature
resolution, not just what gets built. the old no-`-p` build selected the whole
workspace (virtual manifest, no default-members), so features unified
workspace-wide. the slim runtime loses `reqwest/http2`, `reqwest/charset` (nine
`Response::text()` calls in the http config provider mis-decode non-utf8
responses without it), and tracing-subscriber's `tracing-log` (log-crate
records from dependencies silently stop appearing). serde_json map ordering
also differs between the two images - fat gets `preserve_order` via bson, slim
doesn't - so the same config round-trips with different key order. fat isn't
identical to today's build either. suggest pinning `http2` + `charset` on
reqwest and `tracing-log` on tracing-subscriber in `[workspace.dependencies]` -
the root is the one spot that also covers the tarball plugin build.
##########
.github/config/publish.yml:
##########
@@ -96,9 +96,24 @@ components:
platforms: ["linux/amd64", "linux/arm64"]
version_file: "core/connectors/runtime/Cargo.toml"
version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"'
- # Image ships only the runtime binary; plugin .so files are not bundled.
+ # Two published flavors from one Dockerfile. The fat flavor (empty suffix,
+ # runtime-fat stage) bundles every connector plugin and is the default;
+ # -slim (runtime stage) ships the runtime binary only. Each flavor keeps
+ # its own per-arch digest set and manifest tags. Components without a
+ # flavors list build their single default stage.
+ flavors:
+ - suffix: ""
+ target: runtime-fat
+ - suffix: "-slim"
+ target: runtime
+ # The fat image bundles the runtime plus every connector plugin, so any
+ # change under core/connectors/ must refresh :edge. crates: keeps
+ # sensitivity to shared workspace deps in the runtime's closure (e.g.
+ # iggy_common); paths: covers all plugin crates without a hardcoded
+ # per-plugin list, so a newly added connector is gated automatically.
gate:
crates: [iggy-connectors]
+ paths: [core/connectors]
Review Comment:
gate misses `scripts/ci/connector-plugins.sh` - its output decides which
plugins end up in the image, but editing it won't refresh :edge. web-ui's gate
lists its scripts/ci deps for exactly this reason. worth adding
`scripts/ci/third-party-licenses.sh` too, and optionally
`:(exclude)core/connectors/**/*.md` so docs-only edits stop triggering full
image builds (this PR's own README edit trips the gate).
##########
scripts/ci/connector-plugins.sh:
##########
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+# Single source of truth for the connector plugin set: every workspace crate
+# under core/connectors/ that builds a cdylib (a .so the iggy-connectors
+# runtime loads via dlopen). Derived from `cargo metadata` so a newly added
+# connector is picked up automatically by every consumer:
+#
+# - .github/workflows/_build_rust_artifacts.yml (edge tarball plugin list)
+# - core/connectors/runtime/Dockerfile (fat image build + bundle)
+# - .github/actions/utils/validate-third-party-licenses (license gate)
+#
+# Output modes (one plugin per line unless noted):
+# --names crate names (iggy_connector_postgres_sink)
+# --comma-names crate names, one CSV line
+# --package-flags cargo -p flags, one line (-p iggy_connector_... ...)
+# --manifests repo-relative Cargo.toml paths
+# --manifest-flags --manifest flags, one line (for third-party-licenses.sh)
+
+MODE="--names"
+if [[ $# -gt 0 ]]; then
+ MODE="$1"
+fi
+
+METADATA="$(cargo metadata --format-version 1 --no-deps)"
+WORKSPACE_ROOT="$(jq -r '.workspace_root' <<<"$METADATA")"
+
+# cdylib packages whose manifest lives under core/connectors/. The path guard
+# keeps a future non-connector cdylib elsewhere in the workspace out of the
set.
+NAMES="$(jq -r --arg root "$WORKSPACE_ROOT" '
+ .packages[]
+ | select(.manifest_path | startswith($root + "/core/connectors/"))
+ | select(.targets[] | .kind[] == "cdylib")
+ | .name
+' <<<"$METADATA" | sort -u)"
+
+if [[ -z "$NAMES" ]]; then
+ echo "connector-plugins: no cdylib plugin crates found under
core/connectors/" >&2
+ exit 1
+fi
+
+# Map each crate name back to its repo-relative manifest path.
+manifest_for() {
Review Comment:
this spawns one jq per plugin, each re-parsing the full metadata - a single
jq pass can emit the manifests directly. two things to keep if you touch it:
keep the `sort` (cargo metadata package order isn't stable, and this output
feeds the fat-builder layer cache key), and note the emitted name is used both
as `cargo -p` (package name) and `lib<name>.so` (lib target name) - those
diverge for hyphenated packages (`bench-report` -> `bench_report`), so either
emit both fields or assert they're equal.
##########
.github/workflows/publish.yml:
##########
@@ -399,21 +399,38 @@ jobs:
// Output non-Docker, non-Rust targets (SDKs only)
core.setOutput('non_docker_targets',
JSON.stringify(nonDockerTargets.length ? { include: nonDockerTargets } : {
include: [{ key: 'noop', type: 'noop' }] }));
- // Build Docker matrix: components × platforms for native runner
builds
+ // Build Docker matrix: components × flavors × platforms for native
+ // runner builds. A component may declare multiple image flavors
+ // (e.g. fat + slim) built from different Dockerfile stages; each
+ // flavor gets its own suffix and target stage. Components without
a
+ // flavors list build a single default flavor (empty suffix, empty
+ // target = default stage). flavor_suffix/flavor_target ride on
every
+ // matrix entry so the build and manifest jobs keep flavors' digest
+ // sets and tags separate.
const platforms = [
{ platform: 'linux/amd64', arch: 'amd64', runner:
'ubuntu-latest' },
{ platform: 'linux/arm64', arch: 'arm64', runner:
'ubuntu-24.04-arm' }
];
+ const flavorsOf = (key) => {
+ const fl = cfg[key] && cfg[key].flavors;
+ return Array.isArray(fl) && fl.length ? fl : [{ suffix: '',
target: '' }];
+ };
+
const dockerMatrix = [];
+ const dockerComponents = [];
for (const t of dockerTargets) {
- for (const p of platforms) {
- dockerMatrix.push({ ...t, ...p });
+ for (const f of flavorsOf(t.key)) {
+ const flavor = { flavor_suffix: f.suffix || '', flavor_target:
f.target || '' };
Review Comment:
`f.target || ''` means a flavor entry with a suffix but no target silently
builds the default (last) stage - for this dockerfile that's the fat image
published under the `-slim` tag. worth a small validation in the plan script
(or moot if the per-component manifest job shape lands).
##########
core/connectors/runtime/Dockerfile:
##########
@@ -82,34 +106,121 @@ RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG
"linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
*) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
esac && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} -p iggy-connectors
$PROFILE_FLAG && \
+ cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors
/app/iggy-connectors
+
+#
+# Generate the third-party license manifest. Required by ASF release policy:
+# convenience binaries that statically link third-party code MUST include the
+# full license text of every bundled crate inside the artifact. This one covers
+# the runtime only, which is all the slim image ships.
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+ ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary
+
+#
+# Fat builder - runtime binary plus every connector plugin cdylib, and a
+# license manifest covering all of them.
+#
+FROM --platform=$BUILDPLATFORM toolchain AS fat-builder
+ARG PROFILE=release
+ARG TARGETPLATFORM
+ARG LIBC=glibc
+
+# Only this branch reads the derived plugin set, so adding a connector leaves
+# the slim branch's cooked dependency layer untouched.
+COPY --from=planner /app/connector-plugins.txt connector-plugins.txt
+COPY --from=planner /app/connector-manifests.txt connector-manifests.txt
+
+#
+# Cook dependencies
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/app/target,id=cargo-target-fat-${TARGETPLATFORM}-${LIBC}
\
+ case "$TARGETPLATFORM:$LIBC" in \
Review Comment:
this platform-to-triple case block now appears four times in the file.
resolving it once at the end of the toolchain stage (write to a file,
`RUST_TARGET="$(cat /rust-target)"` in each RUN) keeps the expensive layers
platform-independent and leaves one place to add a platform.
##########
core/connectors/runtime/Dockerfile:
##########
@@ -82,34 +106,121 @@ RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG
"linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
*) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
esac && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} -p iggy-connectors
$PROFILE_FLAG && \
+ cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors
/app/iggy-connectors
+
+#
+# Generate the third-party license manifest. Required by ASF release policy:
+# convenience binaries that statically link third-party code MUST include the
+# full license text of every bundled crate inside the artifact. This one covers
+# the runtime only, which is all the slim image ships.
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+ ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary
+
+#
+# Fat builder - runtime binary plus every connector plugin cdylib, and a
+# license manifest covering all of them.
+#
+FROM --platform=$BUILDPLATFORM toolchain AS fat-builder
+ARG PROFILE=release
+ARG TARGETPLATFORM
+ARG LIBC=glibc
+
+# Only this branch reads the derived plugin set, so adding a connector leaves
+# the slim branch's cooked dependency layer untouched.
+COPY --from=planner /app/connector-plugins.txt connector-plugins.txt
+COPY --from=planner /app/connector-manifests.txt connector-manifests.txt
+
+#
+# Cook dependencies
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/app/target,id=cargo-target-fat-${TARGETPLATFORM}-${LIBC}
\
+ case "$TARGETPLATFORM:$LIBC" in \
+ "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \
+ "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \
+ "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \
+ "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
+ *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
+ esac && \
+ PLUGIN_FLAGS="$(sed 's/^/-p /' connector-plugins.txt | tr '\n' ' ')" && \
if [ "$PROFILE" = "debug" ]; then \
- cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors && \
- cp /app/target/${RUST_TARGET}/debug/iggy-connectors /app/iggy-connectors; \
+ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET}
--zigbuild \
+ -p iggy-connectors $PLUGIN_FLAGS; \
else \
- cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors
--release && \
- cp /app/target/${RUST_TARGET}/release/iggy-connectors
/app/iggy-connectors; \
+ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET}
--zigbuild --release \
+ -p iggy-connectors $PLUGIN_FLAGS; \
fi
+COPY . .
+
#
-# Generate third-party license manifest. Required by ASF release policy:
-# convenience binaries that statically link third-party code MUST include
-# the full license text of every bundled crate inside the artifact.
+# Build
#
RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
- TARGET="$(uname -m)-unknown-linux-musl" && \
- curl -sSfL
"https://github.com/EmbarkStudios/cargo-about/releases/download/0.9.0/cargo-about-0.9.0-${TARGET}.tar.gz";
\
- | tar -xz -C /usr/local/bin --strip-components=1
"cargo-about-0.9.0-${TARGET}/cargo-about" && \
- ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary
+
--mount=type=cache,target=/app/target,id=cargo-target-fat-${TARGETPLATFORM}-${LIBC}
\
+ case "$TARGETPLATFORM:$LIBC" in \
+ "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \
+ "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \
+ "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \
+ "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
+ *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
+ esac && \
+ PLUGIN_FLAGS="$(sed 's/^/-p /' connector-plugins.txt | tr '\n' ' ')" && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} -p iggy-connectors
$PLUGIN_FLAGS $PROFILE_FLAG && \
+ cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors
/app/iggy-connectors && \
+ mkdir -p /app/plugins && \
+ while IFS= read -r name; do \
+ [ -z "$name" ] && continue; \
+ so_file="/app/target/${RUST_TARGET}/${PROFILE_DIR}/lib${name}.so"; \
+ if [ ! -f "$so_file" ]; then echo "expected plugin artifact missing:
$so_file" >&2 && exit 1; fi; \
+ cp "$so_file" /app/plugins/; \
Review Comment:
this RUN is plain `sh -c` with no `set -e`, and a while loop's status is the
last iteration's - if `cp` fails mid-loop (enospc is realistic here) and a
later plugin copies fine, the build goes green and the image ships missing a
plugin. the tarball path in this same PR hard-fails on exactly this case. `cp
"$so_file" /app/plugins/ || exit 1` closes it.
##########
.github/workflows/edge-release.yml:
##########
@@ -59,6 +59,17 @@ jobs:
server_version=$(scripts/extract-version.sh rust-server)
echo "server_version=${server_version}" >> "$GITHUB_OUTPUT"
+ - name: Render bundled connector plugin list
+ id: plugins
+ run: |
+ # Derive the plugin list from cargo metadata so the release notes
+ # stay in sync with what is actually built (see
connector-plugins.sh).
+ {
+ echo "list<<PLUGINS_EOF"
+ scripts/ci/connector-plugins.sh --names | sed 's/^/- `/; s/$/`/'
Review Comment:
two things here. no `shell: bash` on the step means no pipefail, so if the
script fails the `sed` still exits 0 and the release notes ship an empty plugin
list, green - the only spot in this PR that degrades silently instead of
failing. and it's the first cargo invocation in a job with no rust setup, so
rustup downloads a toolchain just to print 18 names; a flaky download now kills
the edge release. reusing the list the build job already resolved (exposed
through the collect job) would fix both.
##########
.github/actions/utils/validate-third-party-licenses/action.yml:
##########
@@ -66,9 +66,17 @@ runs:
shell: bash
run: ./scripts/ci/third-party-licenses.sh --validate --manifest
core/bench/dashboard/server/Cargo.toml
- - name: Validate iggy-connect bundle
+ - name: Validate iggy-connect bundle (runtime + connector plugins)
shell: bash
- run: ./scripts/ci/third-party-licenses.sh --validate --manifest
core/connectors/runtime/Cargo.toml
+ run: |
+ # The fat image bundles the runtime plus every connector plugin, so
+ # validate the plugins' dependency closures too. The manifest list is
+ # derived from cargo metadata so a newly added connector is license
+ # validated on the PR that adds it, with no change here.
+ MANIFEST_FLAGS="$(scripts/ci/connector-plugins.sh --manifest-flags)"
Review Comment:
same bin-only problem hits here: the union root silently drops
`iggy-connectors`, so this step now validates only the plugin closures. the
runtime's closure, which the old single-manifest call covered, is no longer
license-checked at all - the step got wider on paper and narrower in practice.
also worth flipping `read-cache: "false"` above while touching this, this job
runs on every PR with a cold cargo home.
##########
core/connectors/runtime/Dockerfile:
##########
@@ -82,34 +106,121 @@ RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARG
"linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
*) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
esac && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} -p iggy-connectors
$PROFILE_FLAG && \
+ cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors
/app/iggy-connectors
+
+#
+# Generate the third-party license manifest. Required by ASF release policy:
+# convenience binaries that statically link third-party code MUST include the
+# full license text of every bundled crate inside the artifact. This one covers
+# the runtime only, which is all the slim image ships.
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+ ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary
+
+#
+# Fat builder - runtime binary plus every connector plugin cdylib, and a
+# license manifest covering all of them.
+#
+FROM --platform=$BUILDPLATFORM toolchain AS fat-builder
Review Comment:
nothing in CI builds this stage before a live publish - the docker-buildx
action has exactly one caller. so the first-ever 18-plugin lto build runs
inside the unchanged 60-minute timeout, and since docker-manifests gates on the
whole publish-docker matrix, one fat leg timing out also blocks manifests and
git tags for every other component in the release. a cheap pre-merge `--target
runtime-fat` build (like the server's runtime-prebuilt smoke build) plus one
timed arm64 run before merging would take the guessing out.
##########
scripts/ci/connector-plugins.sh:
##########
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+# Single source of truth for the connector plugin set: every workspace crate
+# under core/connectors/ that builds a cdylib (a .so the iggy-connectors
+# runtime loads via dlopen). Derived from `cargo metadata` so a newly added
+# connector is picked up automatically by every consumer:
+#
+# - .github/workflows/_build_rust_artifacts.yml (edge tarball plugin list)
+# - core/connectors/runtime/Dockerfile (fat image build + bundle)
+# - .github/actions/utils/validate-third-party-licenses (license gate)
+#
+# Output modes (one plugin per line unless noted):
+# --names crate names (iggy_connector_postgres_sink)
+# --comma-names crate names, one CSV line
+# --package-flags cargo -p flags, one line (-p iggy_connector_... ...)
+# --manifests repo-relative Cargo.toml paths
+# --manifest-flags --manifest flags, one line (for third-party-licenses.sh)
+
+MODE="--names"
+if [[ $# -gt 0 ]]; then
+ MODE="$1"
+fi
+
+METADATA="$(cargo metadata --format-version 1 --no-deps)"
+WORKSPACE_ROOT="$(jq -r '.workspace_root' <<<"$METADATA")"
+
+# cdylib packages whose manifest lives under core/connectors/. The path guard
+# keeps a future non-connector cdylib elsewhere in the workspace out of the
set.
+NAMES="$(jq -r --arg root "$WORKSPACE_ROOT" '
+ .packages[]
+ | select(.manifest_path | startswith($root + "/core/connectors/"))
+ | select(.targets[] | .kind[] == "cdylib")
+ | .name
+' <<<"$METADATA" | sort -u)"
+
+if [[ -z "$NAMES" ]]; then
+ echo "connector-plugins: no cdylib plugin crates found under
core/connectors/" >&2
+ exit 1
+fi
+
+# Map each crate name back to its repo-relative manifest path.
+manifest_for() {
+ jq -r --arg root "$WORKSPACE_ROOT" --arg name "$1" '
+ .packages[]
+ | select(.name == $name)
+ | .manifest_path
+ | ltrimstr($root + "/")
+ ' <<<"$METADATA"
+}
+
+case "$MODE" in
+ --names)
+ echo "$NAMES"
+ ;;
+ --comma-names)
+ paste -sd, - <<<"$NAMES"
+ ;;
+ --package-flags)
Review Comment:
`--package-flags` has no callers anywhere - the dockerfile and the workflow
both build their own `-p` flags from `--names`. drop it (it's also the only
thing here that needs bash 4's mapfile).
##########
.github/workflows/_build_rust_artifacts.yml:
##########
@@ -46,8 +46,8 @@ on:
connector_plugins:
type: string
required: false
- default:
"iggy_connector_elasticsearch_sink,iggy_connector_elasticsearch_source,iggy_connector_iceberg_sink,iggy_connector_postgres_sink,iggy_connector_postgres_source,iggy_connector_quickwit_sink,iggy_connector_random_source,iggy_connector_s3_sink,iggy_connector_stdout_sink,iggy_connector_surrealdb_sink"
- description: "Comma-separated list of connector plugin crates to build
as shared libraries"
+ default: ""
Review Comment:
emptying the default makes `connector_plugins` unreachable - this workflow
is workflow_call-only and the sole caller doesn't pass it, so the override
branch below is dead on arrival. simpler to drop the input and the if/else and
always derive; that also removes the one way the tarball's plugin set could
disagree with the fat image's.
##########
scripts/ci/connector-plugins.sh:
##########
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+# Single source of truth for the connector plugin set: every workspace crate
+# under core/connectors/ that builds a cdylib (a .so the iggy-connectors
+# runtime loads via dlopen). Derived from `cargo metadata` so a newly added
+# connector is picked up automatically by every consumer:
+#
+# - .github/workflows/_build_rust_artifacts.yml (edge tarball plugin list)
+# - core/connectors/runtime/Dockerfile (fat image build + bundle)
+# - .github/actions/utils/validate-third-party-licenses (license gate)
Review Comment:
the consumer list is missing one - the pre-merge rust action derives an
overlapping set inline (bin-or-cdylib, no path guard), so "single source of
truth" oversells it a bit. worth a cross-reference comment there or a mode it
can call.
##########
.github/actions/utils/docker-buildx/action.yml:
##########
@@ -46,6 +46,10 @@ inputs:
description: "Single platform to build (e.g., linux/amd64). If set, builds
only this platform without QEMU. Leave empty for multi-arch build."
required: false
default: ""
+ target:
Review Comment:
nothing records the flavor in image metadata - fat and slim get
byte-identical OCI labels, so a pulled `-slim` image can't be told apart from
the fat one by inspection. an `org.opencontainers.image.variant` label keyed
off the target would fix that.
##########
scripts/ci/connector-plugins.sh:
##########
@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euo pipefail
+
+# Single source of truth for the connector plugin set: every workspace crate
+# under core/connectors/ that builds a cdylib (a .so the iggy-connectors
+# runtime loads via dlopen). Derived from `cargo metadata` so a newly added
+# connector is picked up automatically by every consumer:
+#
+# - .github/workflows/_build_rust_artifacts.yml (edge tarball plugin list)
+# - core/connectors/runtime/Dockerfile (fat image build + bundle)
+# - .github/actions/utils/validate-third-party-licenses (license gate)
+#
+# Output modes (one plugin per line unless noted):
+# --names crate names (iggy_connector_postgres_sink)
+# --comma-names crate names, one CSV line
+# --package-flags cargo -p flags, one line (-p iggy_connector_... ...)
+# --manifests repo-relative Cargo.toml paths
+# --manifest-flags --manifest flags, one line (for third-party-licenses.sh)
+
+MODE="--names"
+if [[ $# -gt 0 ]]; then
+ MODE="$1"
+fi
+
+METADATA="$(cargo metadata --format-version 1 --no-deps)"
+WORKSPACE_ROOT="$(jq -r '.workspace_root' <<<"$METADATA")"
+
+# cdylib packages whose manifest lives under core/connectors/. The path guard
+# keeps a future non-connector cdylib elsewhere in the workspace out of the
set.
+NAMES="$(jq -r --arg root "$WORKSPACE_ROOT" '
+ .packages[]
+ | select(.manifest_path | startswith($root + "/core/connectors/"))
Review Comment:
with this guard, any future cdylib under core/connectors/ (a test fixture,
an example) auto-ships in the default image and the edge tarball. all 18 names
match `iggy_connector_*` - a name-prefix filter would bound the set tighter
than the directory does.
##########
.github/config/publish.yml:
##########
@@ -96,9 +96,24 @@ components:
platforms: ["linux/amd64", "linux/arm64"]
version_file: "core/connectors/runtime/Cargo.toml"
version_regex: '(?m)^\s*version\s*=\s*"([^"]+)"'
- # Image ships only the runtime binary; plugin .so files are not bundled.
+ # Two published flavors from one Dockerfile. The fat flavor (empty suffix,
+ # runtime-fat stage) bundles every connector plugin and is the default;
+ # -slim (runtime stage) ships the runtime binary only. Each flavor keeps
+ # its own per-arch digest set and manifest tags. Components without a
+ # flavors list build their single default stage.
+ flavors:
+ - suffix: ""
+ target: runtime-fat
Review Comment:
tiny naming thing: `target` here means dockerfile stage, but everywhere else
in this repo's CI `target` means the rust triple. `stage` would read clearer
(the docker-buildx input should stay `target` since it matches
build-push-action).
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
