mmodzelewski commented on code in PR #4533:
URL: https://github.com/apache/iggy/pull/4533#discussion_r4389765142
##########
core/connectors/runtime/Dockerfile:
##########
@@ -75,41 +116,117 @@ COPY . .
RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
--mount=type=cache,target=/app/target,id=cargo-target-${TARGETPLATFORM}-${LIBC}
\
- case "$TARGETPLATFORM:$LIBC" in \
- "linux/amd64:musl") RUST_TARGET="x86_64-unknown-linux-musl" ;; \
- "linux/arm64:musl") RUST_TARGET="aarch64-unknown-linux-musl" ;; \
- "linux/amd64:glibc") RUST_TARGET="x86_64-unknown-linux-gnu" ;; \
- "linux/arm64:glibc") RUST_TARGET="aarch64-unknown-linux-gnu" ;; \
- *) echo "Unsupported platform/libc combination: $TARGETPLATFORM/$LIBC" &&
exit 1 ;; \
- esac && \
+ RUST_TARGET="$(cat /rust-target)" && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors
$PROFILE_FLAG && \
+ cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors
/app/iggy-connectors
+
+#
+# Generate the third-party license manifest. Required by ASF release policy:
+# convenience binaries that statically link third-party code MUST include the
+# full license text of every bundled crate inside the artifact. This one covers
+# the runtime only, which is all the slim image ships.
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+ ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary
+
+#
+# Fat builder - runtime binary plus every connector plugin cdylib, and a
+# license manifest covering all of them.
+#
+FROM --platform=$BUILDPLATFORM toolchain AS fat-builder
+ARG PROFILE=release
+ARG TARGETPLATFORM
+ARG LIBC=glibc
+
+# Only this branch reads the derived plugin set, so adding a connector leaves
+# the slim branch's cooked dependency layer untouched. Two lists, because cargo
+# selects a plugin by package name while the artifact is lib<target name>.so.
+COPY --from=planner /app/connector-plugins.txt connector-plugins.txt
+COPY --from=planner /app/connector-packages.txt connector-packages.txt
+COPY --from=planner /app/connector-manifests.txt connector-manifests.txt
+
+#
+# Cook dependencies
+#
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/app/target,id=cargo-target-fat-${TARGETPLATFORM}-${LIBC}
\
+ RUST_TARGET="$(cat /rust-target)" && \
+ PLUGIN_FLAGS="$(cat connector-packages.txt)" && \
if [ "$PROFILE" = "debug" ]; then \
- cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors && \
- cp /app/target/${RUST_TARGET}/debug/iggy-connectors /app/iggy-connectors; \
+ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET}
--zigbuild \
+ --bin iggy-connectors && \
+ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET}
--zigbuild \
+ $PLUGIN_FLAGS; \
else \
- cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors
--release && \
- cp /app/target/${RUST_TARGET}/release/iggy-connectors
/app/iggy-connectors; \
+ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET}
--zigbuild --release \
+ --bin iggy-connectors && \
+ cargo chef cook --recipe-path recipe.json --target ${RUST_TARGET}
--zigbuild --release \
+ $PLUGIN_FLAGS; \
fi
+COPY . .
+
#
-# Generate third-party license manifest. Required by ASF release policy:
-# convenience binaries that statically link third-party code MUST include
-# the full license text of every bundled crate inside the artifact.
+# Build
#
RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
- TARGET="$(uname -m)-unknown-linux-musl" && \
- curl -sSfL
"https://github.com/EmbarkStudios/cargo-about/releases/download/0.9.0/cargo-about-0.9.0-${TARGET}.tar.gz";
\
- | tar -xz -C /usr/local/bin --strip-components=1
"cargo-about-0.9.0-${TARGET}/cargo-about" && \
- ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml --output /app/LICENSE-binary
+
--mount=type=cache,target=/app/target,id=cargo-target-fat-${TARGETPLATFORM}-${LIBC}
\
+ RUST_TARGET="$(cat /rust-target)" && \
+ PLUGIN_FLAGS="$(cat connector-packages.txt)" && \
+ if [ "$PROFILE" = "debug" ]; then PROFILE_DIR=debug; PROFILE_FLAG=""; else
PROFILE_DIR=release; PROFILE_FLAG="--release"; fi && \
+ cargo zigbuild --locked --target ${RUST_TARGET} --bin iggy-connectors
$PROFILE_FLAG && \
+ cargo zigbuild --locked --target ${RUST_TARGET} $PLUGIN_FLAGS
$PROFILE_FLAG && \
+ cp /app/target/${RUST_TARGET}/${PROFILE_DIR}/iggy-connectors
/app/iggy-connectors && \
+ mkdir -p /app/plugins && \
+ while IFS= read -r name; do \
+ [ -z "$name" ] && continue; \
+ so_file="/app/target/${RUST_TARGET}/${PROFILE_DIR}/lib${name}.so"; \
+ if [ ! -f "$so_file" ]; then echo "expected plugin artifact missing:
$so_file" >&2 && exit 1; fi; \
+ cp "$so_file" /app/plugins/ || exit 1; \
+ done < connector-plugins.txt
#
-# Final runtime - Debian trixie Slim
+# Generate the third-party license manifest covering the runtime plus every
+# bundled plugin closure. Same ASF policy as the slim manifest above; each
+# image ships the manifest that matches what it actually bundles, so the slim
+# image never over-declares plugin dependencies it does not carry.
#
-FROM debian:trixie-slim AS runtime
+RUN
--mount=type=cache,target=/usr/local/cargo/registry,id=cargo-registry-${TARGETPLATFORM}-${LIBC}
\
+
--mount=type=cache,target=/usr/local/cargo/git,id=cargo-git-${TARGETPLATFORM}-${LIBC}
\
+ MANIFEST_FLAGS="$(sed 's/^/--manifest /' connector-manifests.txt | tr '\n'
' ')" && \
+ ./scripts/ci/third-party-licenses.sh --generate --manifest
core/connectors/runtime/Cargo.toml $MANIFEST_FLAGS --output
/app/LICENSE-binary-fat
+
+#
+# Runtime base - Debian trixie Slim, shared by both published flavors.
+#
+FROM debian:trixie-slim AS runtime-base
Review Comment:
Critical: `runtime-base` installs no `ca-certificates`, so reqwest 0.13
`build()` fails with "No CA certificates were loaded from the system", even for
`http://` URLs. Seven plugins, `http_sink` among them, fail `open()`, and
`bigquery_sink` panics and aborts the runtime. Install `ca-certificates` with
`--no-install-recommends` here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
