This is an automated email from the ASF dual-hosted git repository.
kevinjqliu pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/main by this push:
new eb8fe538f0 Add Flink Quickstart docker image (#15124)
eb8fe538f0 is described below
commit eb8fe538f0dcec74c9db5cb3c718185241836577
Author: Robin Moffatt <[email protected]>
AuthorDate: Mon Feb 16 18:27:22 2026 +0000
Add Flink Quickstart docker image (#15124)
---
docker/iceberg-flink-quickstart/Dockerfile | 49 +++++++
docker/iceberg-flink-quickstart/README.md | 79 ++++++++++++
docker/iceberg-flink-quickstart/docker-compose.yml | 142 +++++++++++++++++++++
3 files changed, 270 insertions(+)
diff --git a/docker/iceberg-flink-quickstart/Dockerfile
b/docker/iceberg-flink-quickstart/Dockerfile
new file mode 100644
index 0000000000..cdfa156d53
--- /dev/null
+++ b/docker/iceberg-flink-quickstart/Dockerfile
@@ -0,0 +1,49 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Version arguments - can be overridden at build time
+ARG FLINK_VERSION=2.0
+
+FROM apache/flink:${FLINK_VERSION}-java21
+
+SHELL ["/bin/bash", "-c"]
+
+# Redeclare ARG variables after FROM to make them available in subsequent
layers
+ARG ICEBERG_FLINK_RUNTIME_VERSION=2.0
+ARG ICEBERG_VERSION=1.10.1
+ARG HADOOP_VERSION=3.4.2
+
+# Switch to flink user for installation
+USER flink
+
+WORKDIR /opt/flink
+
+# Install Iceberg Flink runtime and AWS bundle JARs
+RUN echo "-> Install JARs: Dependencies for Iceberg" && \
+ mkdir -p ./lib/iceberg && pushd $_ && \
+ curl -fO
https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-flink-runtime-${ICEBERG_FLINK_RUNTIME_VERSION}/${ICEBERG_VERSION}/iceberg-flink-runtime-${ICEBERG_FLINK_RUNTIME_VERSION}-${ICEBERG_VERSION}.jar
&& \
+ curl -fO
https://repo.maven.apache.org/maven2/org/apache/iceberg/iceberg-aws-bundle/${ICEBERG_VERSION}/iceberg-aws-bundle-${ICEBERG_VERSION}.jar
&& \
+ popd
+
+# Install Hadoop client JARs (API + shaded runtime bundle)
+RUN echo "-> Install JARs: Hadoop" && \
+ mkdir -p ./lib/hadoop && pushd $_ && \
+ curl -fO
https://repo.maven.apache.org/maven2/org/apache/hadoop/hadoop-client-api/${HADOOP_VERSION}/hadoop-client-api-${HADOOP_VERSION}.jar
&& \
+ curl -fO
https://repo.maven.apache.org/maven2/org/apache/hadoop/hadoop-client-runtime/${HADOOP_VERSION}/hadoop-client-runtime-${HADOOP_VERSION}.jar
&& \
+ popd
diff --git a/docker/iceberg-flink-quickstart/README.md
b/docker/iceberg-flink-quickstart/README.md
new file mode 100644
index 0000000000..c7243cac31
--- /dev/null
+++ b/docker/iceberg-flink-quickstart/README.md
@@ -0,0 +1,79 @@
+<!--
+ - Licensed to the Apache Software Foundation (ASF) under one
+ - or more contributor license agreements. See the NOTICE file
+ - distributed with this work for additional information
+ - regarding copyright ownership. The ASF licenses this file
+ - to you under the Apache License, Version 2.0 (the
+ - "License"); you may not use this file except in compliance
+ - with the License. You may obtain a copy of the License at
+ -
+ - http://www.apache.org/licenses/LICENSE-2.0
+ -
+ - Unless required by applicable law or agreed to in writing,
+ - software distributed under the License is distributed on an
+ - "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ - KIND, either express or implied. See the License for the
+ - specific language governing permissions and limitations
+ - under the License.
+ -->
+
+# Iceberg Flink Quickstart Docker Image
+
+A pre-configured Apache Flink image with Apache Iceberg dependencies for
quickly getting started with Iceberg on Flink.
+
+## Overview
+
+This Docker image extends the official Apache Flink image to include:
+
+- Iceberg Flink runtime
+- Iceberg AWS bundle for S3/Glue support
+- Minimal Hadoop dependencies necessary for Flink
+
+## Build Arguments
+
+The following build arguments can be customized when building the image:
+
+| Argument | Default | Description |
+|----------|---------|-------------|
+| `FLINK_VERSION` | `2.0` | Apache Flink version |
+| `ICEBERG_FLINK_RUNTIME_VERSION` | `2.0` | Iceberg Flink runtime version |
+| `ICEBERG_VERSION` | `1.10.1` | Apache Iceberg version |
+| `HADOOP_VERSION` | `3.4.2` | Apache Hadoop version |
+
+## Building Locally
+
+To build the image locally with default versions:
+
+```bash
+docker build -t apache/iceberg-flink-quickstart
docker/iceberg-flink-quickstart/
+```
+
+To build with custom versions:
+
+```bash
+docker build \
+ --build-arg FLINK_VERSION=2.0 \
+ --build-arg ICEBERG_VERSION=1.10.1 \
+ -t apache/iceberg-flink-quickstart \
+ docker/iceberg-flink-quickstart/
+```
+
+## Usage
+
+The easiest way to get started is using the quickstart docker-compose file
from the repository root:
+
+```bash
+docker compose -f docker/iceberg-flink-quickstart/docker-compose.yml up -d
--build
+```
+
+Then connect to Flink SQL client:
+
+```bash
+docker exec -it jobmanager ./bin/sql-client.sh
+```
+
+To stop the stack:
+
+```bash
+docker compose -f docker/iceberg-flink-quickstart/docker-compose.yml down
+```
diff --git a/docker/iceberg-flink-quickstart/docker-compose.yml
b/docker/iceberg-flink-quickstart/docker-compose.yml
new file mode 100644
index 0000000000..955d06a504
--- /dev/null
+++ b/docker/iceberg-flink-quickstart/docker-compose.yml
@@ -0,0 +1,142 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# Flink Quickstart with Apache Iceberg
+#
+# Usage:
+# docker compose -f docker/iceberg-flink-quickstart/docker-compose.yml up -d
--build
+#
+# Connect to SQL client:
+# docker exec -it jobmanager ./bin/sql-client.sh
+services:
+ # Flink JobManager
+ jobmanager:
+ build: .
+ hostname: jobmanager
+ container_name: jobmanager
+ depends_on:
+ iceberg-rest:
+ condition: service_healthy
+ networks:
+ iceberg_net:
+ ports:
+ - "8081:8081"
+ command: jobmanager
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:8081/overview";]
+ start_period: 30s
+ interval: 5s
+ timeout: 3s
+ retries: 5
+ environment:
+ FLINK_PROPERTIES: |
+ jobmanager.rpc.address: jobmanager
+ taskmanager.numberOfTaskSlots: 2
+ parallelism.default: 2
+ AWS_REGION: us-east-1
+ AWS_ACCESS_KEY_ID: admin
+ AWS_SECRET_ACCESS_KEY: password
+ S3_ENDPOINT: http://minio:9000
+
+ # Flink TaskManager
+ taskmanager:
+ build: .
+ hostname: taskmanager
+ depends_on:
+ jobmanager:
+ condition: service_healthy
+ networks:
+ iceberg_net:
+ command: taskmanager
+ deploy:
+ replicas: 1
+ environment:
+ FLINK_PROPERTIES: |
+ jobmanager.rpc.address: jobmanager
+ taskmanager.numberOfTaskSlots: 2
+ parallelism.default: 2
+ AWS_REGION: us-east-1
+ AWS_ACCESS_KEY_ID: admin
+ AWS_SECRET_ACCESS_KEY: password
+ S3_ENDPOINT: http://minio:9000
+
+ # Iceberg REST Catalog
+ iceberg-rest:
+ image: apache/iceberg-rest-fixture
+ hostname: iceberg-rest
+ depends_on:
+ create-bucket:
+ condition: service_completed_successfully
+ networks:
+ iceberg_net:
+ ports:
+ - "8181:8181"
+ healthcheck:
+ test: ["CMD", "curl", "-f", "http://localhost:8181/v1/config";]
+ interval: 5s
+ timeout: 3s
+ retries: 5
+ environment:
+ AWS_REGION: us-east-1
+ CATALOG_WAREHOUSE: s3://warehouse/
+ CATALOG_IO__IMPL: org.apache.iceberg.aws.s3.S3FileIO
+ CATALOG_S3_ENDPOINT: http://minio:9000
+ CATALOG_S3_ACCESS__KEY__ID: admin
+ CATALOG_S3_SECRET__ACCESS__KEY: password
+
+ # MinIO for S3-compatible object storage
+ minio:
+ image: minio/minio
+ hostname: minio
+ environment:
+ MINIO_ROOT_USER: admin
+ MINIO_ROOT_PASSWORD: password
+ MINIO_DOMAIN: minio
+ networks:
+ iceberg_net:
+ aliases:
+ - warehouse.minio
+ ports:
+ - "9000:9000"
+ - "9001:9001"
+ command: server /data --console-address ":9001"
+ healthcheck:
+ test: ["CMD", "mc", "ready", "local"]
+ interval: 5s
+ timeout: 5s
+ retries: 5
+
+ # Create the warehouse bucket
+ create-bucket:
+ image: minio/mc
+ depends_on:
+ minio:
+ condition: service_healthy
+ networks:
+ iceberg_net:
+ entrypoint: |
+ /bin/sh -c "
+ until (/usr/bin/mc alias set minio http://minio:9000 admin password) do
echo '...waiting...' && sleep 1; done;
+ /usr/bin/mc rm -r --force minio/warehouse;
+ /usr/bin/mc mb minio/warehouse;
+ /usr/bin/mc policy set public minio/warehouse;
+ "
+
+networks:
+ iceberg_net:
