This is an automated email from the ASF dual-hosted git repository.
github-merge-queue[bot] pushed a commit to branch release/v1.2
in repository https://gitbox.apache.org/repos/asf/texera.git
The following commit(s) were added to refs/heads/release/v1.2 by this push:
new ba12d74392 feat(deploy, v1.2): replace MinIO with RustFS as the
default object store (#8555)
ba12d74392 is described below
commit ba12d7439249e328bb1b1d758fcc110f046afa73
Author: Xuan Gu <[email protected]>
AuthorDate: Fri Sep 18 02:34:24 2026 +0000
feat(deploy, v1.2): replace MinIO with RustFS as the default object store
(#8555)
### What changes were proposed in this PR?
Backport of #8542 to `release/v1.2`. Fresh 1.2 deployments are broken by
the MinIO image removal: the released compose tarball fails at image
pull, the K8s lakekeeper-init job dies downloading `mc` from `dl.min.io`
(410 Gone), and CI on this branch fails at image pull.
Main adaptations for v1.2:
- The chart has no `_helpers.tpl` and no external-S3 support, so
templates reference the RustFS subchart's service/Secret directly,
including two deployment templates that hardcode the MinIO secret on
v1.2.
- lakekeeper-init creates the bucket from an `rc` init container instead
of downloading `mc` at runtime; CI uses the preinstalled AWS CLI.
- Test files are adjusted to v1.2's package layout; the
`StagedFileCleanupJob` change does not apply here.
Migration note: volume/claim names change (`minio_data` ->
`rustfs_data`, `minio-data-pvc` -> `rustfs-data-pvc`); no in-place data
migration for now.
### Any related issues, documentation, discussions?
Discussion in #3998.
Original change: #8542.
### How was this PR tested?
Compose and Helm render cleanly with no minio left. Tested the
single-node deployment.
### Was this PR authored or co-authored using generative AI tooling?
Generated-by: Claude Fable 5 (Claude Code)
---------
Co-authored-by: Ian Maxon <[email protected]>
Co-authored-by: Claude Fable 5 <[email protected]>
Co-authored-by: Xinyuan Lin <[email protected]>
Co-authored-by: Meng Wang <[email protected]>
---
.github/workflows/build.yml | 28 +++++---
amber/LICENSE-binary-python | 3 +-
.../texera/web/service/WorkflowService.scala | 2 +-
.../packaging/test_state_materialization_e2e.py | 2 +-
.../core/storage/iceberg/test_iceberg_document.py | 4 +-
.../pytexera/storage/test_large_binary_manager.py | 4 +-
bin/bootstrap-lakekeeper.sh | 2 +-
bin/k8s/Chart.yaml | 6 +-
bin/k8s/templates/external-names.yaml | 6 +-
bin/k8s/templates/file-service-deployment.yaml | 10 +--
bin/k8s/templates/gateway-routes.yaml | 10 +--
bin/k8s/templates/gateway.yaml | 12 ++--
bin/k8s/templates/lakekeeper-init-job.yaml | 63 ++++++++++++-----
...io-persistence.yaml => rustfs-persistence.yaml} | 8 +--
...workflow-computing-unit-manager-deployment.yaml | 10 +--
bin/k8s/values-development.yaml | 52 +++++++++-----
bin/k8s/values.yaml | 67 +++++++++++++-----
bin/local-dev/README.md | 2 +-
bin/single-node/.env | 10 +--
bin/single-node/README.md | 4 +-
bin/single-node/docker-compose.yml | 65 ++++++++++--------
build.sbt | 1 +
common/config/src/main/resources/storage.conf | 2 +-
common/workflow-core/build.sbt | 5 +-
.../texera/service/util/S3StorageClient.scala | 8 +--
.../texera/service/util/RustFSContainer.scala | 80 ++++++++++++++++++++++
.../texera/service/util/S3StorageTestBase.scala | 21 +++---
.../guide-for-developers.md | 6 +-
docs/getting-started/installing-using-docker.md | 4 +-
docs/getting-started/run-on-kubernetes.md | 4 +-
docs/overview.md | 2 +-
docs/tutorials/guide-to-launch-lakekeeper.md | 6 +-
file-service/build.sbt | 3 +-
file-service/src/main/resources/docker-compose.yml | 25 ++++---
.../{minio-config.yml => rustfs-config.yml} | 23 ++++---
.../org/apache/texera/service/MockLakeFS.scala | 60 ++++++++--------
36 files changed, 401 insertions(+), 219 deletions(-)
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
index 5e3ada2c31..2d468dac51 100644
--- a/.github/workflows/build.yml
+++ b/.github/workflows/build.yml
@@ -399,17 +399,19 @@ jobs:
run: psql -h localhost -U postgres -v DB_NAME=texera_db_for_test_cases
-f sql/texera_ddl.sql
env:
PGPASSWORD: postgres
- - name: Start MinIO
+ - name: Start RustFS
run: |
- docker run -d --name minio --network host \
- -e MINIO_ROOT_USER=texera_minio \
- -e MINIO_ROOT_PASSWORD=password \
- minio/minio:RELEASE.2025-02-28T09-55-16Z server /data
- for i in $(seq 1 3); do
- curl -sf http://localhost:9000/minio/health/live && break
- echo "Waiting for MinIO... (attempt $i)"
+ docker run -d --name rustfs --network host \
+ -e RUSTFS_ACCESS_KEY=texera_rustfs \
+ -e RUSTFS_SECRET_KEY=password \
+ -e RUSTFS_REGION=us-west-2 \
+ rustfs/rustfs:1.0.0-rc.6
+ for i in $(seq 1 15); do
+ curl -sf http://localhost:9000/health && break
+ echo "Waiting for RustFS... (attempt $i)"
sleep 1
done
+ curl -sf http://localhost:9000/health
- name: Start Lakekeeper
env:
LAKEKEEPER__PG_DATABASE_URL_READ:
postgres://postgres:postgres@localhost:5432/texera_lakekeeper
@@ -461,9 +463,13 @@ jobs:
LAKEKEEPER_BASE=${REST_URI%/catalog}
LAKEKEEPER_BASE=${LAKEKEEPER_BASE%/}
- docker run --rm --network host --entrypoint sh minio/mc -c \
- "mc alias set minio $S3_ENDPOINT $S3_USERNAME $S3_PASSWORD && \
- mc mb --ignore-existing minio/$S3_BUCKET"
+ # Bucket creation goes through the AWS CLI, which is preinstalled on
+ # the runner image. CreateBucket on an existing bucket is a no-op on
+ # RustFS, which keeps re-runs idempotent.
+ AWS_ACCESS_KEY_ID="$S3_USERNAME" \
+ AWS_SECRET_ACCESS_KEY="$S3_PASSWORD" \
+ AWS_DEFAULT_REGION="$S3_REGION" \
+ aws s3api create-bucket --bucket "$S3_BUCKET" --endpoint-url
"$S3_ENDPOINT"
curl -sf -X POST -H 'Content-Type: application/json' \
-d
'{"project-id":"00000000-0000-0000-0000-000000000000","project-name":"default"}'
\
"$LAKEKEEPER_BASE/management/v1/project" || true
diff --git a/amber/LICENSE-binary-python b/amber/LICENSE-binary-python
index a571897d52..11fd33977a 100644
--- a/amber/LICENSE-binary-python
+++ b/amber/LICENSE-binary-python
@@ -300,6 +300,7 @@ Dependencies under the BSD 3-Clause License
Python packages:
- cached-property==1.5.2
- click==8.4.2
+ - cloudpickle==3.1.2
- contourpy==1.3.3
- cycler==0.12.1
- fsspec==2025.9.0
@@ -365,7 +366,7 @@ Dependencies under the Python Software Foundation License
Python packages:
- aiohappyeyeballs==2.7.1
- matplotlib==3.11.1
- - typing-extensions==4.14.1
+ - typing-extensions==4.16.0
--------------------------------------------------------------------------------
Dependencies under the MIT-CMU License
diff --git
a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala
b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala
index f6c67159ae..b1cc506b5d 100644
--- a/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala
+++ b/amber/src/main/scala/org/apache/texera/web/service/WorkflowService.scala
@@ -349,7 +349,7 @@ class WorkflowService(
* 2. Clears URI references from the execution registry
* 3. Safely clears all result and console message documents
* 4. Expires Iceberg snapshots for runtime statistics
- * 5. Deletes this execution's large binaries from MinIO
+ * 5. Deletes this execution's large binaries from the object store
*
* @param eid The execution identity to clean up resources for
*/
diff --git
a/amber/src/test/python/core/architecture/packaging/test_state_materialization_e2e.py
b/amber/src/test/python/core/architecture/packaging/test_state_materialization_e2e.py
index cfc4f7f676..93508b6e9a 100644
---
a/amber/src/test/python/core/architecture/packaging/test_state_materialization_e2e.py
+++
b/amber/src/test/python/core/architecture/packaging/test_state_materialization_e2e.py
@@ -80,7 +80,7 @@ _WAREHOUSE_DIR =
tempfile.mkdtemp(prefix="texera-state-e2e-warehouse-")
@pytest.fixture(scope="module", autouse=True)
def sqlite_iceberg_catalog():
"""Inject a sqlite-backed SqlCatalog so the test runs without external
- iceberg infra (postgres/minio).
+ iceberg infra (postgres/rustfs).
Module-scoped so all tests in this file share one warehouse, and so
namespace creation only happens once. We save/restore the original
diff --git
a/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py
b/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py
index 5e61b4ed68..0b264b4508 100644
--- a/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py
+++ b/amber/src/test/python/core/storage/iceberg/test_iceberg_document.py
@@ -55,8 +55,8 @@ StorageConfig.initialize(
commit_batch_size=4096,
s3_endpoint="http://localhost:9000";,
s3_region="us-east-1",
- s3_auth_username="minioadmin",
- s3_auth_password="minioadmin",
+ s3_auth_username="rustfsadmin",
+ s3_auth_password="rustfsadmin",
s3_large_binaries_base_uri="s3://texera-large-binaries/objects/0/",
)
diff --git
a/amber/src/test/python/pytexera/storage/test_large_binary_manager.py
b/amber/src/test/python/pytexera/storage/test_large_binary_manager.py
index e1da5eee18..ae24df206a 100644
--- a/amber/src/test/python/pytexera/storage/test_large_binary_manager.py
+++ b/amber/src/test/python/pytexera/storage/test_large_binary_manager.py
@@ -46,8 +46,8 @@ def _init_storage_config():
commit_batch_size=1000,
s3_endpoint="http://localhost:9000";,
s3_region="us-east-1",
- s3_auth_username="minioadmin",
- s3_auth_password="minioadmin",
+ s3_auth_username="rustfsadmin",
+ s3_auth_password="rustfsadmin",
s3_large_binaries_base_uri=TEST_BASE_URI,
)
diff --git a/bin/bootstrap-lakekeeper.sh b/bin/bootstrap-lakekeeper.sh
index 17452e9f7d..3fd9f1d6a5 100755
--- a/bin/bootstrap-lakekeeper.sh
+++ b/bin/bootstrap-lakekeeper.sh
@@ -42,7 +42,7 @@
STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME="${STORAGE_ICEBERG_CATALOG_REST_WARE
STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET="${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET:-texera-iceberg}"
STORAGE_S3_REGION="${STORAGE_S3_REGION:-us-west-2}"
STORAGE_S3_ENDPOINT="${STORAGE_S3_ENDPOINT:-http://localhost:9000}";
-STORAGE_S3_AUTH_USERNAME="${STORAGE_S3_AUTH_USERNAME:-texera_minio}"
+STORAGE_S3_AUTH_USERNAME="${STORAGE_S3_AUTH_USERNAME:-texera_rustfs}"
STORAGE_S3_AUTH_PASSWORD="${STORAGE_S3_AUTH_PASSWORD:-password}"
# Lakekeeper binary — defaults to `lakekeeper` on $PATH (e.g. after
diff --git a/bin/k8s/Chart.yaml b/bin/k8s/Chart.yaml
index 548d9e531e..5bd5f1fa98 100644
--- a/bin/k8s/Chart.yaml
+++ b/bin/k8s/Chart.yaml
@@ -46,9 +46,9 @@ dependencies:
version: 16.5.6
repository: https://charts.bitnami.com/bitnami
- - name: minio
- version: 15.0.7
- repository: https://charts.bitnami.com/bitnami
+ - name: rustfs
+ version: 1.0.0-rc.6
+ repository: https://charts.rustfs.com
- name: lakefs
version: 1.8.1
diff --git a/bin/k8s/templates/external-names.yaml
b/bin/k8s/templates/external-names.yaml
index c79ce40f33..0e180bd37b 100644
--- a/bin/k8s/templates/external-names.yaml
+++ b/bin/k8s/templates/external-names.yaml
@@ -74,11 +74,11 @@ to access services in the main namespace using the same
service names.
) | nindent 0 }}
---
-{{/* MinIO ExternalName */}}
+{{/* RustFS ExternalName */}}
{{- include "external-name-service" (dict
- "name" (printf "%s-minio" .Release.Name)
+ "name" (printf "%s-rustfs-svc" .Release.Name)
"namespace" $workflowComputingUnitPoolNamespace
- "externalName" (printf "%s-minio.%s.svc.cluster.local" .Release.Name
$namespace)
+ "externalName" (printf "%s-rustfs-svc.%s.svc.cluster.local" .Release.Name
$namespace)
) | nindent 0 }}
---
diff --git a/bin/k8s/templates/file-service-deployment.yaml
b/bin/k8s/templates/file-service-deployment.yaml
index 6c9a4041f9..f56085737a 100644
--- a/bin/k8s/templates/file-service-deployment.yaml
+++ b/bin/k8s/templates/file-service-deployment.yaml
@@ -41,17 +41,17 @@ spec:
env:
# LakeFS & S3 Access
- name: STORAGE_S3_ENDPOINT
- value: http://{{ .Release.Name }}-minio:9000
+ value: http://{{ .Release.Name }}-rustfs-svc:9000
- name: STORAGE_S3_AUTH_USERNAME
valueFrom:
secretKeyRef:
- name: {{ .Release.Name }}-minio
- key: root-user
+ name: {{ .Release.Name }}-rustfs-secret
+ key: RUSTFS_ACCESS_KEY
- name: STORAGE_S3_AUTH_PASSWORD
valueFrom:
secretKeyRef:
- name: {{ .Release.Name }}-minio
- key: root-password
+ name: {{ .Release.Name }}-rustfs-secret
+ key: RUSTFS_SECRET_KEY
- name: STORAGE_LAKEFS_ENDPOINT
value: http://{{ .Release.Name }}-lakefs:8000/api/v1
- name: STORAGE_LAKEFS_AUTH_USERNAME
diff --git a/bin/k8s/templates/gateway-routes.yaml
b/bin/k8s/templates/gateway-routes.yaml
index 935d38bb07..ace2fc5ee7 100644
--- a/bin/k8s/templates/gateway-routes.yaml
+++ b/bin/k8s/templates/gateway-routes.yaml
@@ -159,24 +159,24 @@ spec:
port: 3001
{{- end }}
---
-# MinIO Route
-{{- if .Values.minio.gateway.enabled }}
+# RustFS Route
+{{- if .Values.rustfs.gateway.enabled }}
apiVersion: gateway.networking.k8s.io/v1
kind: HTTPRoute
metadata:
- name: texera-minio-route
+ name: texera-rustfs-route
namespace: {{ .Release.Namespace }}
spec:
parentRefs:
- name: {{ .Release.Name }}-gateway
hostnames:
- - {{ .Values.minio.gateway.hostname }}
+ - {{ .Values.rustfs.gateway.hostname }}
rules:
- matches:
- path:
type: PathPrefix
value: /
backendRefs:
- - name: {{ .Release.Name }}-minio
+ - name: {{ .Release.Name }}-rustfs-svc
port: 9000
{{- end }}
diff --git a/bin/k8s/templates/gateway.yaml b/bin/k8s/templates/gateway.yaml
index 868439a673..dc79e39d6d 100644
--- a/bin/k8s/templates/gateway.yaml
+++ b/bin/k8s/templates/gateway.yaml
@@ -59,23 +59,23 @@ spec:
mode: Terminate
certificateRefs:
- name: {{ (index .Values "gatewayConfig" "tlsSecretName") | default
(printf "%s-cert" .Release.Name) }}
- {{- if .Values.minio.gateway.enabled }}
- - name: minio-http
+ {{- if .Values.rustfs.gateway.enabled }}
+ - name: rustfs-http
protocol: HTTP
port: 80
- hostname: {{ .Values.minio.gateway.hostname }}
+ hostname: {{ .Values.rustfs.gateway.hostname }}
allowedRoutes:
namespaces:
from: Same
- - name: minio-https
+ - name: rustfs-https
protocol: HTTPS
port: 443
- hostname: {{ .Values.minio.gateway.hostname }}
+ hostname: {{ .Values.rustfs.gateway.hostname }}
allowedRoutes:
namespaces:
from: Same
tls:
mode: Terminate
certificateRefs:
- - name: {{ .Values.minio.gateway.tlsSecretName | default (printf
"%s-minio-tls" .Release.Name) }}
+ - name: {{ .Values.rustfs.gateway.tlsSecretName | default (printf
"%s-rustfs-tls" .Release.Name) }}
{{- end }}
diff --git a/bin/k8s/templates/lakekeeper-init-job.yaml
b/bin/k8s/templates/lakekeeper-init-job.yaml
index 0a3540b605..161638c896 100644
--- a/bin/k8s/templates/lakekeeper-init-job.yaml
+++ b/bin/k8s/templates/lakekeeper-init-job.yaml
@@ -28,22 +28,62 @@ spec:
name: {{ .Release.Name }}-lakekeeper-init
spec:
restartPolicy: Never
+ # Creates the Iceberg warehouse bucket before the warehouse is
registered.
+ # `rc` is RustFS's S3 client; using its image keeps the client out of the
+ # main container, which would otherwise have to fetch a binary at
runtime.
+ initContainers:
+ - name: create-bucket
+ image: {{ .Values.lakekeeperInit.bucketClientImage }}
+ env:
+ - name: STORAGE_S3_ENDPOINT
+ value: http://{{ .Release.Name }}-rustfs-svc:9000
+ - name: STORAGE_S3_AUTH_USERNAME
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Release.Name }}-rustfs-secret
+ key: RUSTFS_ACCESS_KEY
+ - name: STORAGE_S3_AUTH_PASSWORD
+ valueFrom:
+ secretKeyRef:
+ name: {{ .Release.Name }}-rustfs-secret
+ key: RUSTFS_SECRET_KEY
+ - name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET
+ value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }}
+ - name: STORAGE_ICEBERG_CATALOG_REST_REGION
+ value: {{ .Values.lakekeeperInit.warehouse.region | quote }}
+ command:
+ - /bin/sh
+ - -c
+ - |
+ set -e
+
+ echo "Waiting for the object store at ${STORAGE_S3_ENDPOINT}..."
+ until rc alias set local "${STORAGE_S3_ENDPOINT}" \
+ "${STORAGE_S3_AUTH_USERNAME}"
"${STORAGE_S3_AUTH_PASSWORD}" \
+ --region "${STORAGE_ICEBERG_CATALOG_REST_REGION}" \
+ --bucket-lookup path > /dev/null 2>&1; do
+ sleep 3
+ done
+
+ echo "Initializing object-store bucket
'${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'..."
+ rc bucket create --ignore-existing
"local/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}"
+ echo "Bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' is
ready."
containers:
- name: lakekeeper-init
image: alpine:3.19
env:
- name: STORAGE_S3_ENDPOINT
- value: http://{{ .Release.Name }}-minio:9000
+ value: http://{{ .Release.Name }}-rustfs-svc:9000
- name: STORAGE_S3_AUTH_USERNAME
valueFrom:
secretKeyRef:
- name: {{ .Release.Name }}-minio
- key: root-user
+ name: {{ .Release.Name }}-rustfs-secret
+ key: RUSTFS_ACCESS_KEY
- name: STORAGE_S3_AUTH_PASSWORD
valueFrom:
secretKeyRef:
- name: {{ .Release.Name }}-minio
- key: root-password
+ name: {{ .Release.Name }}-rustfs-secret
+ key: RUSTFS_SECRET_KEY
- name: STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET
value: {{ .Values.lakekeeperInit.warehouse.s3Bucket | quote }}
- name: STORAGE_ICEBERG_CATALOG_REST_REGION
@@ -62,9 +102,7 @@ spec:
- |
set -e
- apk add --no-cache curl ca-certificates wget
- wget -q https://dl.min.io/client/mc/release/linux-amd64/mc -O
/usr/local/bin/mc
- chmod +x /usr/local/bin/mc
+ apk add --no-cache curl ca-certificates
check_status() {
if [ "$1" -ge 200 ] && [ "$1" -lt 300 ]; then
@@ -85,14 +123,7 @@ spec:
sleep 3
done
- echo "Step 1: Initializing MinIO bucket
'${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'..."
- mc alias set minio "${STORAGE_S3_ENDPOINT}"
"${STORAGE_S3_AUTH_USERNAME}" "${STORAGE_S3_AUTH_PASSWORD}" || true
- if mc ls minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET} >
/dev/null 2>&1; then
- echo "MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'
already exists."
- else
- mc mb minio/${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}
- echo "MinIO bucket '${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}'
created successfully."
- fi
+ echo "Step 1: object-store bucket
'${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET}' was created by the create-bucket
init container."
echo "Step 2: Initializing default project..."
PROJECT_PAYLOAD="{\"project-id\":\"${LAKEKEEPER_PROJECT_ID}\",\"project-name\":\"${LAKEKEEPER_PROJECT_NAME}\"}"
diff --git a/bin/k8s/templates/minio-persistence.yaml
b/bin/k8s/templates/rustfs-persistence.yaml
similarity index 92%
rename from bin/k8s/templates/minio-persistence.yaml
rename to bin/k8s/templates/rustfs-persistence.yaml
index 94d22e5e3a..173f93481d 100644
--- a/bin/k8s/templates/minio-persistence.yaml
+++ b/bin/k8s/templates/rustfs-persistence.yaml
@@ -17,11 +17,11 @@
{{/* Define storage path configuration, please change it to your own path and
make sure the path exists with the right permission*/}}
{{/* This path only works for local-path storage class */}}
-{{- $hostBasePath := .Values.persistence.minioHostLocalPath }}
+{{- $hostBasePath := .Values.persistence.rustfsHostLocalPath }}
-{{- if .Values.minio.persistence.enabled }}
-{{- $name := "minio" }}
-{{- $persistence := .Values.minio.persistence }}
+{{- if .Values.rustfs.persistence.enabled }}
+{{- $name := "rustfs" }}
+{{- $persistence := .Values.rustfs.persistence }}
{{- $volumeName := printf "%s-data-pv" $name }}
{{- $claimName := printf "%s-data-pvc" $name }}
{{- $storageClass := $persistence.storageClass | default "local-path" }}
diff --git a/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml
b/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml
index 7a0185cd46..299297f133 100644
--- a/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml
+++ b/bin/k8s/templates/workflow-computing-unit-manager-deployment.yaml
@@ -81,17 +81,17 @@ spec:
value: http://{{ .Values.fileService.name
}}-svc:9092/api/dataset/did/upload
# S3 Access (for R UDF large binary support)
- name: STORAGE_S3_ENDPOINT
- value: http://{{ .Release.Name }}-minio:9000
+ value: http://{{ .Release.Name }}-rustfs-svc:9000
- name: STORAGE_S3_AUTH_USERNAME
valueFrom:
secretKeyRef:
- name: {{ .Release.Name }}-minio
- key: root-user
+ name: {{ .Release.Name }}-rustfs-secret
+ key: RUSTFS_ACCESS_KEY
- name: STORAGE_S3_AUTH_PASSWORD
valueFrom:
secretKeyRef:
- name: {{ .Release.Name }}-minio
- key: root-password
+ name: {{ .Release.Name }}-rustfs-secret
+ key: RUSTFS_SECRET_KEY
# LakeFS Access (should be removed in production environment)
- name: STORAGE_LAKEFS_ENDPOINT
value: http://{{ .Release.Name }}-lakefs.{{ .Release.Namespace
}}:8000/api/v1
diff --git a/bin/k8s/values-development.yaml b/bin/k8s/values-development.yaml
index aa64d6ecbc..db82a3b417 100644
--- a/bin/k8s/values-development.yaml
+++ b/bin/k8s/values-development.yaml
@@ -22,7 +22,7 @@ texera:
imageTag: latest
global:
- # Required by Bitnami sub-charts (postgresql, minio) to allow custom images
+ # Required by the Bitnami postgresql sub-chart to allow custom images
security:
allowInsecureImages: true
@@ -34,10 +34,10 @@ global:
# - false: PVCs will remain after uninstall to preserve the data
persistence:
removeAfterUninstall: true
- minioHostLocalPath: ""
+ rustfsHostLocalPath: ""
postgresqlHostLocalPath: ""
-# Part 1: the configuration of Postgres, Minio and LakeFS
+# Part 1: the configuration of Postgres, RustFS and LakeFS
postgresql:
image:
repository: groonga/pgroonga
@@ -68,34 +68,50 @@ postgresql:
initdb:
scriptsConfigMap: "postgresql-init-script"
-minio:
- mode: standalone
+rustfs:
+ replicaCount: 1
+ drivesPerNode: 1
+ mode:
+ standalone:
+ enabled: true
+ existingClaim:
+ dataClaim: "rustfs-data-pvc"
+ distributed:
+ enabled: false
image:
- repository: bitnamilegacy/minio
- tag: 2025.3.12-debian-12-r0
+ rustfs:
+ repository: rustfs/rustfs
+ tag: 1.0.0-rc.6
resources:
requests:
memory: "256Mi"
limits:
memory: "256Mi"
+ ingress:
+ enabled: false
gateway:
enabled: false
- hostname: "" # the url for the minio, e.g. "minio.example.com"
- tlsSecretName: "" # e.g. "minio-tls-secret"
- auth:
- rootUser: texera_minio
- rootPassword: password
+ hostname: "" # the url for the object store, e.g. "rustfs.example.com"
+ tlsSecretName: "" # e.g. "rustfs-tls-secret"
+ secret:
+ rustfs:
+ access_key: texera_rustfs
+ secret_key: password
+ config:
+ rustfs:
+ region: us-west-2
+ obs_log_directory: ""
+ log_level: warn
service:
- # In production, use ClusterIP to avoid exposing the minio to the internet
+ # In production, use ClusterIP to avoid exposing the object store to the
internet
# type: ClusterIP
type: NodePort
- nodePorts:
- api: 31000
+ endpoint:
+ nodePort: 31000
persistence:
enabled: true
size: 20Gi
storageClass: local-path
- existingClaim: "minio-data-pvc"
lakefs:
secrets:
@@ -113,12 +129,12 @@ lakefs:
blockstore:
type: s3
s3:
- endpoint: http://texera-minio:9000
+ endpoint: http://texera-rustfs-svc:9000
pre_signed_expiry: 15m
pre_signed_endpoint: http://localhost:31000
force_path_style: true
credentials:
- access_key_id: texera_minio
+ access_key_id: texera_rustfs
secret_access_key: password
# Part2: configurations of Texera-related micro services
diff --git a/bin/k8s/values.yaml b/bin/k8s/values.yaml
index 313930d10a..d7faf7b946 100644
--- a/bin/k8s/values.yaml
+++ b/bin/k8s/values.yaml
@@ -22,7 +22,7 @@ texera:
imageTag: 1.2.0-incubating
global:
- # Required by Bitnami sub-charts (postgresql, minio) to allow custom images
+ # Required by the Bitnami postgresql sub-chart to allow custom images
security:
allowInsecureImages: true
@@ -34,10 +34,10 @@ global:
# - false: PVCs will remain after uninstall to preserve the data
persistence:
removeAfterUninstall: true
- minioHostLocalPath: ""
+ rustfsHostLocalPath: ""
postgresqlHostLocalPath: ""
-# Part 1: the configuration of Postgres, Minio and LakeFS
+# Part 1: the configuration of Postgres, RustFS and LakeFS
postgresql:
image:
repository: groonga/pgroonga
@@ -65,29 +65,56 @@ postgresql:
initdb:
scriptsConfigMap: "postgresql-init-script"
-minio:
- mode: standalone
+rustfs:
+ # Single-node object store, matching the previous standalone MinIO. The
+ # upstream chart defaults to a 4-replica distributed cluster; Texera's
+ # in-cluster store is a convenience for single-node installs, so keep it to
+ # one pod and one drive.
+ replicaCount: 1
+ drivesPerNode: 1
+ mode:
+ standalone:
+ enabled: true
+ existingClaim:
+ dataClaim: "rustfs-data-pvc"
+ distributed:
+ enabled: false
image:
- repository: bitnamilegacy/minio
- tag: 2025.3.12-debian-12-r0
+ rustfs:
+ repository: rustfs/rustfs
+ tag: 1.0.0-rc.6
+ # Texera fronts the object store with Envoy Gateway (the `gateway` block
+ # below), not with an Ingress -- the upstream chart turns one on by default.
+ ingress:
+ enabled: false
gateway:
enabled: false
- hostname: "" # the url for the minio, e.g. "minio.example.com"
- tlsSecretName: "" # e.g. "minio-tls-secret"
- auth:
- rootUser: texera_minio
- rootPassword: password
+ hostname: "" # the url for the object store, e.g. "rustfs.example.com"
+ tlsSecretName: "" # e.g. "rustfs-tls-secret"
+ secret:
+ rustfs:
+ access_key: texera_rustfs
+ secret_key: password
+ config:
+ rustfs:
+ # Must match the lakefs/lakekeeperInit region: it is part of the SigV4
scope.
+ region: us-west-2
+ # Empty => log to stdout (`kubectl logs`) instead of a log PVC.
+ obs_log_directory: ""
+ log_level: warn
service:
- # In production, use ClusterIP to avoid exposing the minio to the internet
+ # In production, use ClusterIP to avoid exposing the object store to the
internet
# type: ClusterIP
type: NodePort
- nodePorts:
- api: 31000
+ endpoint:
+ nodePort: 31000
+ # The chart's own PVC template is unused here: Texera provisions the claim
+ # itself (templates/rustfs-persistence.yaml) so that the local-path
+ # host directory stays under persistence.rustfsHostLocalPath.
persistence:
enabled: true
size: 20Gi
storageClass: local-path
- existingClaim: "minio-data-pvc"
lakefs:
secrets:
@@ -105,12 +132,12 @@ lakefs:
blockstore:
type: s3
s3:
- endpoint: http://texera-minio:9000
+ endpoint: http://texera-rustfs-svc:9000
pre_signed_expiry: 15m
pre_signed_endpoint: http://localhost:31000
force_path_style: true
credentials:
- access_key_id: texera_minio
+ access_key_id: texera_rustfs
secret_access_key: password
lakekeeper:
@@ -137,6 +164,10 @@ lakekeeper:
lakekeeperInit:
enabled: true
+ # S3 client image used by the create-bucket init container. `rc` is RustFS's
+ # client; any image whose entrypoint is an `mc`-style client with
+ # `alias set` / `bucket create` would do.
+ bucketClientImage: rustfs/rc:v0.1.35
defaultProject:
id: "00000000-0000-0000-0000-000000000000"
name: default
diff --git a/bin/local-dev/README.md b/bin/local-dev/README.md
index a560f9098b..56df4154fa 100644
--- a/bin/local-dev/README.md
+++ b/bin/local-dev/README.md
@@ -42,7 +42,7 @@ bin/local-dev/
├── main.sh shell engine — sbt builds, service
lifecycle, port checks
├── tui.py Textual dashboard surfaced by
`bin/local-dev.sh -i`
├── docker-compose.override.yml overlay on top of
bin/single-node/docker-compose.yml
-│ (host-LAN-IP MinIO endpoint, Lakekeeper
warehouse, etc.)
+│ (host-LAN-IP RustFS endpoint, Lakekeeper
warehouse, etc.)
└── tests/
├── test_local_dev_sh.sh bash smoke: license header, syntax, version,
--help,
│ error-on-bad-input, regression guards
diff --git a/bin/single-node/.env b/bin/single-node/.env
index 555e14db7d..4a8ba397bf 100644
--- a/bin/single-node/.env
+++ b/bin/single-node/.env
@@ -18,7 +18,7 @@
# Public host and ports exposed by the deployment
TEXERA_HOST=http://localhost
TEXERA_PORT=8080
-MINIO_PORT=9000
+RUSTFS_PORT=9000
# Log level for all Texera services (valid values: ERROR, WARN, INFO, DEBUG)
TEXERA_SERVICE_LOG_LEVEL=INFO
@@ -36,8 +36,8 @@ USER_SYS_ADMIN_PASSWORD=texera
POSTGRES_USER=texera
POSTGRES_PASSWORD=password
-# S3 (MinIO) credentials
-STORAGE_S3_AUTH_USERNAME=texera_minio
+# S3 (RustFS) credentials
+STORAGE_S3_AUTH_USERNAME=texera_rustfs
STORAGE_S3_AUTH_PASSWORD=password
# LakeFS server configuration
@@ -46,7 +46,7 @@ LAKEFS_INSTALLATION_ACCESS_KEY_ID=AKIAIOSFOLKFSSAMPLES
LAKEFS_INSTALLATION_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
LAKEFS_BLOCKSTORE_TYPE=s3
LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE=true
-LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://texera-minio:9000
+LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://texera-rustfs:9000
LAKEFS_AUTH_ENCRYPT_SECRET_KEY=random_string_for_lakefs
LAKEFS_LOGGING_LEVEL=INFO
LAKEFS_STATS_ENABLED=1
@@ -60,7 +60,7 @@ LAKEKEEPER__PG_ENCRYPTION_KEY=texera_key
LAKEKEEPER_BASE_URI=http://texera-lakekeeper:8181
# Texera storage endpoints
-STORAGE_S3_ENDPOINT=http://texera-minio:9000
+STORAGE_S3_ENDPOINT=http://texera-rustfs:9000
STORAGE_S3_REGION=us-west-2
STORAGE_LAKEFS_ENDPOINT=http://texera-lakefs:8000/api/v1
STORAGE_JDBC_URL=jdbc:postgresql://texera-postgres:5432/texera_db?currentSchema=texera_db,public
diff --git a/bin/single-node/README.md b/bin/single-node/README.md
index 62d4834297..77e0f63407 100644
--- a/bin/single-node/README.md
+++ b/bin/single-node/README.md
@@ -153,11 +153,11 @@ All changes below are to the `.env` file in the
installation folder, unless othe
### Run Texera on other ports
By default, Texera uses:
- Port 8080 for its web service
-- Port 9000 for its MinIO storage service
+- Port 9000 for its RustFS storage service
To change these ports, open the `.env` file and update the corresponding
variables:
- For the web service port (8080): change `TEXERA_PORT=8080` to your desired
port, e.g., `TEXERA_PORT=8081`.
-- For the MinIO port (9000): change `MINIO_PORT=9000` to your desired port,
e.g., `MINIO_PORT=9001`.
+- For the RustFS port (9000): change `RUSTFS_PORT=9000` to your desired port,
e.g., `RUSTFS_PORT=9001`.
### Change the locations of Texera data
By default, Docker manages Texera's data locations. To change them to your own
locations:
diff --git a/bin/single-node/docker-compose.yml
b/bin/single-node/docker-compose.yml
index 64247d9d66..52f3a89a0e 100644
--- a/bin/single-node/docker-compose.yml
+++ b/bin/single-node/docker-compose.yml
@@ -18,33 +18,43 @@
name: texera-single-node
services:
# Part1: Specification of the storage services used by Texera
- # MinIO is an S3-compatible object storage used to store datasets and files.
- minio:
- image: minio/minio:RELEASE.2025-02-28T09-55-16Z
- container_name: texera-minio
+ # RustFS is an S3-compatible object storage used to store datasets and files.
+ rustfs:
+ image: rustfs/rustfs:1.0.0-rc.6
+ container_name: texera-rustfs
ports:
- - "${MINIO_PORT:-9000}:9000"
+ - "${RUSTFS_PORT:-9000}:9000"
env_file:
- .env
environment:
- - MINIO_ROOT_USER=${STORAGE_S3_AUTH_USERNAME}
- - MINIO_ROOT_PASSWORD=${STORAGE_S3_AUTH_PASSWORD}
+ - RUSTFS_ACCESS_KEY=${STORAGE_S3_AUTH_USERNAME}
+ - RUSTFS_SECRET_KEY=${STORAGE_S3_AUTH_PASSWORD}
+ # Must match STORAGE_S3_REGION: the region is part of the SigV4 scope,
and
+ # LakeFS sends its blockstore region on every request.
+ - RUSTFS_REGION=${STORAGE_S3_REGION}
+ - RUSTFS_CONSOLE_ENABLE=true
+ - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
+ # Empty log directory => log to stdout, so `docker compose logs rustfs`
+ # works and no second volume is needed. `warn` keeps that readable; the
+ # server is very chatty at info.
+ - RUSTFS_OBS_LOG_DIRECTORY=
+ - RUSTFS_OBS_LOGGER_LEVEL=warn
volumes:
- - minio_data:/data
- command: server --console-address ":9001" /data
+ - rustfs_data:/data
healthcheck:
- test: ["CMD", "curl", "-sf", "http://localhost:9000/minio/health/live";]
+ test: ["CMD", "curl", "-sf", "http://localhost:9000/health";]
interval: 5s
timeout: 3s
retries: 10
# This job creates the S3 bucket used by the Iceberg warehouse that the
- # Lakekeeper service manages.
- minio-init:
- image: minio/mc:RELEASE.2025-05-21T01-59-54Z
- container_name: texera-minio-init
+ # Lakekeeper service manages. `rc` is RustFS's S3 client, the counterpart of
+ # the `mc` client this used to run.
+ rustfs-init:
+ image: rustfs/rc:v0.1.35
+ container_name: texera-rustfs-init
depends_on:
- minio:
+ rustfs:
condition: service_healthy
env_file:
- .env
@@ -53,9 +63,10 @@ services:
command:
- |
set -e
- mc alias set local "$$STORAGE_S3_ENDPOINT"
"$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD"
- mc mb --ignore-existing
"local/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET"
- echo "MinIO bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' is
ready."
+ rc alias set local "$$STORAGE_S3_ENDPOINT"
"$$STORAGE_S3_AUTH_USERNAME" "$$STORAGE_S3_AUTH_PASSWORD" \
+ --region "$$STORAGE_S3_REGION" --bucket-lookup path
+ rc bucket create --ignore-existing
"local/$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET"
+ echo "RustFS bucket '$$STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET' is
ready."
# PostgreSQL with PGroonga extension for full-text search.
# Used by lakeFS and Texera's metadata storage.
@@ -83,13 +94,13 @@ services:
depends_on:
postgres:
condition: service_healthy
- minio:
+ rustfs:
condition: service_started
env_file:
- .env
environment:
- # This port also need to be changed if the port of MinIO service is
changed
- -
LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT=${TEXERA_HOST}:${MINIO_PORT:-9000}
+ # This port also need to be changed if the port of RustFS service is
changed
+ -
LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT=${TEXERA_HOST}:${RUSTFS_PORT:-9000}
-
LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=${STORAGE_S3_AUTH_USERNAME}
-
LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY=${STORAGE_S3_AUTH_PASSWORD}
entrypoint: ["/bin/sh", "-c"]
@@ -126,7 +137,7 @@ services:
depends_on:
postgres:
condition: service_healthy
- minio:
+ rustfs:
condition: service_started
lakekeeper-migrate:
condition: service_completed_successfully
@@ -142,14 +153,14 @@ services:
start_period: 10s
# One-shot init container that creates the Lakekeeper default project and
- # the Iceberg warehouse pointing at the MinIO bucket prepared by minio-init.
+ # the Iceberg warehouse pointing at the RustFS bucket prepared by
rustfs-init.
lakekeeper-init:
image: alpine:3.19
container_name: texera-lakekeeper-init
depends_on:
lakekeeper:
condition: service_healthy
- minio-init:
+ rustfs-init:
condition: service_completed_successfully
env_file:
- .env
@@ -222,7 +233,7 @@ services:
# The warehouse persists its S3 endpoint in Lakekeeper's own DB. The
# local-dev launcher sets STORAGE_S3_ENDPOINT to the host LAN IP so
# both this container and the host JVMs (Iceberg remote-signing)
reach
- # the same MinIO — but that IP changes across networks / DHCP leases.
+ # the same RustFS — but that IP changes across networks / DHCP
leases.
# This init is idempotent and would otherwise just skip, leaving a
# stale endpoint that breaks workflow execution with an opaque
Iceberg
# RESTException. Refresh the stored endpoint on every run (#6195).
@@ -314,7 +325,7 @@ services:
container_name: file-service
restart: unless-stopped
depends_on:
- minio:
+ rustfs:
condition: service_started
lakefs:
condition: service_healthy
@@ -536,6 +547,6 @@ networks:
# persistent volumes
volumes:
- minio_data:
+ rustfs_data:
postgres_data:
workflow_result_data:
\ No newline at end of file
diff --git a/build.sbt b/build.sbt
index 67b838a2a0..11f2e9447e 100644
--- a/build.sbt
+++ b/build.sbt
@@ -167,6 +167,7 @@ lazy val FileService = (project in file("file-service"))
.dependsOn(WorkflowCore, Auth, Config)
.configs(Test)
.dependsOn(DAO % "test->test") // test scope dependency
+ .dependsOn(WorkflowCore % "test->test") // reuse RustFSContainer in
MockLakeFS
.settings(
dependencyOverrides ++= Seq(
// override it as io.dropwizard 4 require 2.16.1 or higher
diff --git a/common/config/src/main/resources/storage.conf
b/common/config/src/main/resources/storage.conf
index 12a9919e04..244d2a171e 100644
--- a/common/config/src/main/resources/storage.conf
+++ b/common/config/src/main/resources/storage.conf
@@ -125,7 +125,7 @@ storage {
}
auth {
- username = "texera_minio"
+ username = "texera_rustfs"
username = ${?STORAGE_S3_AUTH_USERNAME}
password = "password"
diff --git a/common/workflow-core/build.sbt b/common/workflow-core/build.sbt
index bdaa923827..391113f089 100644
--- a/common/workflow-core/build.sbt
+++ b/common/workflow-core/build.sbt
@@ -87,8 +87,9 @@ libraryDependencies ++= Seq(
"org.scalatest" %% "scalatest" % "3.2.15" % Test, //
ScalaTest
"junit" % "junit" % "4.13.2" % Test, // JUnit
"com.novocode" % "junit-interface" % "0.11" % Test, // SBT
interface for JUnit
- "com.dimafeng" %% "testcontainers-scala-scalatest" % testcontainersVersion %
Test, // Testcontainers ScalaTest integration
- "com.dimafeng" %% "testcontainers-scala-minio" % testcontainersVersion %
Test // MinIO Testcontainer Scala integration
+ // RustFS has no dedicated testcontainers-scala module; RustFSContainer
builds it on the
+ // GenericContainer that testcontainers-scala-core (pulled in by -scalatest)
provides.
+ "com.dimafeng" %% "testcontainers-scala-scalatest" % testcontainersVersion %
Test // Testcontainers ScalaTest integration
)
diff --git
a/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala
b/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala
index 956de75520..11a9056055 100644
---
a/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala
+++
b/common/workflow-core/src/main/scala/org/apache/texera/service/util/S3StorageClient.scala
@@ -30,7 +30,7 @@ import java.io.InputStream
import scala.jdk.CollectionConverters._
/**
- * S3Storage provides an abstraction for S3-compatible storage (e.g., MinIO).
+ * S3Storage provides an abstraction for S3-compatible storage (e.g., RustFS).
* - Uses credentials and endpoint from StorageConfig.
* - Supports object upload, download, listing, and deletion.
*/
@@ -44,14 +44,14 @@ object S3StorageClient {
// Cap how many failed keys are listed in the error message.
private[util] val MAX_LISTED_DELETE_ERRORS = 10
- // Initialize MinIO-compatible S3 Client
+ // Initialize the S3 client for the configured S3-compatible endpoint
private lazy val s3Client: S3Client = {
val credentials = AwsBasicCredentials.create(StorageConfig.s3Username,
StorageConfig.s3Password)
S3Client
.builder()
.credentialsProvider(StaticCredentialsProvider.create(credentials))
.region(Region.of(StorageConfig.s3Region))
- .endpointOverride(java.net.URI.create(StorageConfig.s3Endpoint)) //
MinIO URL
+ .endpointOverride(java.net.URI.create(StorageConfig.s3Endpoint)) //
object-store URL
.serviceConfiguration(
S3Configuration.builder().pathStyleAccessEnabled(true).build()
)
@@ -106,7 +106,7 @@ object S3StorageClient {
* A trailing `/` is added when missing so the prefix matches on a path
boundary (`a/b` deletes
* `a/b/file` but not `a/bc/file`). An empty prefix would match the whole
bucket and is rejected.
*
- * @param bucketName Target S3/MinIO bucket.
+ * @param bucketName Target S3 bucket.
* @param directoryPrefix Non-empty key prefix to delete.
*/
def deleteDirectory(bucketName: String, directoryPrefix: String): Unit = {
diff --git
a/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala
b/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala
new file mode 100644
index 0000000000..fd07258f73
--- /dev/null
+++
b/common/workflow-core/src/test/scala/org/apache/texera/service/util/RustFSContainer.scala
@@ -0,0 +1,80 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.texera.service.util
+
+import com.dimafeng.testcontainers.GenericContainer
+import org.testcontainers.containers.wait.strategy.Wait
+
+import java.time.Duration
+
+/**
+ * Factory for the RustFS container used by the S3-backed test suites.
+ *
+ * testcontainers-scala ships a `MinIOContainer` module but no RustFS one, so
this builds the
+ * equivalent on top of `GenericContainer`. The differences from
`MinIOContainer` that matter:
+ *
+ * - credentials come from `RUSTFS_ACCESS_KEY` / `RUSTFS_SECRET_KEY` rather
than
+ * `MINIO_ROOT_USER` / `MINIO_ROOT_PASSWORD`;
+ * - readiness is `GET /health` on the S3 port, not `GET
/minio/health/live`;
+ * - `RUSTFS_REGION` must match the region the AWS SDK signs with, because
the region is part
+ * of the SigV4 credential scope.
+ *
+ * The console (9001) is left disabled: no test drives it, and starting it
only widens the
+ * surface the readiness check has to wait for.
+ */
+object RustFSContainer {
+
+ /** Image pinned in lockstep with `bin/single-node/docker-compose.yml` and
`bin/k8s/values.yaml`. */
+ val ImageName: String = "rustfs/rustfs:1.0.0-rc.6"
+
+ /** S3 API port inside the container. */
+ val Port: Int = 9000
+
+ val DefaultUser: String = "texera_rustfs"
+ val DefaultPassword: String = "password"
+ val DefaultRegion: String = "us-west-2"
+
+ def apply(
+ userName: String = DefaultUser,
+ password: String = DefaultPassword,
+ region: String = DefaultRegion
+ ): GenericContainer = {
+ val container = GenericContainer(
+ dockerImage = ImageName,
+ exposedPorts = Seq(Port),
+ env = Map(
+ "RUSTFS_ACCESS_KEY" -> userName,
+ "RUSTFS_SECRET_KEY" -> password,
+ "RUSTFS_REGION" -> region,
+ "RUSTFS_VOLUMES" -> "/data",
+ // Log to stdout so a failed start shows up in the testcontainers log
consumer
+ // instead of a file inside the container. `warn` keeps that readable.
+ "RUSTFS_OBS_LOG_DIRECTORY" -> "",
+ "RUSTFS_OBS_LOGGER_LEVEL" -> "warn"
+ ),
+ waitStrategy = Wait
+ .forHttp("/health")
+ .forPort(Port)
+ .forStatusCode(200)
+ .withStartupTimeout(Duration.ofMinutes(2))
+ )
+ container
+ }
+}
diff --git
a/common/workflow-core/src/test/scala/org/apache/texera/service/util/S3StorageTestBase.scala
b/common/workflow-core/src/test/scala/org/apache/texera/service/util/S3StorageTestBase.scala
index 1d3b77dbb1..4a25535493 100644
---
a/common/workflow-core/src/test/scala/org/apache/texera/service/util/S3StorageTestBase.scala
+++
b/common/workflow-core/src/test/scala/org/apache/texera/service/util/S3StorageTestBase.scala
@@ -19,14 +19,13 @@
package org.apache.texera.service.util
-import com.dimafeng.testcontainers.MinIOContainer
+import com.dimafeng.testcontainers.GenericContainer
import org.apache.texera.amber.config.StorageConfig
import org.scalatest.{BeforeAndAfterAll, Suite}
-import org.testcontainers.utility.DockerImageName
/**
- * Base trait for tests requiring S3 storage (MinIO).
- * Provides access to a single shared MinIO container across all test suites.
+ * Base trait for tests requiring S3 storage (RustFS).
+ * Provides access to a single shared RustFS container across all test suites.
*
* Usage: Mix this trait into any test suite that needs S3 storage.
*/
@@ -40,21 +39,17 @@ trait S3StorageTestBase extends BeforeAndAfterAll { this:
Suite =>
}
object S3StorageTestBase {
- private lazy val container: MinIOContainer = {
- val c = MinIOContainer(
- dockerImageName =
DockerImageName.parse("minio/minio:RELEASE.2025-02-28T09-55-16Z"),
- userName = "texera_minio",
- password = "password"
- )
+ private lazy val container: GenericContainer = {
+ val c = RustFSContainer(region = StorageConfig.s3Region)
c.start()
- val endpoint = s"http://${c.host}:${c.mappedPort(9000)}"
+ val endpoint = s"http://${c.host}:${c.mappedPort(RustFSContainer.Port)}"
StorageConfig.s3Endpoint = endpoint
- println(s"[S3Storage] Started shared MinIO at $endpoint")
+ println(s"[S3Storage] Started shared RustFS at $endpoint")
sys.addShutdownHook {
- println("[S3Storage] Stopping shared MinIO...")
+ println("[S3Storage] Stopping shared RustFS...")
c.stop()
}
diff --git a/docs/contribution-guidelines/guide-for-developers.md
b/docs/contribution-guidelines/guide-for-developers.md
index 6871b2c636..c6e7a4f5b7 100644
--- a/docs/contribution-guidelines/guide-for-developers.md
+++ b/docs/contribution-guidelines/guide-for-developers.md
@@ -147,9 +147,9 @@ Execute `sql/iceberg_postgres_catalog.sql` to create the
database for storing I
psql -U postgres -f "sql/iceberg_postgres_catalog.sql"
```
-### Setup the LakeFS+Minio locally
+### Setup the LakeFS+RustFS locally
-Texera requires [LakeFS](https://lakefs.io/) and
S3([Minio](https://min.io/docs/minio/kubernetes/upstream/index.html) is one of
the implementations) as the dataset storage. Setting up these two storage
services locally are required to make Texera's dataset feature functioning.
+Texera requires [LakeFS](https://lakefs.io/) and
S3([RustFS](https://docs.rustfs.com/) is one of the implementations) as the
dataset storage. Setting up these two storage services locally are required to
make Texera's dataset feature functioning.
Install [Docker
Desktop](https://docs.docker.com/desktop/setup/install/mac-install/) which
contains both docker engine and docker compose. Make sure you launch the Docker
after installing it.
@@ -160,7 +160,7 @@ cd file-service/src/main/resources
Edit `docker-compose.yml` by: search for `volumes` in the file and follow the
instructions in the comment. This step is required otherwise your data will be
lost if containers are deleted
-Execute the following command to start LakeFS and Minio:
+Execute the following command to start LakeFS and RustFS:
```
docker compose up
```
diff --git a/docs/getting-started/installing-using-docker.md
b/docs/getting-started/installing-using-docker.md
index 52968886d9..ce8b28df23 100644
--- a/docs/getting-started/installing-using-docker.md
+++ b/docs/getting-started/installing-using-docker.md
@@ -146,11 +146,11 @@ All changes below are to the `.env` file in the
installation folder, unless othe
### Run Texera on other ports
By default, Texera uses:
- Port 8080 for its web service
-- Port 9000 for its MinIO storage service
+- Port 9000 for its RustFS storage service
To change these ports, open the `.env` file and update the corresponding
variables:
- For the web service port (8080): change `TEXERA_PORT=8080` to your desired
port, e.g., `TEXERA_PORT=8081`.
-- For the MinIO port (9000): change `MINIO_PORT=9000` to your desired port,
e.g., `MINIO_PORT=9001`.
+- For the RustFS port (9000): change `RUSTFS_PORT=9000` to your desired port,
e.g., `RUSTFS_PORT=9001`.
### Change the locations of Texera data
By default, Docker manages Texera's data locations. To change them to your own
locations:
diff --git a/docs/getting-started/run-on-kubernetes.md
b/docs/getting-started/run-on-kubernetes.md
index 6503b8dc91..d51c3e3ce0 100644
--- a/docs/getting-started/run-on-kubernetes.md
+++ b/docs/getting-started/run-on-kubernetes.md
@@ -110,12 +110,12 @@ Once the deployments are running, you can access the
Texera web interface.
### File Upload Error
-If you see an error when trying to upload a file to a dataset, you may need to
forward the port for MinIO (our file storage service).
+If you see an error when trying to upload a file to a dataset, you may need to
forward the port for RustFS (our file storage service).
Run the following command in a new terminal:
```bash
-kubectl port-forward -n texera-dev service/texera-minio 31000:9000
+kubectl port-forward -n texera-dev service/texera-rustfs-svc 31000:9000
```
This maps the service's port `9000` to your local port `31000`.
diff --git a/docs/overview.md b/docs/overview.md
index 2b232db723..08e1adde76 100644
--- a/docs/overview.md
+++ b/docs/overview.md
@@ -37,7 +37,7 @@ At its core, Texera acts as a bridge between a highly
accessible frontend and a
1. **Web-Based Interface (Frontend):** A rich GUI running directly in your
browser. It allows users to construct data processing pipelines by dragging and
dropping blocks on a canvas. No installation is required on client machines.
2. **Distributed Engine (Backend):** When a workflow is submitted, the Texera
engine compiles the graphical representation into an optimized, distributed
execution plan. It then spins up computing units to process massive datasets in
parallel.
-3. **Storage Integration:** Texera integrates smoothly with modern data lake
and storage technologies (like LakeFS and MinIO) to persistently log runs and
save datasets securely.
+3. **Storage Integration:** Texera integrates smoothly with modern data lake
and storage technologies (like LakeFS and RustFS) to persistently log runs and
save datasets securely.
---
diff --git a/docs/tutorials/guide-to-launch-lakekeeper.md
b/docs/tutorials/guide-to-launch-lakekeeper.md
index a2eb5d9a81..fc4eedfb14 100644
--- a/docs/tutorials/guide-to-launch-lakekeeper.md
+++ b/docs/tutorials/guide-to-launch-lakekeeper.md
@@ -71,7 +71,7 @@ For more information of why using RESTCatalog, see [Issue
#4126](https://github.
+LAKEKEEPER__PG_DATABASE_URL_WRITE="postgres://<user>:<urlencoded_password>@<host>:5432/texera_lakekeeper"
```
- If you have customized storage-related values in
`common/config/src/main/resources/storage.conf` (for example, the bucket name,
S3 endpoint, or MinIO credentials), check the below environment variables in
the script and modify their values accordingly:
+ If you have customized storage-related values in
`common/config/src/main/resources/storage.conf` (for example, the bucket name,
S3 endpoint, or RustFS credentials), check the below environment variables in
the script and modify their values accordingly:
```shell
# Storage settings — must stay in sync with storage.conf
@@ -81,7 +81,7 @@
STORAGE_ICEBERG_CATALOG_REST_WAREHOUSE_NAME="${STORAGE_ICEBERG_CATALOG_REST_WARE
STORAGE_ICEBERG_CATALOG_REST_REGION="${STORAGE_ICEBERG_CATALOG_REST_REGION:-us-west-2}"
STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET="${STORAGE_ICEBERG_CATALOG_REST_S3_BUCKET:-texera-iceberg}"
STORAGE_S3_ENDPOINT="${STORAGE_S3_ENDPOINT:-http://localhost:9000}";
-STORAGE_S3_AUTH_USERNAME="${STORAGE_S3_AUTH_USERNAME:-texera_minio}"
+STORAGE_S3_AUTH_USERNAME="${STORAGE_S3_AUTH_USERNAME:-texera_rustfs}"
STORAGE_S3_AUTH_PASSWORD="${STORAGE_S3_AUTH_PASSWORD:-password}"
```
@@ -96,7 +96,7 @@
STORAGE_S3_AUTH_PASSWORD="${STORAGE_S3_AUTH_PASSWORD:-password}"
1. Start Lakekeeper if it's not already running (on http://localhost:8181)
2. Bootstrap the Lakekeeper server (creates the default project)
- 3. Create the texera-iceberg bucket in MinIO if it doesn't exist
+ 3. Create the texera-iceberg bucket in RustFS if it doesn't exist
4. Register the texera warehouse with Lakekeeper, pointing at that bucket
diff --git a/file-service/build.sbt b/file-service/build.sbt
index c93f5e3230..f816443d21 100644
--- a/file-service/build.sbt
+++ b/file-service/build.sbt
@@ -74,9 +74,10 @@ libraryDependencies ++= Seq(
"org.mockito" % "mockito-core" % mockitoVersion % Test, //
Mockito for mocking
"org.assertj" % "assertj-core" % assertjVersion % Test, //
AssertJ for assertions
"com.novocode" % "junit-interface" % "0.11" % Test, // SBT
interface for JUnit
+ // RustFS has no dedicated testcontainers-scala module; MockLakeFS uses the
shared
+ // RustFSContainer from WorkflowCore's test sources (see the test->test dep
in build.sbt).
"com.dimafeng" %% "testcontainers-scala-scalatest" % testcontainersVersion %
Test, // Testcontainers ScalaTest integration
"com.dimafeng" %% "testcontainers-scala-postgresql" % testcontainersVersion
% Test, // PostgreSQL Testcontainer Scala integration
- "com.dimafeng" %% "testcontainers-scala-minio" % testcontainersVersion %
Test, // MinIO Testcontainer Scala integration
)
/////////////////////////////////////////////////////////////////////////////
diff --git a/file-service/src/main/resources/docker-compose.yml
b/file-service/src/main/resources/docker-compose.yml
index f4d2a07256..95088150b1 100644
--- a/file-service/src/main/resources/docker-compose.yml
+++ b/file-service/src/main/resources/docker-compose.yml
@@ -17,20 +17,23 @@
name: texera-lakefs
services:
- minio:
- image: minio/minio:RELEASE.2025-02-28T09-55-16Z
- container_name: texera-lakefs-minio
+ rustfs:
+ image: rustfs/rustfs:1.0.0-rc.6
+ container_name: texera-lakefs-rustfs
restart: unless-stopped
ports:
- "9000:9000"
- "9001:9001"
environment:
- - MINIO_ROOT_USER=texera_minio
- - MINIO_ROOT_PASSWORD=password
- command: server --console-address ":9001" /data
+ - RUSTFS_ACCESS_KEY=texera_rustfs
+ - RUSTFS_SECRET_KEY=password
+ # Must match the region the AWS SDK signs with: it is part of the SigV4
scope.
+ - RUSTFS_REGION=us-west-2
+ - RUSTFS_CONSOLE_ENABLE=true
+ - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
# The lines below are recommended to mount a host path in order to persist
your data even if the container is removed.
volumes:
- - minio_data:/data
+ - rustfs_data:/data
postgres:
image: postgres:15
@@ -56,16 +59,16 @@ services:
depends_on:
postgres:
condition: service_healthy
- minio:
+ rustfs:
condition: service_started
ports:
- "8000:8000"
environment:
- LAKEFS_BLOCKSTORE_TYPE=s3
- LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE=true
- - LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://minio:9000
+ - LAKEFS_BLOCKSTORE_S3_ENDPOINT=http://rustfs:9000
- LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT=http://localhost:9000
- - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=texera_minio
+ - LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID=texera_rustfs
- LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY=password
- LAKEFS_AUTH_ENCRYPT_SECRET_KEY=random_string_for_lakefs
- LAKEFS_LOGGING_LEVEL=INFO
@@ -94,5 +97,5 @@ networks:
# Named Docker volumes — uncomment the following lines if you mounted host
paths above.
volumes:
- minio_data:
+ rustfs_data:
postgres_data:
diff --git a/file-service/src/main/resources/minio-config.yml
b/file-service/src/main/resources/rustfs-config.yml
similarity index 69%
rename from file-service/src/main/resources/minio-config.yml
rename to file-service/src/main/resources/rustfs-config.yml
index 0622b92bdc..f29fed9e27 100644
--- a/file-service/src/main/resources/minio-config.yml
+++ b/file-service/src/main/resources/rustfs-config.yml
@@ -18,15 +18,20 @@
version: '3.8'
services:
- minio:
- image: minio/minio:latest
- container_name: minio
+ rustfs:
+ image: rustfs/rustfs:1.0.0-rc.6
+ container_name: rustfs
ports:
- - "9500:9000" # MinIO API
- - "9501:9001" # MinIO Console UI
+ - "9500:9000" # S3 API
+ - "9501:9001" # Console UI
environment:
- - MINIO_ROOT_USER=texera_minio
- - MINIO_ROOT_PASSWORD=password
+ - RUSTFS_ACCESS_KEY=texera_rustfs
+ - RUSTFS_SECRET_KEY=password
+ - RUSTFS_REGION=us-west-2
+ - RUSTFS_CONSOLE_ENABLE=true
+ - RUSTFS_CONSOLE_ADDRESS=0.0.0.0:9001
volumes:
- -
/Users/baijiadong/Desktop/chenlab/texera/core/file-service/src/main/user-resources/minio:/data
- command: server --console-address ":9001" /data
\ No newline at end of file
+ - rustfs_data:/data
+
+volumes:
+ rustfs_data:
\ No newline at end of file
diff --git
a/file-service/src/test/scala/org/apache/texera/service/MockLakeFS.scala
b/file-service/src/test/scala/org/apache/texera/service/MockLakeFS.scala
index 62e60bd267..35ddbb4405 100644
--- a/file-service/src/test/scala/org/apache/texera/service/MockLakeFS.scala
+++ b/file-service/src/test/scala/org/apache/texera/service/MockLakeFS.scala
@@ -22,7 +22,7 @@ package org.apache.texera.service
import com.dimafeng.testcontainers._
import io.lakefs.clients.sdk.{ApiClient, RepositoriesApi}
import org.apache.texera.amber.config.StorageConfig
-import org.apache.texera.service.util.S3StorageClient
+import org.apache.texera.service.util.{RustFSContainer, S3StorageClient}
import org.scalatest.{BeforeAndAfterAll, Suite}
import org.testcontainers.containers.Network
import org.testcontainers.utility.DockerImageName
@@ -34,7 +34,7 @@ import software.amazon.awssdk.services.s3.S3Configuration
import java.net.URI
/**
- * Trait to spin up a LakeFS + MinIO + Postgres stack using Testcontainers,
+ * Trait to spin up a LakeFS + RustFS + Postgres stack using Testcontainers,
* similar to how MockTexeraDB uses EmbeddedPostgres.
*/
trait MockLakeFS extends ForAllTestContainer with BeforeAndAfterAll { self:
Suite =>
@@ -53,29 +53,27 @@ trait MockLakeFS extends ForAllTestContainer with
BeforeAndAfterAll { self: Suit
postgres.container.withNetwork(network)
// LakeFS bakes the pre-signed endpoint into its env before containers start,
- // so MinIO cannot use a dynamically mapped host port: presigned URLs must be
- // reachable from the host at an address known ahead of time. Reserve a free
- // host port and pin MinIO's 9000 to it.
- val minioHostPort: Int = {
+ // so the object store cannot use a dynamically mapped host port: presigned
URLs
+ // must be reachable from the host at an address known ahead of time.
Reserve a
+ // free host port and pin the S3 port to it.
+ val objectStoreHostPort: Int = {
val socket = new java.net.ServerSocket(0)
try socket.getLocalPort
finally socket.close()
}
- // MinIO for object storage
- val minio = MinIOContainer(
- dockerImageName =
DockerImageName.parse("minio/minio:RELEASE.2025-02-28T09-55-16Z"),
- userName = "texera_minio",
- password = "password"
- )
- minio.container.withNetwork(network)
- minio.container.withCreateContainerCmdModifier { cmd =>
+ // RustFS for object storage
+ val objectStore: GenericContainer = RustFSContainer()
+ objectStore.container.withNetwork(network)
+ objectStore.container.withCreateContainerCmdModifier { cmd =>
import com.github.dockerjava.api.model.{ExposedPort, PortBinding, Ports}
- // setting explicit bindings replaces them all, so 9001 (console) must keep
- // a dynamic binding or the container readiness check never passes
+ // RustFSContainer exposes only the S3 port, so pinning it is the whole
+ // binding set; the console is not started.
cmd.getHostConfig.withPortBindings(
- new PortBinding(Ports.Binding.bindPort(minioHostPort),
ExposedPort.tcp(9000)),
- new PortBinding(Ports.Binding.empty(), ExposedPort.tcp(9001))
+ new PortBinding(
+ Ports.Binding.bindPort(objectStoreHostPort),
+ ExposedPort.tcp(RustFSContainer.Port)
+ )
)
}
@@ -98,10 +96,11 @@ trait MockLakeFS extends ForAllTestContainer with
BeforeAndAfterAll { self: Suit
env = Map(
"LAKEFS_BLOCKSTORE_TYPE" -> "s3",
"LAKEFS_BLOCKSTORE_S3_FORCE_PATH_STYLE" -> "true",
- "LAKEFS_BLOCKSTORE_S3_ENDPOINT" ->
s"http://${minio.container.getNetworkAliases.get(0)}:9000",
- "LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT" ->
s"http://localhost:$minioHostPort";,
- "LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID" -> "texera_minio",
- "LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY" -> "password",
+ "LAKEFS_BLOCKSTORE_S3_ENDPOINT" ->
s"http://${objectStore.container.getNetworkAliases
+ .get(0)}:${RustFSContainer.Port}",
+ "LAKEFS_BLOCKSTORE_S3_PRE_SIGNED_ENDPOINT" ->
s"http://localhost:$objectStoreHostPort";,
+ "LAKEFS_BLOCKSTORE_S3_CREDENTIALS_ACCESS_KEY_ID" ->
RustFSContainer.DefaultUser,
+ "LAKEFS_BLOCKSTORE_S3_CREDENTIALS_SECRET_ACCESS_KEY" ->
RustFSContainer.DefaultPassword,
"LAKEFS_AUTH_ENCRYPT_SECRET_KEY" -> "random_string_for_lakefs",
"LAKEFS_LOGGING_LEVEL" -> "INFO",
"LAKEFS_STATS_ENABLED" -> "1",
@@ -114,10 +113,11 @@ trait MockLakeFS extends ForAllTestContainer with
BeforeAndAfterAll { self: Suit
)
lakefs.container.withNetwork(network)
- override val container = MultipleContainers(postgres, minio, lakefs)
+ override val container = MultipleContainers(postgres, objectStore, lakefs)
def lakefsBaseUrl: String =
s"http://${lakefs.host}:${lakefs.mappedPort(8000)}"
- def minioEndpoint: String = s"http://${minio.host}:${minio.mappedPort(9000)}"
+ def objectStoreEndpoint: String =
+
s"http://${objectStore.host}:${objectStore.mappedPort(RustFSContainer.Port)}"
def lakefsApiBasePath: String = s"$lakefsBaseUrl/api/v1"
// ---- Clients (lazy so they initialize after containers are started) ----
@@ -134,20 +134,20 @@ trait MockLakeFS extends ForAllTestContainer with
BeforeAndAfterAll { self: Suit
lazy val repositoriesApi: RepositoriesApi = new
RepositoriesApi(lakefsApiClient)
/**
- * S3 client instance for testing pointed at MinIO.
+ * S3 client instance for testing pointed at RustFS.
*
* Notes:
- * - Region can be any value for MinIO, but MUST match what your signing
expects.
- * so we use that.
+ * - The region MUST match RUSTFS_REGION: it is part of the SigV4
credential scope.
* - Path-style is important: http://host:port/bucket/key
*/
lazy val s3Client: S3Client = {
//Temporal credentials for testing purposes only
- val creds = AwsBasicCredentials.create("texera_minio", "password")
+ val creds =
+ AwsBasicCredentials.create(RustFSContainer.DefaultUser,
RustFSContainer.DefaultPassword)
S3Client
.builder()
.endpointOverride(URI.create(StorageConfig.s3Endpoint)) // set in
afterStart()
- .region(Region.US_WEST_2) // Required for `.build()`; not important in
this test config.
+ .region(Region.US_WEST_2) // must match RustFSContainer's RUSTFS_REGION
.credentialsProvider(StaticCredentialsProvider.create(creds))
.serviceConfiguration(S3Configuration.builder().pathStyleAccessEnabled(true).build())
.build()
@@ -172,7 +172,7 @@ trait MockLakeFS extends ForAllTestContainer with
BeforeAndAfterAll { self: Suit
}
// replace storage endpoints in StorageConfig
- StorageConfig.s3Endpoint = minioEndpoint
+ StorageConfig.s3Endpoint = objectStoreEndpoint
StorageConfig.lakefsEndpoint = lakefsApiBasePath
// create S3 bucket used by lakeFS in tests
