This is an automated email from the ASF dual-hosted git repository. maximebeauchemin pushed a commit to branch docker-compose-mem in repository https://gitbox.apache.org/repos/asf/superset.git
commit 9b6eaab99e85f31e78fc68bf6901a227deb6f054 Author: Maxime Beauchemin <[email protected]> AuthorDate: Wed May 29 16:42:49 2024 -0700 chore: make docker-compose use less memory Playing a few tricks to make docker-compose use less memory: - not starting the `superset-tests-worker` service on a normal `docker-compose up` by using `profile`. From my understanding this service is used only by `scripts/tests/run.sh`, and it starts/stops it on its own - doesn't assume its already running. This saves ~900MB, while maintaining 8 workers to run tests by default, but makes it confiurable if you wanted more you could more easily change it now - starting only 2 workers by default for `superset-worker` as opposed to the default of 8, which is not necessary for dev workflows. This saves maybe ~600MB --- docker-compose.yml | 5 +++++ docker/docker-bootstrap.sh | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index a697f8f90e..de0f12db2e 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -172,6 +172,8 @@ services: required: true - path: docker/.env-local # optional override required: false + environment: + CELERYD_CONCURRENCY: 2 restart: unless-stopped depends_on: *superset-depends-on user: *superset-user @@ -211,12 +213,15 @@ services: required: true - path: docker/.env-local # optional override required: false + profiles: + - optional environment: DATABASE_HOST: localhost DATABASE_DB: test REDIS_CELERY_DB: 2 REDIS_RESULTS_DB: 3 REDIS_HOST: localhost + CELERYD_CONCURRENCY: 8 network_mode: host depends_on: *superset-depends-on user: *superset-user diff --git a/docker/docker-bootstrap.sh b/docker/docker-bootstrap.sh index 2f86342714..d68734d43d 100755 --- a/docker/docker-bootstrap.sh +++ b/docker/docker-bootstrap.sh @@ -38,7 +38,8 @@ fi case "${1}" in worker) echo "Starting Celery worker..." - celery --app=superset.tasks.celery_app:app worker -O fair -l INFO + # setting up only 2 workers by default to contain memory usage + celery --app=superset.tasks.celery_app:app worker -O fair -l INFO --concurrency=${CELERYD_CONCURRENCY:-2} ;; beat) echo "Starting Celery beat..."
