This is an automated email from the ASF dual-hosted git repository.
raulcd pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new 94ed8aa540 GH-38295: [CI][R] Free up disk space for Azure Pipelines
jobs (#38302)
94ed8aa540 is described below
commit 94ed8aa5408bf9a344478de69ca012b65262ebf6
Author: Sutou Kouhei <[email protected]>
AuthorDate: Tue Oct 17 18:12:29 2023 +0900
GH-38295: [CI][R] Free up disk space for Azure Pipelines jobs (#38302)
### Rationale for this change
test-r-rhub-ubuntu-gcc-release-latest doesn't have enough disk space.
### What changes are included in this PR?
Remove pre-installed files on Azure Pipelines too.
### Are these changes tested?
Yes.
### Are there any user-facing changes?
No.
* Closes: #38295
Authored-by: Sutou Kouhei <[email protected]>
Signed-off-by: Raúl Cumplido <[email protected]>
---
ci/scripts/util_free_space.sh | 154 +++++++++++++++++++++---------------------
dev/tasks/macros.jinja | 5 ++
dev/tasks/r/azure.linux.yml | 103 ++++++++++++++--------------
3 files changed, 130 insertions(+), 132 deletions(-)
diff --git a/ci/scripts/util_free_space.sh b/ci/scripts/util_free_space.sh
index acea074fee..0518869d06 100755
--- a/ci/scripts/util_free_space.sh
+++ b/ci/scripts/util_free_space.sh
@@ -19,81 +19,79 @@
set -eux
-if [ "${GITHUB_ACTIONS}" = "true" ]; then
- df -h
- echo "::group::/usr/local/*"
- du -hsc /usr/local/*
- echo "::endgroup::"
- # ~1GB
- sudo rm -rf \
- /usr/local/aws-cli \
- /usr/local/aws-sam-cil \
- /usr/local/julia* || :
- echo "::group::/usr/local/bin/*"
- du -hsc /usr/local/bin/*
- echo "::endgroup::"
- # ~1GB (From 1.2GB to 214MB)
- sudo rm -rf \
- /usr/local/bin/aliyun \
- /usr/local/bin/aws \
- /usr/local/bin/aws_completer \
- /usr/local/bin/azcopy \
- /usr/local/bin/bicep \
- /usr/local/bin/cmake-gui \
- /usr/local/bin/cpack \
- /usr/local/bin/helm \
- /usr/local/bin/hub \
- /usr/local/bin/kubectl \
- /usr/local/bin/minikube \
- /usr/local/bin/node \
- /usr/local/bin/packer \
- /usr/local/bin/pulumi* \
- /usr/local/bin/sam \
- /usr/local/bin/stack \
- /usr/local/bin/terraform || :
- # 142M
- sudo rm -rf /usr/local/bin/oc || : \
- echo "::group::/usr/local/share/*"
- du -hsc /usr/local/share/*
- echo "::endgroup::"
- # 506MB
- sudo rm -rf /usr/local/share/chromium || :
- # 1.3GB
- sudo rm -rf /usr/local/share/powershell || :
- echo "::group::/usr/local/lib/*"
- du -hsc /usr/local/lib/*
- echo "::endgroup::"
- # 15GB
- sudo rm -rf /usr/local/lib/android || :
- # 341MB
- sudo rm -rf /usr/local/lib/heroku || :
- # 1.2GB
- sudo rm -rf /usr/local/lib/node_modules || :
- echo "::group::/opt/*"
- du -hsc /opt/*
- echo "::endgroup::"
- # 679MB
- sudo rm -rf /opt/az || :
- echo "::group::/opt/microsoft/*"
- du -hsc /opt/microsoft/*
- echo "::endgroup::"
- # 197MB
- sudo rm -rf /opt/microsoft/powershell || :
- echo "::group::/opt/hostedtoolcache/*"
- du -hsc /opt/hostedtoolcache/*
- echo "::endgroup::"
- # 5.3GB
- sudo rm -rf /opt/hostedtoolcache/CodeQL || :
- # 1.4GB
- sudo rm -rf /opt/hostedtoolcache/go || :
- # 489MB
- sudo rm -rf /opt/hostedtoolcache/PyPy || :
- # 376MB
- sudo rm -rf /opt/hostedtoolcache/node || :
- # Remove Web browser packages
- sudo apt purge -y \
- firefox \
- google-chrome-stable \
- microsoft-edge-stable
- df -h
-fi
+df -h
+echo "::group::/usr/local/*"
+du -hsc /usr/local/*
+echo "::endgroup::"
+# ~1GB
+sudo rm -rf \
+ /usr/local/aws-cli \
+ /usr/local/aws-sam-cil \
+ /usr/local/julia* || :
+echo "::group::/usr/local/bin/*"
+du -hsc /usr/local/bin/*
+echo "::endgroup::"
+# ~1GB (From 1.2GB to 214MB)
+sudo rm -rf \
+ /usr/local/bin/aliyun \
+ /usr/local/bin/aws \
+ /usr/local/bin/aws_completer \
+ /usr/local/bin/azcopy \
+ /usr/local/bin/bicep \
+ /usr/local/bin/cmake-gui \
+ /usr/local/bin/cpack \
+ /usr/local/bin/helm \
+ /usr/local/bin/hub \
+ /usr/local/bin/kubectl \
+ /usr/local/bin/minikube \
+ /usr/local/bin/node \
+ /usr/local/bin/packer \
+ /usr/local/bin/pulumi* \
+ /usr/local/bin/sam \
+ /usr/local/bin/stack \
+ /usr/local/bin/terraform || :
+# 142M
+sudo rm -rf /usr/local/bin/oc || : \
+echo "::group::/usr/local/share/*"
+du -hsc /usr/local/share/*
+echo "::endgroup::"
+# 506MB
+sudo rm -rf /usr/local/share/chromium || :
+# 1.3GB
+sudo rm -rf /usr/local/share/powershell || :
+echo "::group::/usr/local/lib/*"
+du -hsc /usr/local/lib/*
+echo "::endgroup::"
+# 15GB
+sudo rm -rf /usr/local/lib/android || :
+# 341MB
+sudo rm -rf /usr/local/lib/heroku || :
+# 1.2GB
+sudo rm -rf /usr/local/lib/node_modules || :
+echo "::group::/opt/*"
+du -hsc /opt/*
+echo "::endgroup::"
+# 679MB
+sudo rm -rf /opt/az || :
+echo "::group::/opt/microsoft/*"
+du -hsc /opt/microsoft/*
+echo "::endgroup::"
+# 197MB
+sudo rm -rf /opt/microsoft/powershell || :
+echo "::group::/opt/hostedtoolcache/*"
+du -hsc /opt/hostedtoolcache/*
+echo "::endgroup::"
+# 5.3GB
+sudo rm -rf /opt/hostedtoolcache/CodeQL || :
+# 1.4GB
+sudo rm -rf /opt/hostedtoolcache/go || :
+# 489MB
+sudo rm -rf /opt/hostedtoolcache/PyPy || :
+# 376MB
+sudo rm -rf /opt/hostedtoolcache/node || :
+# Remove Web browser packages
+sudo apt purge -y \
+ firefox \
+ google-chrome-stable \
+ microsoft-edge-stable
+df -h
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index 3bcff2f635..9ee95dec15 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -182,6 +182,11 @@ on:
displayName: Clone arrow
{% endmacro %}
+{%- macro azure_free_space() -%}
+ - script: arrow/ci/scripts/util_free_space.sh
+ displayName: Free up disk space
+{% endmacro %}
+
{%- macro azure_upload_releases(pattern) -%}
- task: UsePythonVersion@0
inputs:
diff --git a/dev/tasks/r/azure.linux.yml b/dev/tasks/r/azure.linux.yml
index 0cbc70ed00..e26a59629f 100644
--- a/dev/tasks/r/azure.linux.yml
+++ b/dev/tasks/r/azure.linux.yml
@@ -16,62 +16,57 @@
{% import 'macros.jinja' as macros with context %}
jobs:
- - job: linux
- pool:
- vmImage: ubuntu-latest
- timeoutInMinutes: 360
- {% if env is defined %}
- variables:
+- job: linux
+ pool:
+ vmImage: ubuntu-latest
+ timeoutInMinutes: 360
+ {% if env is defined %}
+ variables:
{% for key, value in env.items() %}
- {{ key }}: {{ value }}
+ {{ key }}: {{ value }}
{% endfor %}
- {% endif %}
- steps:
- - script: |
- set -ex
- git clone --no-checkout {{ arrow.remote }} arrow
- git -C arrow fetch -t {{ arrow.remote }} {{ arrow.branch }}
- git -C arrow checkout FETCH_HEAD
- git -C arrow submodule update --init --recursive
- displayName: Clone arrow
+ {% endif %}
+ steps:
+ {{ macros.azure_checkout_arrow() }}
+ {{ macros.azure_free_space() }}
- - script: |
- set -ex
- docker -v
- docker-compose -v
- cd arrow
- export R_ORG={{ r_org }}
- export R_IMAGE={{ r_image }}
- export R_TAG={{ r_tag }}
- export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
- export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
- docker-compose pull --ignore-pull-failures r
- docker-compose build r
- displayName: Docker build
- env:
- {{ macros.azure_set_sccache_envvars()|indent(8) }}
+ - script: |
+ set -ex
+ docker -v
+ docker-compose -v
+ cd arrow
+ export R_ORG={{ r_org }}
+ export R_IMAGE={{ r_image }}
+ export R_TAG={{ r_tag }}
+ export DEVTOOLSET_VERSION={{ devtoolset_version|default("") }}
+ export R_CUSTOM_CCACHE={{ r_custom_ccache|default("false") }}
+ docker-compose pull --ignore-pull-failures r
+ docker-compose build r
+ displayName: Docker build
+ env:
+ {{ macros.azure_set_sccache_envvars()|indent(4) }}
- - script: |
- set -ex
- cd arrow
- export R_ORG={{ r_org }}
- export R_IMAGE={{ r_image }}
- export R_TAG={{ r_tag }}
- export ARROW_R_DEV={{ not_cran|default("TRUE") }}
- # Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true
- # if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
- docker-compose run {{ flags|default("") }} r
- displayName: Docker run
- env:
- {{ macros.azure_set_sccache_envvars()|indent(8) }}
+ - script: |
+ set -ex
+ cd arrow
+ export R_ORG={{ r_org }}
+ export R_IMAGE={{ r_image }}
+ export R_TAG={{ r_tag }}
+ export ARROW_R_DEV={{ not_cran|default("TRUE") }}
+ # Note that by default, ci/scripts/r_test.sh sets NOT_CRAN=true
+ # if ARROW_R_DEV=TRUE. Pass `-e NOT_CRAN=false` to turn that off.
+ docker-compose run {{ flags|default("") }} r
+ displayName: Docker run
+ env:
+ {{ macros.azure_set_sccache_envvars()|indent(4) }}
- - script: |
- set -ex
- cat arrow/r/check/arrow.Rcheck/00install.out
- displayName: Dump install logs
- condition: succeededOrFailed()
- - script: |
- set -ex
- cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout*
- displayName: Dump test logs
- condition: succeededOrFailed()
+ - script: |
+ set -ex
+ cat arrow/r/check/arrow.Rcheck/00install.out
+ displayName: Dump install logs
+ condition: succeededOrFailed()
+ - script: |
+ set -ex
+ cat arrow/r/check/arrow.Rcheck/tests/testthat.Rout*
+ displayName: Dump test logs
+ condition: succeededOrFailed()
