This is an automated email from the ASF dual-hosted git repository.
apitrou pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/arrow.git
The following commit(s) were added to refs/heads/main by this push:
new be3b78902f GH-40954: [CI] Fix use of obsolete docker-compose command
on Github Actions (#40949)
be3b78902f is described below
commit be3b78902ff2871e637af2c340e24b75853e32e7
Author: Antoine Pitrou <[email protected]>
AuthorDate: Wed Apr 3 10:33:31 2024 +0200
GH-40954: [CI] Fix use of obsolete docker-compose command on Github Actions
(#40949)
### Rationale for this change
The `docker-compose` utility is progressively being removed from
GHA-provided runners:
https://github.com/actions/runner-images/issues/9557
### What changes are included in this PR?
Use `docker` client CLI directly instead of `docker-compose` where possible.
### Are these changes tested?
Yes, this should fix the sporadic CI failures because of the above
migration.
### Are there any user-facing changes?
No, except additional optional env var `ARCHERY_DEBUG`.
* GitHub Issue: #40954
Authored-by: Antoine Pitrou <[email protected]>
Signed-off-by: Antoine Pitrou <[email protected]>
---
.github/workflows/archery.yml | 2 +
.github/workflows/cpp.yml | 5 ++
.github/workflows/dev.yml | 4 ++
.github/workflows/docs.yml | 2 +
.github/workflows/docs_light.yml | 2 +
.github/workflows/go.yml | 9 +++
.github/workflows/integration.yml | 2 +
.github/workflows/java.yml | 2 +
.github/workflows/java_jni.yml | 2 +
.github/workflows/js.yml | 4 ++
.github/workflows/python.yml | 2 +
.github/workflows/r.yml | 2 +
.github/workflows/ruby.yml | 2 +
.github/workflows/swift.yml | 2 +
dev/archery/archery/cli.py | 1 +
dev/archery/archery/docker/cli.py | 79 +++++++---------------
dev/archery/archery/docker/core.py | 50 +++++++++-----
.../archery/docker/tests/test_docker_cli.py | 19 +-----
dev/tasks/java-jars/github.yml | 2 +
dev/tasks/linux-packages/github.linux.yml | 1 +
dev/tasks/macros.jinja | 4 ++
dev/tasks/python-wheels/github.linux.yml | 2 +
dev/tasks/tasks.yml | 4 ++
23 files changed, 117 insertions(+), 87 deletions(-)
diff --git a/.github/workflows/archery.yml b/.github/workflows/archery.yml
index cb783dd66c..2aa6b0dcbb 100644
--- a/.github/workflows/archery.yml
+++ b/.github/workflows/archery.yml
@@ -32,7 +32,9 @@ on:
- 'docker-compose.yml'
env:
+ ARCHERY_DEBUG: 1
ARCHERY_DEFAULT_BRANCH: ${{ github.event.repository.default_branch }}
+ ARCHERY_USE_DOCKER_CLI: 1
concurrency:
group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{
github.workflow }}
diff --git a/.github/workflows/cpp.yml b/.github/workflows/cpp.yml
index e8e41f1bcb..1d10be3b5b 100644
--- a/.github/workflows/cpp.yml
+++ b/.github/workflows/cpp.yml
@@ -53,6 +53,7 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
ARROW_ENABLE_TIMING_TESTS: OFF
DOCKER_VOLUME_PREFIX: ".docker/"
@@ -94,6 +95,7 @@ jobs:
cat <<JSON >> "$GITHUB_OUTPUT"
{
"arch": "arm64v8",
+ "archery-use-docker-cli": "0",
"clang-tools": "10",
"image": "ubuntu-cpp",
"llvm": "10",
@@ -118,6 +120,9 @@ jobs:
include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
env:
ARCH: ${{ matrix.arch }}
+ # By default, use Docker CLI because docker-compose v1 is obsolete,
+ # except where the Docker client version is too old.
+ ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
ARROW_SIMD_LEVEL: ${{ matrix.simd-level }}
CLANG_TOOLS: ${{ matrix.clang-tools }}
LLVM: ${{ matrix.llvm }}
diff --git a/.github/workflows/dev.yml b/.github/workflows/dev.yml
index 37fda2e313..8af5832f15 100644
--- a/.github/workflows/dev.yml
+++ b/.github/workflows/dev.yml
@@ -29,6 +29,10 @@ concurrency:
permissions:
contents: read
+env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
+
jobs:
lint:
diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml
index 9c7701f25f..fe49e275d9 100644
--- a/.github/workflows/docs.yml
+++ b/.github/workflows/docs.yml
@@ -24,6 +24,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
ARROW_ENABLE_TIMING_TESTS: OFF
DOCKER_VOLUME_PREFIX: ".docker/"
diff --git a/.github/workflows/docs_light.yml b/.github/workflows/docs_light.yml
index 6ec4c3d53d..376c87651d 100644
--- a/.github/workflows/docs_light.yml
+++ b/.github/workflows/docs_light.yml
@@ -33,6 +33,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
ARROW_ENABLE_TIMING_TESTS: OFF
DOCKER_VOLUME_PREFIX: ".docker/"
diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml
index 7fca385282..11dc29dcae 100644
--- a/.github/workflows/go.yml
+++ b/.github/workflows/go.yml
@@ -41,6 +41,10 @@ concurrency:
permissions:
contents: read
+env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
+
jobs:
docker-targets:
@@ -75,12 +79,14 @@ jobs:
{
"arch-label": "ARM64",
"arch": "arm64v8",
+ "archery-use-docker-cli": "0",
"go": "1.21",
"runs-on": ["self-hosted", "arm", "linux"]
},
{
"arch-label": "ARM64",
"arch": "arm64v8",
+ "archery-use-docker-cli": "0",
"go": "1.22",
"runs-on": ["self-hosted", "arm", "linux"]
}
@@ -101,6 +107,9 @@ jobs:
include: ${{ fromJson(needs.docker-targets.outputs.targets) }}
env:
ARCH: ${{ matrix.arch }}
+ # By default, use Docker CLI because docker-compose v1 is obsolete,
+ # except where the Docker client version is too old.
+ ARCHERY_USE_DOCKER_CLI: ${{ matrix.archery-use-docker-cli || '1' }}
GO: ${{ matrix.go }}
steps:
- name: Checkout Arrow
diff --git a/.github/workflows/integration.yml
b/.github/workflows/integration.yml
index 0f186ff6a4..2c3499c160 100644
--- a/.github/workflows/integration.yml
+++ b/.github/workflows/integration.yml
@@ -51,6 +51,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"
jobs:
diff --git a/.github/workflows/java.yml b/.github/workflows/java.yml
index 423f54cd93..611e202ca0 100644
--- a/.github/workflows/java.yml
+++ b/.github/workflows/java.yml
@@ -45,6 +45,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"
jobs:
diff --git a/.github/workflows/java_jni.yml b/.github/workflows/java_jni.yml
index 790ffd5c65..958216ac76 100644
--- a/.github/workflows/java_jni.yml
+++ b/.github/workflows/java_jni.yml
@@ -45,6 +45,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"
jobs:
diff --git a/.github/workflows/js.yml b/.github/workflows/js.yml
index dab89da44c..c9b7d7b742 100644
--- a/.github/workflows/js.yml
+++ b/.github/workflows/js.yml
@@ -38,6 +38,10 @@ concurrency:
permissions:
contents: read
+env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
+
jobs:
docker:
diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml
index 1147ac13e6..2db9b17e89 100644
--- a/.github/workflows/python.yml
+++ b/.github/workflows/python.yml
@@ -41,6 +41,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"
jobs:
diff --git a/.github/workflows/r.yml b/.github/workflows/r.yml
index 78677499f3..05c85fa6dc 100644
--- a/.github/workflows/r.yml
+++ b/.github/workflows/r.yml
@@ -51,6 +51,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"
jobs:
diff --git a/.github/workflows/ruby.yml b/.github/workflows/ruby.yml
index 311c1c822b..ea3e61d557 100644
--- a/.github/workflows/ruby.yml
+++ b/.github/workflows/ruby.yml
@@ -53,6 +53,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"
jobs:
diff --git a/.github/workflows/swift.yml b/.github/workflows/swift.yml
index f55e9e7750..3f039315b5 100644
--- a/.github/workflows/swift.yml
+++ b/.github/workflows/swift.yml
@@ -41,6 +41,8 @@ permissions:
contents: read
env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
DOCKER_VOLUME_PREFIX: ".docker/"
jobs:
diff --git a/dev/archery/archery/cli.py b/dev/archery/archery/cli.py
index 32921afb2e..5fa41e28a3 100644
--- a/dev/archery/archery/cli.py
+++ b/dev/archery/archery/cli.py
@@ -44,6 +44,7 @@ BOOL = ArrowBool()
@click.group(context_settings={"help_option_names": ["-h", "--help"]})
@click.option("--debug", type=BOOL, is_flag=True, default=False,
+ envvar='ARCHERY_DEBUG',
help="Increase logging with debugging output.")
@click.option("--pdb", type=BOOL, is_flag=True, default=False,
help="Invoke pdb on uncaught exception.")
diff --git a/dev/archery/archery/docker/cli.py
b/dev/archery/archery/docker/cli.py
index 20d9a16138..e6baf0ca1f 100644
--- a/dev/archery/archery/docker/cli.py
+++ b/dev/archery/archery/docker/cli.py
@@ -46,10 +46,19 @@ def _mock_compose_calls(compose):
callback=validate_arrow_sources,
help="Specify Arrow source directory.")
@click.option('--dry-run/--execute', default=False,
- help="Display the docker-compose commands instead of executing "
- "them.")
+ help="Display the docker commands instead of executing them.")
[email protected]('--using-docker-cli', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_CLI',
+ help="Use docker CLI directly for building instead of calling "
+ "docker-compose. This may help to reuse cached layers.")
[email protected]('--using-docker-buildx', default=False, is_flag=True,
+ envvar='ARCHERY_USE_DOCKER_BUILDX',
+ help="Use buildx with docker CLI directly for building instead "
+ "of calling docker-compose or the plain docker build "
+ "command. This option makes the build cache reusable "
+ "across hosts.")
@click.pass_context
-def docker(ctx, src, dry_run):
+def docker(ctx, src, dry_run, using_docker_cli, using_docker_buildx):
"""
Interact with docker-compose based builds.
"""
@@ -64,7 +73,10 @@ def docker(ctx, src, dry_run):
# take the docker-compose parameters like PYTHON, PANDAS, UBUNTU from the
# environment variables to keep the usage similar to docker-compose
+ using_docker_cli |= using_docker_buildx
compose = DockerCompose(config_path, params=os.environ,
+ using_docker=using_docker_cli,
+ using_buildx=using_docker_buildx,
debug=ctx.obj.get('debug', False))
if dry_run:
_mock_compose_calls(compose)
@@ -83,24 +95,19 @@ def check_config(obj):
@docker.command('pull')
@click.argument('image')
[email protected]('--using-docker-cli', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_CLI',
- help="Use docker CLI directly for pulling instead of calling "
- "docker-compose. This may help to reuse cached layers.")
@click.option('--pull-leaf/--no-leaf', default=True,
help="Whether to pull leaf images too.")
@click.option('--ignore-pull-failures/--no-ignore-pull-failures', default=True,
help="Whether to ignore pull failures.")
@click.pass_obj
-def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
- ignore_pull_failures):
+def docker_pull(obj, image, *, pull_leaf, ignore_pull_failures):
"""
Execute docker-compose pull.
"""
compose = obj['compose']
try:
- compose.pull(image, pull_leaf=pull_leaf, using_docker=using_docker_cli,
+ compose.pull(image, pull_leaf=pull_leaf,
ignore_pull_failures=ignore_pull_failures)
except UndefinedImage as e:
raise click.ClickException(
@@ -115,16 +122,6 @@ def docker_pull(obj, image, *, using_docker_cli, pull_leaf,
@click.argument('image')
@click.option('--force-pull/--no-pull', default=True,
help="Whether to force pull the image and its ancestor images")
[email protected]('--using-docker-cli', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_CLI',
- help="Use docker CLI directly for building instead of calling "
- "docker-compose. This may help to reuse cached layers.")
[email protected]('--using-docker-buildx', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_BUILDX',
- help="Use buildx with docker CLI directly for building instead "
- "of calling docker-compose or the plain docker build "
- "command. This option makes the build cache reusable "
- "across hosts.")
@click.option('--use-cache/--no-cache', default=True,
help="Whether to use cache when building the image and its "
"ancestor images")
@@ -133,22 +130,17 @@ def docker_pull(obj, image, *, using_docker_cli,
pull_leaf,
"passed as the argument. To disable caching for both the "
"image and its ancestors use --no-cache option.")
@click.pass_obj
-def docker_build(obj, image, *, force_pull, using_docker_cli,
- using_docker_buildx, use_cache, use_leaf_cache):
+def docker_build(obj, image, *, force_pull, use_cache, use_leaf_cache):
"""
Execute docker-compose builds.
"""
compose = obj['compose']
- using_docker_cli |= using_docker_buildx
try:
if force_pull:
- compose.pull(image, pull_leaf=use_leaf_cache,
- using_docker=using_docker_cli)
+ compose.pull(image, pull_leaf=use_leaf_cache)
compose.build(image, use_cache=use_cache,
use_leaf_cache=use_leaf_cache,
- using_docker=using_docker_cli,
- using_buildx=using_docker_buildx,
pull_parents=force_pull)
except UndefinedImage as e:
raise click.ClickException(
@@ -172,16 +164,6 @@ def docker_build(obj, image, *, force_pull,
using_docker_cli,
help="Whether to force build the image and its ancestor images")
@click.option('--build-only', default=False, is_flag=True,
help="Pull and/or build the image, but do not run it")
[email protected]('--using-docker-cli', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_CLI',
- help="Use docker CLI directly for building instead of calling "
- "docker-compose. This may help to reuse cached layers.")
[email protected]('--using-docker-buildx', default=False, is_flag=True,
- envvar='ARCHERY_USE_DOCKER_BUILDX',
- help="Use buildx with docker CLI directly for building instead "
- "of calling docker-compose or the plain docker build "
- "command. This option makes the build cache reusable "
- "across hosts.")
@click.option('--use-cache/--no-cache', default=True,
help="Whether to use cache when building the image and its "
"ancestor images")
@@ -191,7 +173,7 @@ def docker_build(obj, image, *, force_pull,
using_docker_cli,
"image and its ancestors use --no-cache option.")
@click.option('--resource-limit', default=None,
help="A CPU/memory limit preset to mimic CI environments like "
- "GitHub Actions. Implies --using-docker-cli. Note that "
+ "GitHub Actions. Mandates --using-docker-cli. Note that "
"exporting ARCHERY_DOCKER_BIN=\"sudo docker\" is likely "
"required, unless Docker is configured with cgroups v2 "
"(else Docker will silently ignore the limits).")
@@ -199,8 +181,8 @@ def docker_build(obj, image, *, force_pull,
using_docker_cli,
help="Set volume within the container")
@click.pass_obj
def docker_run(obj, image, command, *, env, user, force_pull, force_build,
- build_only, using_docker_cli, using_docker_buildx, use_cache,
- use_leaf_cache, resource_limit, volume):
+ build_only, use_cache, use_leaf_cache, resource_limit,
+ volume):
"""
Execute docker-compose builds.
@@ -234,18 +216,14 @@ def docker_run(obj, image, command, *, env, user,
force_pull, force_build,
archery docker run ubuntu-cpp bash
"""
compose = obj['compose']
- using_docker_cli |= using_docker_buildx
env = dict(kv.split('=', 1) for kv in env)
try:
if force_pull:
- compose.pull(image, pull_leaf=use_leaf_cache,
- using_docker=using_docker_cli)
+ compose.pull(image, pull_leaf=use_leaf_cache)
if force_build:
compose.build(image, use_cache=use_cache,
- use_leaf_cache=use_leaf_cache,
- using_docker=using_docker_cli,
- using_buildx=using_docker_buildx)
+ use_leaf_cache=use_leaf_cache)
if build_only:
return
compose.run(
@@ -253,7 +231,6 @@ def docker_run(obj, image, command, *, env, user,
force_pull, force_build,
command=command,
env=env,
user=user,
- using_docker=using_docker_cli,
resource_limit=resource_limit,
volumes=volume
)
@@ -273,15 +250,11 @@ def docker_run(obj, image, command, *, env, user,
force_pull, force_build,
@click.option('--password', '-p', required=False,
envvar='ARCHERY_DOCKER_PASSWORD',
help='Docker repository password')
[email protected]('--using-docker-cli', default=False, is_flag=True,
- help="Use docker CLI directly for building instead of calling "
- "docker-compose. This may help to reuse cached layers.")
@click.pass_obj
-def docker_compose_push(obj, image, user, password, using_docker_cli):
+def docker_compose_push(obj, image, user, password):
"""Push the generated docker-compose image."""
compose = obj['compose']
- compose.push(image, user=user, password=password,
- using_docker=using_docker_cli)
+ compose.push(image, user=user, password=password)
@docker.command('images')
diff --git a/dev/archery/archery/docker/core.py
b/dev/archery/archery/docker/core.py
index 184d980875..38720e5856 100644
--- a/dev/archery/archery/docker/core.py
+++ b/dev/archery/archery/docker/core.py
@@ -58,12 +58,21 @@ class UndefinedImage(Exception):
class ComposeConfig:
- def __init__(self, config_path, dotenv_path, compose_bin, params=None):
+ def __init__(self, config_path, dotenv_path, compose_bin,
+ using_docker=False, using_buildx=False,
+ params=None, debug=False):
+ self.using_docker = using_docker
+ self.using_buildx = using_buildx
+ self.debug = debug
config_path = _ensure_path(config_path)
if dotenv_path:
dotenv_path = _ensure_path(dotenv_path)
else:
dotenv_path = config_path.parent / '.env'
+ if self.debug:
+ # Log docker version
+ Docker().run('version')
+
self._read_env(dotenv_path, params)
self._read_config(config_path, compose_bin)
@@ -122,8 +131,13 @@ class ComposeConfig:
)
# trigger docker-compose's own validation
- compose = Command('docker-compose')
- args = ['--file', str(config_path), 'config']
+ if self.using_docker:
+ compose = Docker()
+ args = ['compose']
+ else:
+ compose = Command('docker-compose')
+ args = []
+ args += ['--file', str(config_path), 'config']
result = compose.run(*args, env=self.env, check=False,
stderr=subprocess.PIPE, stdout=subprocess.PIPE)
@@ -164,12 +178,13 @@ class Docker(Command):
class DockerCompose(Command):
def __init__(self, config_path, dotenv_path=None, compose_bin=None,
- params=None, debug=False):
+ using_docker=False, using_buildx=False, params=None,
+ debug=False):
compose_bin = default_bin(compose_bin, 'docker-compose')
self.config = ComposeConfig(config_path, dotenv_path, compose_bin,
- params)
+ params=params, using_docker=using_docker,
+ using_buildx=using_buildx, debug=debug)
self.bin = compose_bin
- self.debug = debug
self.pull_memory = set()
def clear_pull_memory(self):
@@ -215,14 +230,13 @@ class DockerCompose(Command):
)
)
- def pull(self, service_name, pull_leaf=True, using_docker=False,
- ignore_pull_failures=True):
+ def pull(self, service_name, pull_leaf=True, ignore_pull_failures=True):
def _pull(service):
args = ['pull']
if service['image'] in self.pull_memory:
return
- if using_docker:
+ if self.config.using_docker:
try:
self._execute_docker(*args, service['image'])
except Exception as e:
@@ -245,7 +259,7 @@ class DockerCompose(Command):
_pull(service)
def build(self, service_name, use_cache=True, use_leaf_cache=True,
- using_docker=False, using_buildx=False, pull_parents=True):
+ pull_parents=True):
def _build(service, use_cache):
if 'build' not in service:
# nothing to do
@@ -273,7 +287,7 @@ class DockerCompose(Command):
if self.config.env.get('BUILDKIT_INLINE_CACHE') == '1':
args.extend(['--build-arg', 'BUILDKIT_INLINE_CACHE=1'])
- if using_buildx:
+ if self.config.using_buildx:
for k, v in service['build'].get('args', {}).items():
args.extend(['--build-arg', '{}={}'.format(k, v)])
@@ -295,9 +309,9 @@ class DockerCompose(Command):
service['build'].get('context', '.')
])
self._execute_docker("buildx", "build", *args)
- elif using_docker:
+ elif self.config.using_docker:
# better for caching
- if self.debug:
+ if self.config.debug:
args.append("--progress=plain")
for k, v in service['build'].get('args', {}).items():
args.extend(['--build-arg', '{}={}'.format(k, v)])
@@ -310,7 +324,7 @@ class DockerCompose(Command):
])
self._execute_docker("build", *args)
else:
- if self.debug:
+ if self.config.debug:
args.append("--progress=plain")
self._execute_compose("build", *args, service['name'])
@@ -322,7 +336,7 @@ class DockerCompose(Command):
_build(service, use_cache=use_cache and use_leaf_cache)
def run(self, service_name, command=None, *, env=None, volumes=None,
- user=None, using_docker=False, resource_limit=None):
+ user=None, resource_limit=None):
service = self.config.get(service_name)
args = []
@@ -337,7 +351,7 @@ class DockerCompose(Command):
for volume in volumes:
args.extend(['--volume', volume])
- if using_docker or service['need_gpu'] or resource_limit:
+ if self.config.using_docker or service['need_gpu'] or resource_limit:
# use gpus, requires docker>=19.03
if service['need_gpu']:
args.extend(['--gpus', 'all'])
@@ -399,9 +413,9 @@ class DockerCompose(Command):
args.append(command)
self._execute_compose('run', '--rm', *args)
- def push(self, service_name, user=None, password=None, using_docker=False):
+ def push(self, service_name, user=None, password=None):
def _push(service):
- if using_docker:
+ if self.config.using_docker:
return self._execute_docker('push', service['image'])
else:
return self._execute_compose('push', service['name'])
diff --git a/dev/archery/archery/docker/tests/test_docker_cli.py
b/dev/archery/archery/docker/tests/test_docker_cli.py
index ab39c7b9db..c117a3edff 100644
--- a/dev/archery/archery/docker/tests/test_docker_cli.py
+++ b/dev/archery/archery/docker/tests/test_docker_cli.py
@@ -33,14 +33,12 @@ def test_docker_run_with_custom_command(run, build, pull):
assert result.exit_code == 0
pull.assert_called_once_with(
- "ubuntu-cpp", pull_leaf=True, using_docker=False
+ "ubuntu-cpp", pull_leaf=True,
)
build.assert_called_once_with(
"ubuntu-cpp",
use_cache=True,
use_leaf_cache=True,
- using_docker=False,
- using_buildx=False
)
run.assert_called_once_with(
"ubuntu-cpp",
@@ -48,7 +46,6 @@ def test_docker_run_with_custom_command(run, build, pull):
env={},
resource_limit=None,
user=None,
- using_docker=False,
volumes=(),
)
@@ -75,14 +72,12 @@ def test_docker_run_options(run, build, pull):
result = CliRunner().invoke(docker, args)
assert result.exit_code == 0
pull.assert_called_once_with(
- "ubuntu-cpp", pull_leaf=True, using_docker=False
+ "ubuntu-cpp", pull_leaf=True,
)
build.assert_called_once_with(
"ubuntu-cpp",
use_cache=True,
use_leaf_cache=True,
- using_docker=False,
- using_buildx=False
)
run.assert_called_once_with(
"ubuntu-cpp",
@@ -90,7 +85,6 @@ def test_docker_run_options(run, build, pull):
env={"ARROW_GANDIVA": "OFF", "ARROW_FLIGHT": "ON"},
resource_limit=None,
user="root",
- using_docker=False,
volumes=(
"./build:/build",
"./ccache:/ccache:delegated",
@@ -126,7 +120,6 @@ def test_docker_limit_options(run):
env={"ARROW_GANDIVA": "OFF", "ARROW_FLIGHT": "ON"},
resource_limit="github",
user="root",
- using_docker=False,
volumes=(
"./build:/build",
"./ccache:/ccache:delegated",
@@ -145,7 +138,6 @@ def test_docker_run_without_pulling_or_building(run):
env={},
resource_limit=None,
user=None,
- using_docker=False,
volumes=(),
)
@@ -157,14 +149,12 @@ def test_docker_run_only_pulling_and_building(build,
pull):
result = CliRunner().invoke(docker, args)
assert result.exit_code == 0
pull.assert_called_once_with(
- "ubuntu-cpp", pull_leaf=True, using_docker=False
+ "ubuntu-cpp", pull_leaf=True,
)
build.assert_called_once_with(
"ubuntu-cpp",
use_cache=True,
use_leaf_cache=True,
- using_docker=False,
- using_buildx=False
)
@@ -187,8 +177,6 @@ def test_docker_run_without_build_cache(run, build):
"ubuntu-cpp",
use_cache=False,
use_leaf_cache=False,
- using_docker=False,
- using_buildx=False
)
run.assert_called_once_with(
"ubuntu-cpp",
@@ -196,6 +184,5 @@ def test_docker_run_without_build_cache(run, build):
env={},
resource_limit=None,
user="me",
- using_docker=False,
volumes=(),
)
diff --git a/dev/tasks/java-jars/github.yml b/dev/tasks/java-jars/github.yml
index 03cbcc7c98..0437ee7864 100644
--- a/dev/tasks/java-jars/github.yml
+++ b/dev/tasks/java-jars/github.yml
@@ -30,6 +30,7 @@ jobs:
ARCH: {{ '${{ matrix.platform.archery_arch }}' }}
ARCH_ALIAS: {{ '${{ matrix.platform.archery_arch_alias }}' }}
ARCH_SHORT: {{ '${{ matrix.platform.archery_arch_short }}' }}
+ ARCHERY_USE_DOCKER_CLI: {{ "${{matrix.platform.archery_use_docker_cli ||
'1'}}" }}
strategy:
fail-fast: false
matrix:
@@ -44,6 +45,7 @@ jobs:
archery_arch: "arm64v8"
archery_arch_alias: "aarch64"
archery_arch_short: "arm64"
+ archery_use_docker_cli: "0"
steps:
{{ macros.github_checkout_arrow()|indent }}
{{ macros.github_free_space()|indent }}
diff --git a/dev/tasks/linux-packages/github.linux.yml
b/dev/tasks/linux-packages/github.linux.yml
index 6de3edfce0..9e24835b8b 100644
--- a/dev/tasks/linux-packages/github.linux.yml
+++ b/dev/tasks/linux-packages/github.linux.yml
@@ -29,6 +29,7 @@ jobs:
{% endif %}
env:
ARCHITECTURE: {{ architecture }}
+ ARCHERY_USE_DOCKER_CLI: {{ '0' if architecture == 'arm64' else '1' }}
steps:
{{ macros.github_checkout_arrow()|indent }}
{{ macros.github_login_dockerhub()|indent }}
diff --git a/dev/tasks/macros.jinja b/dev/tasks/macros.jinja
index bcafe53066..f55a7f9481 100644
--- a/dev/tasks/macros.jinja
+++ b/dev/tasks/macros.jinja
@@ -23,6 +23,10 @@ on:
push:
branches:
- "*-github-*"
+
+env:
+ ARCHERY_DEBUG: 1
+ ARCHERY_USE_DOCKER_CLI: 1
{% endmacro %}
{%- macro github_checkout_arrow(fetch_depth=1, submodules="recursive",
action_v="4") -%}
diff --git a/dev/tasks/python-wheels/github.linux.yml
b/dev/tasks/python-wheels/github.linux.yml
index 41b18684ce..0ff3c56b69 100644
--- a/dev/tasks/python-wheels/github.linux.yml
+++ b/dev/tasks/python-wheels/github.linux.yml
@@ -31,8 +31,10 @@ jobs:
# archery uses these environment variables
{% if arch == "amd64" %}
ARCH: amd64
+ ARCHERY_USE_DOCKER_CLI: 1
{% else %}
ARCH: arm64v8
+ ARCHERY_USE_DOCKER_CLI: 0
{% endif %}
PYTHON: "{{ python_version }}"
diff --git a/dev/tasks/tasks.yml b/dev/tasks/tasks.yml
index 5e1ef8d13b..cf46cb8c6a 100644
--- a/dev/tasks/tasks.yml
+++ b/dev/tasks/tasks.yml
@@ -1456,12 +1456,16 @@ tasks:
ci: github
template: docker-tests/github.cuda.yml
params:
+ env:
+ ARCHERY_USE_DOCKER_CLI: 0
image: ubuntu-cuda-cpp
test-cuda-python:
ci: github
template: docker-tests/github.cuda.yml
params:
+ env:
+ ARCHERY_USE_DOCKER_CLI: 0
image: ubuntu-cuda-python
############################## Fuzz tests #################################
