This is an automated email from the ASF dual-hosted git repository.
shahar1 pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git
The following commit(s) were added to refs/heads/v3-4-test by this push:
new 0a494eea1aa [v3-4-test] Stop CI jobs from rejecting every restored
image snapshot (#76418) (#77090)
0a494eea1aa is described below
commit 0a494eea1aaa3163b26fcb7e25813ffcd3bc38c2
Author: github-actions[bot]
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Nov 14 18:34:28 2026 +0200
[v3-4-test] Stop CI jobs from rejecting every restored image snapshot
(#76418) (#77090)
(cherry picked from commit 5e3dff5d5e8b6cd9c14dac6619c447b9b83a4952)
Signed-off-by: PoAn Yang <[email protected]>
Co-authored-by: PoAn Yang <[email protected]>
---
scripts/ci/docker_data_root_snapshot.sh | 10 ++-
scripts/tests/ci/test_docker_data_root_snapshot.py | 82 ++++++++++++++++++++++
2 files changed, 90 insertions(+), 2 deletions(-)
diff --git a/scripts/ci/docker_data_root_snapshot.sh
b/scripts/ci/docker_data_root_snapshot.sh
index 350d89f774a..8a33cf9cb40 100755
--- a/scripts/ci/docker_data_root_snapshot.sh
+++ b/scripts/ci/docker_data_root_snapshot.sh
@@ -88,12 +88,18 @@ function start_daemon() {
# Identifies the image by its ID and by the file tree its layers combine into:
a restore that loses
# a whiteout or an opaque-directory marker changes the tree while the ID stays
the same. The listing
# is sorted on the host, since sorting it in the container would add sort's
temporary files to it.
+# The listing leaves out two kinds of size that differ for the same image. A
directory's size
+# depends on the order its entries were added in and on the hash seed of the
filesystem. The size
+# of a file that Docker writes for each container depends on the host. For
example,
+# /etc/resolv.conf holds the host's DNS search domain.
function image_fingerprint() {
local image=$1
local image_id tree
image_id="$(docker image inspect --format '{{.Id}}' "${image}")" || return
1
tree="$(timeout "${DAEMON_TIMEOUT_SECONDS}" docker run --rm --pull never
--network none \
- --entrypoint find "${image}" / -xdev -printf '%p %y %m %U %G %s\n' |
LC_ALL=C sort | sha256sum)" \
+ --entrypoint find "${image}" / -xdev \
+ \( -type d -o -path /etc/hostname -o -path /etc/hosts -o -path
/etc/resolv.conf \) \
+ -printf '%p %y %m %U %G\n' -o -printf '%p %y %m %U %G %s\n' | LC_ALL=C
sort | sha256sum)" \
|| return 1
echo "${image_id} ${tree%% *}"
}
@@ -184,7 +190,7 @@ function restore_snapshot() {
start_daemon
if ! restored_image="$(image_fingerprint "${IMAGE_ID}")" \
|| [[ "${restored_image}" != "${EXPECTED_IMAGE}" ]]; then
- echo "The restored image does not match the snapshot:
'${restored_image:-}' != '${EXPECTED_IMAGE}'"
+ echo "::warning::The restored image does not match the snapshot:
'${restored_image:-}' != '${EXPECTED_IMAGE}'"
exit 4
fi
RESTORE_IN_PROGRESS=false
diff --git a/scripts/tests/ci/test_docker_data_root_snapshot.py
b/scripts/tests/ci/test_docker_data_root_snapshot.py
index 3a7b394cc30..24869abd413 100644
--- a/scripts/tests/ci/test_docker_data_root_snapshot.py
+++ b/scripts/tests/ci/test_docker_data_root_snapshot.py
@@ -34,6 +34,10 @@ STOP = ["sudo", "timeout", "60", "systemctl", "stop",
"docker.socket", "docker"]
START = ["sudo", "timeout", "60", "systemctl", "start", "docker"]
REMOVE_STORE = ["sudo", "rm", "-rf", "/var/lib/docker/image",
"/var/lib/docker/overlay2"]
TREE = hashlib.sha256(b"/ d 755\n/tmp d 1777\n").hexdigest()
+needs_gnu_find = pytest.mark.skipif(
+ subprocess.run(["find", "--version"], capture_output=True,
check=False).returncode != 0,
+ reason="Needs GNU find, which the image and the CI runners have",
+)
@pytest.fixture
@@ -78,6 +82,15 @@ if [[ "${name}" == "docker" ]]; then
if [[ "${RUN_EXIT:-0}" != 0 ]]; then
exit "${RUN_EXIT}"
fi
+ if [[ -n "${IMAGE_ROOT:-}" ]]; then
+ # Runs the script's find expression with IMAGE_ROOT in place
of the container's /.
+ args=()
+ for arg in "${@:11}"; do
+ [[ "${arg}" == /* ]] && arg="${IMAGE_ROOT}${arg}"
+ args+=("${arg}")
+ done
+ exec find "${IMAGE_ROOT}/" "${args[@]}"
+ fi
printf '%b' "${TREE_LISTING:-/tmp d 1777\\n/ d 755\\n}"
;;
esac
@@ -102,6 +115,19 @@ def snapshot(tmp_path):
return snapshot_file
[email protected]
+def image_root(tmp_path):
+ """Stands in for the container's /, so the script's real find expression
has a tree to walk."""
+ root = tmp_path / "rootfs"
+ (root / "etc").mkdir(parents=True)
+ (root / "etc" / "hostname").write_text("abcdef123456\n")
+ (root / "etc" / "hosts").write_text("127.0.0.1\tlocalhost\n")
+ (root / "etc" / "resolv.conf").write_text("nameserver 168.63.129.16\n")
+ (root / "usr").mkdir()
+ (root / "usr" / "file").write_text("content\n")
+ return root
+
+
def run_script(env, *args):
command = ["bash", "--noprofile", "--norc", str(SCRIPT), *args]
result = subprocess.run(
@@ -197,6 +223,7 @@ def
test_restore_empties_store_again_when_image_does_not_match(fake_tools, snaps
result = run_script({**fake_tools, **env}, "restore", str(snapshot))
assert result.returncode == 4
+ assert "::warning::" in result.stdout
commands = read_commands(fake_tools)
start = commands.index(START)
after_start = commands[start + 1 :]
@@ -204,6 +231,61 @@ def
test_restore_empties_store_again_when_image_does_not_match(fake_tools, snaps
assert after_start[-1] == START
+def leave_directory_larger(root):
+ directory = root / "usr"
+ size = directory.stat().st_size
+ entries = [directory / f"temporary-{i}" for i in range(2000)]
+ for entry in entries:
+ entry.touch()
+ for entry in entries:
+ entry.unlink()
+ if directory.stat().st_size == size:
+ pytest.skip("This filesystem shrinks a directory when its entries are
deleted")
+
+
+def change_group(path):
+ group = next((gid for gid in os.getgroups() if gid != path.stat().st_gid),
None)
+ if group is None:
+ pytest.skip("Changing the group needs a user with a second group")
+ os.chown(path, -1, group)
+
+
+@needs_gnu_find
[email protected](
+ ("change", "expected_exit"),
+ [
+ pytest.param(leave_directory_larger, 0, id="directory-size"),
+ pytest.param(
+ lambda root: (root / "etc" / "hostname").write_text("a longer
hostname\n"), 0, id="hostname"
+ ),
+ pytest.param(
+ lambda root: (root / "etc" / "hosts").write_text("127.0.0.1
localhost a\n"), 0, id="hosts"
+ ),
+ pytest.param(
+ lambda root: (root / "etc" / "resolv.conf").write_text("nameserver
1.1.1.1\nsearch a.internal\n"),
+ 0,
+ id="resolv-conf",
+ ),
+ pytest.param(lambda root: (root / "usr" / "file").write_text("cont"),
4, id="truncated-file"),
+ pytest.param(lambda root: change_group(root / "usr" / "file"), 4,
id="file-group"),
+ pytest.param(lambda root: change_group(root / "usr"), 4,
id="directory-group"),
+ ],
+)
+def test_restore_compares_only_what_the_image_decides(
+ fake_tools, tmp_path, image_root, change, expected_exit
+):
+ snapshot_file = tmp_path / "snapshot.tar.zst"
+ env = {**fake_tools, "IMAGE_ROOT": str(image_root), "CI_IMAGES": "abc123"}
+ created = run_script(env, "create", str(snapshot_file))
+ assert created.returncode == 0, created.stdout + created.stderr
+ snapshot_file.write_bytes(b"")
+
+ change(image_root)
+ result = run_script(env, "restore", str(snapshot_file))
+
+ assert result.returncode == expected_exit, result.stdout + result.stderr
+
+
def test_create_keeps_only_the_ci_image(fake_tools, tmp_path):
snapshot_file = tmp_path / "out.tar.zst"
env = {**fake_tools, "CI_IMAGES": "abc123\\nabc123\\n", "ALL_IMAGES":
"abc123\\nother1\\nabc123\\n"}