This is an automated email from the ASF dual-hosted git repository.

shahar1 pushed a commit to branch v3-4-test
in repository https://gitbox.apache.org/repos/asf/airflow.git


The following commit(s) were added to refs/heads/v3-4-test by this push:
     new 0a494eea1aa [v3-4-test] Stop CI jobs from rejecting every restored 
image snapshot (#76418) (#77090)
0a494eea1aa is described below

commit 0a494eea1aaa3163b26fcb7e25813ffcd3bc38c2
Author: github-actions[bot] 
<41898282+github-actions[bot]@users.noreply.github.com>
AuthorDate: Sat Nov 14 18:34:28 2026 +0200

    [v3-4-test] Stop CI jobs from rejecting every restored image snapshot 
(#76418) (#77090)
    
    (cherry picked from commit 5e3dff5d5e8b6cd9c14dac6619c447b9b83a4952)
    
    Signed-off-by: PoAn Yang <[email protected]>
    Co-authored-by: PoAn Yang <[email protected]>
---
 scripts/ci/docker_data_root_snapshot.sh            | 10 ++-
 scripts/tests/ci/test_docker_data_root_snapshot.py | 82 ++++++++++++++++++++++
 2 files changed, 90 insertions(+), 2 deletions(-)

diff --git a/scripts/ci/docker_data_root_snapshot.sh 
b/scripts/ci/docker_data_root_snapshot.sh
index 350d89f774a..8a33cf9cb40 100755
--- a/scripts/ci/docker_data_root_snapshot.sh
+++ b/scripts/ci/docker_data_root_snapshot.sh
@@ -88,12 +88,18 @@ function start_daemon() {
 # Identifies the image by its ID and by the file tree its layers combine into: 
a restore that loses
 # a whiteout or an opaque-directory marker changes the tree while the ID stays 
the same. The listing
 # is sorted on the host, since sorting it in the container would add sort's 
temporary files to it.
+# The listing leaves out two kinds of size that differ for the same image. A 
directory's size
+# depends on the order its entries were added in and on the hash seed of the 
filesystem. The size
+# of a file that Docker writes for each container depends on the host. For 
example,
+# /etc/resolv.conf holds the host's DNS search domain.
 function image_fingerprint() {
     local image=$1
     local image_id tree
     image_id="$(docker image inspect --format '{{.Id}}' "${image}")" || return 
1
     tree="$(timeout "${DAEMON_TIMEOUT_SECONDS}" docker run --rm --pull never 
--network none \
-        --entrypoint find "${image}" / -xdev -printf '%p %y %m %U %G %s\n' | 
LC_ALL=C sort | sha256sum)" \
+        --entrypoint find "${image}" / -xdev \
+        \( -type d -o -path /etc/hostname -o -path /etc/hosts -o -path 
/etc/resolv.conf \) \
+        -printf '%p %y %m %U %G\n' -o -printf '%p %y %m %U %G %s\n' | LC_ALL=C 
sort | sha256sum)" \
         || return 1
     echo "${image_id} ${tree%% *}"
 }
@@ -184,7 +190,7 @@ function restore_snapshot() {
     start_daemon
     if ! restored_image="$(image_fingerprint "${IMAGE_ID}")" \
             || [[ "${restored_image}" != "${EXPECTED_IMAGE}" ]]; then
-        echo "The restored image does not match the snapshot: 
'${restored_image:-}' != '${EXPECTED_IMAGE}'"
+        echo "::warning::The restored image does not match the snapshot: 
'${restored_image:-}' != '${EXPECTED_IMAGE}'"
         exit 4
     fi
     RESTORE_IN_PROGRESS=false
diff --git a/scripts/tests/ci/test_docker_data_root_snapshot.py 
b/scripts/tests/ci/test_docker_data_root_snapshot.py
index 3a7b394cc30..24869abd413 100644
--- a/scripts/tests/ci/test_docker_data_root_snapshot.py
+++ b/scripts/tests/ci/test_docker_data_root_snapshot.py
@@ -34,6 +34,10 @@ STOP = ["sudo", "timeout", "60", "systemctl", "stop", 
"docker.socket", "docker"]
 START = ["sudo", "timeout", "60", "systemctl", "start", "docker"]
 REMOVE_STORE = ["sudo", "rm", "-rf", "/var/lib/docker/image", 
"/var/lib/docker/overlay2"]
 TREE = hashlib.sha256(b"/ d 755\n/tmp d 1777\n").hexdigest()
+needs_gnu_find = pytest.mark.skipif(
+    subprocess.run(["find", "--version"], capture_output=True, 
check=False).returncode != 0,
+    reason="Needs GNU find, which the image and the CI runners have",
+)
 
 
 @pytest.fixture
@@ -78,6 +82,15 @@ if [[ "${name}" == "docker" ]]; then
             if [[ "${RUN_EXIT:-0}" != 0 ]]; then
                 exit "${RUN_EXIT}"
             fi
+            if [[ -n "${IMAGE_ROOT:-}" ]]; then
+                # Runs the script's find expression with IMAGE_ROOT in place 
of the container's /.
+                args=()
+                for arg in "${@:11}"; do
+                    [[ "${arg}" == /* ]] && arg="${IMAGE_ROOT}${arg}"
+                    args+=("${arg}")
+                done
+                exec find "${IMAGE_ROOT}/" "${args[@]}"
+            fi
             printf '%b' "${TREE_LISTING:-/tmp d 1777\\n/ d 755\\n}"
             ;;
     esac
@@ -102,6 +115,19 @@ def snapshot(tmp_path):
     return snapshot_file
 
 
[email protected]
+def image_root(tmp_path):
+    """Stands in for the container's /, so the script's real find expression 
has a tree to walk."""
+    root = tmp_path / "rootfs"
+    (root / "etc").mkdir(parents=True)
+    (root / "etc" / "hostname").write_text("abcdef123456\n")
+    (root / "etc" / "hosts").write_text("127.0.0.1\tlocalhost\n")
+    (root / "etc" / "resolv.conf").write_text("nameserver 168.63.129.16\n")
+    (root / "usr").mkdir()
+    (root / "usr" / "file").write_text("content\n")
+    return root
+
+
 def run_script(env, *args):
     command = ["bash", "--noprofile", "--norc", str(SCRIPT), *args]
     result = subprocess.run(
@@ -197,6 +223,7 @@ def 
test_restore_empties_store_again_when_image_does_not_match(fake_tools, snaps
     result = run_script({**fake_tools, **env}, "restore", str(snapshot))
 
     assert result.returncode == 4
+    assert "::warning::" in result.stdout
     commands = read_commands(fake_tools)
     start = commands.index(START)
     after_start = commands[start + 1 :]
@@ -204,6 +231,61 @@ def 
test_restore_empties_store_again_when_image_does_not_match(fake_tools, snaps
     assert after_start[-1] == START
 
 
+def leave_directory_larger(root):
+    directory = root / "usr"
+    size = directory.stat().st_size
+    entries = [directory / f"temporary-{i}" for i in range(2000)]
+    for entry in entries:
+        entry.touch()
+    for entry in entries:
+        entry.unlink()
+    if directory.stat().st_size == size:
+        pytest.skip("This filesystem shrinks a directory when its entries are 
deleted")
+
+
+def change_group(path):
+    group = next((gid for gid in os.getgroups() if gid != path.stat().st_gid), 
None)
+    if group is None:
+        pytest.skip("Changing the group needs a user with a second group")
+    os.chown(path, -1, group)
+
+
+@needs_gnu_find
[email protected](
+    ("change", "expected_exit"),
+    [
+        pytest.param(leave_directory_larger, 0, id="directory-size"),
+        pytest.param(
+            lambda root: (root / "etc" / "hostname").write_text("a longer 
hostname\n"), 0, id="hostname"
+        ),
+        pytest.param(
+            lambda root: (root / "etc" / "hosts").write_text("127.0.0.1 
localhost a\n"), 0, id="hosts"
+        ),
+        pytest.param(
+            lambda root: (root / "etc" / "resolv.conf").write_text("nameserver 
1.1.1.1\nsearch a.internal\n"),
+            0,
+            id="resolv-conf",
+        ),
+        pytest.param(lambda root: (root / "usr" / "file").write_text("cont"), 
4, id="truncated-file"),
+        pytest.param(lambda root: change_group(root / "usr" / "file"), 4, 
id="file-group"),
+        pytest.param(lambda root: change_group(root / "usr"), 4, 
id="directory-group"),
+    ],
+)
+def test_restore_compares_only_what_the_image_decides(
+    fake_tools, tmp_path, image_root, change, expected_exit
+):
+    snapshot_file = tmp_path / "snapshot.tar.zst"
+    env = {**fake_tools, "IMAGE_ROOT": str(image_root), "CI_IMAGES": "abc123"}
+    created = run_script(env, "create", str(snapshot_file))
+    assert created.returncode == 0, created.stdout + created.stderr
+    snapshot_file.write_bytes(b"")
+
+    change(image_root)
+    result = run_script(env, "restore", str(snapshot_file))
+
+    assert result.returncode == expected_exit, result.stdout + result.stderr
+
+
 def test_create_keeps_only_the_ci_image(fake_tools, tmp_path):
     snapshot_file = tmp_path / "out.tar.zst"
     env = {**fake_tools, "CI_IMAGES": "abc123\\nabc123\\n", "ALL_IMAGES": 
"abc123\\nother1\\nabc123\\n"}

Reply via email to