Module: Mesa Branch: main Commit: 60cd0af06c081a6762d0598a9dfbbfc37c2b65d3 URL: http://cgit.freedesktop.org/mesa/mesa/commit/?id=60cd0af06c081a6762d0598a9dfbbfc37c2b65d3
Author: Guilherme Gallo <guilherme.ga...@collabora.com> Date: Thu Oct 26 08:54:54 2023 -0300 ci/lava: Add unit tests covering job definition Add two unit tests related to the LAVA job definition. test_generate_lava_job_definition_sanity checks for the most important fields, deploy actions, namespaces etc. test_lava_job_definition compares the generated definition with static skeleton YAML files committed inside tests/data folder. Signed-off-by: Guilherme Gallo <guilherme.ga...@collabora.com> Part-of: <https://gitlab.freedesktop.org/mesa/mesa/-/merge_requests/25912> --- .gitlab-ci/lava/utils/ssh_job_definition.py | 1 - .../FASTBOOT_force_uart=False_job_definition.yaml | 142 +++++++++++++++ .../FASTBOOT_force_uart=True_job_definition.yaml | 96 ++++++++++ .../UBOOT_force_uart=False_job_definition.yaml | 114 ++++++++++++ .../data/UBOOT_force_uart=True_job_definition.yaml | 70 ++++++++ .gitlab-ci/tests/utils/test_lava_job_definition.py | 197 +++++++++++++++++++++ 6 files changed, 619 insertions(+), 1 deletion(-) diff --git a/.gitlab-ci/lava/utils/ssh_job_definition.py b/.gitlab-ci/lava/utils/ssh_job_definition.py index e3bfad1ba96..eae8f6c2328 100644 --- a/.gitlab-ci/lava/utils/ssh_job_definition.py +++ b/.gitlab-ci/lava/utils/ssh_job_definition.py @@ -181,7 +181,6 @@ def wrap_final_deploy_action(final_deploy_action: dict): "namespace": "dut", "failure_retry": NUMBER_OF_ATTEMPTS_LAVA_BOOT, "timeout": {"minutes": 10}, - "timeouts": {"http-download": {"minutes": 2}}, } final_deploy_action.update(wrap) diff --git a/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml b/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml new file mode 100644 index 00000000000..b2b5ee7e1f9 --- /dev/null +++ b/.gitlab-ci/tests/data/FASTBOOT_force_uart=False_job_definition.yaml @@ -0,0 +1,142 @@ +job_name: 'test-project: my_pipeline_info' +device_type: my_fastboot_device_type +visibility: + group: + - my_visibility_group +priority: 75 +context: + extra_nfsroot_args: ' init=/init rootwait usbcore.quirks=0bda:8153:k' +timeouts: + job: + minutes: 10 + actions: + depthcharge-retry: + minutes: 4 + depthcharge-start: + minutes: 1 + depthcharge-action: + minutes: 15 +actions: +- deploy: + timeout: + minutes: 10 + to: nfs + nfsrootfs: + url: None/lava-rootfs.tar.zst + compression: zstd + namespace: dut +- deploy: + timeout: + minutes: 5 + to: downloads + os: oe + images: + kernel: + url: None/None + dtb: + url: None/my_dtb_filename.dtb + postprocess: + docker: + image: registry.gitlab.collabora.com/lava/health-check-docker + steps: + - cat Image.gz my_dtb_filename.dtb > Image.gz+dtb + - mkbootimg --kernel Image.gz+dtb --cmdline "root=/dev/nfs rw nfsroot=$NFS_SERVER_IP:$NFS_ROOTFS,tcp,hard + rootwait ip=dhcp init=/init" --pagesize 4096 --base 0x80000000 -o boot.img + namespace: dut +- deploy: + timeout: + minutes: 10 + to: fastboot + docker: + image: registry.gitlab.collabora.com/lava/health-check-docker + images: + boot: + url: downloads://boot.img + namespace: dut + failure_retry: 3 +- boot: + timeout: + minutes: 2 + docker: + image: registry.gitlab.collabora.com/lava/health-check-docker + failure_retry: 3 + method: fastboot + prompts: + - 'lava-shell:' + commands: + - set_active a + namespace: dut + auto_login: + login_commands: + - dropbear -R -B + - touch /dut_ready + login_prompt: 'ogin:' + username: '' +- test: + namespace: dut + definitions: + - from: inline + name: setup-ssh-server + path: inline-setup-ssh-server + repository: + metadata: + format: Lava-Test Test Definition 1.0 + name: dut-env-export + run: + steps: + - |- + echo test FASTBOOT + - export -p > /dut-env-vars.sh +- test: + namespace: container + timeout: + minutes: 10 + failure_retry: 3 + definitions: + - name: docker_ssh_client + from: inline + path: inline/docker_ssh_client.yaml + repository: + metadata: + name: mesa + description: Mesa test plan + format: Lava-Test Test Definition 1.0 + run: + steps: + - |- + set -ex + timeout 1m bash << EOF + while [ -z "$(lava-target-ip)" ]; do + echo Waiting for DUT to join LAN; + sleep 1; + done + EOF + + ping -c 5 -w 60 $(lava-target-ip) + + lava_ssh_test_case() { + set -x + local test_case="${1}" + shift + lava-test-case "${test_case}" --shell \ + ssh ${SSH_PTY_ARGS:--T} \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + root@$(lava-target-ip) "${@}" + } + - lava_ssh_test_case 'wait_for_dut_login' << EOF + - while [ ! -e /dut_ready ]; do sleep 1; done; + - EOF + - |- + lava_ssh_test_case 'artifact_download' 'bash --' << EOF + source /dut-env-vars.sh + set -ex + curl -L --retry 4 -f --retry-all-errors --retry-delay 60 None | tar -xz -C / + mkdir -p /ci/project/dir + curl -L --retry 4 -f --retry-all-errors --retry-delay 60 None | tar --zstd -x -C /ci/project/dir + echo Could not find jwt file, disabling S3 requests... + sed -i '/S3_RESULTS_UPLOAD/d' /set-job-env-vars.sh + EOF + - export SSH_PTY_ARGS=-tt + - lava_ssh_test_case 'test-project_dut' '"cd / && /init-stage2.sh"' + docker: + image: diff --git a/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml b/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml new file mode 100644 index 00000000000..65b817d805e --- /dev/null +++ b/.gitlab-ci/tests/data/FASTBOOT_force_uart=True_job_definition.yaml @@ -0,0 +1,96 @@ +job_name: 'test-project: my_pipeline_info' +device_type: my_fastboot_device_type +visibility: + group: + - my_visibility_group +priority: 75 +context: + extra_nfsroot_args: ' init=/init rootwait usbcore.quirks=0bda:8153:k' +timeouts: + job: + minutes: 10 + actions: + depthcharge-retry: + minutes: 4 + depthcharge-start: + minutes: 1 + depthcharge-action: + minutes: 15 +actions: +- deploy: + timeout: + minutes: 10 + to: nfs + nfsrootfs: + url: None/lava-rootfs.tar.zst + compression: zstd +- deploy: + timeout: + minutes: 5 + to: downloads + os: oe + images: + kernel: + url: None/None + dtb: + url: None/my_dtb_filename.dtb + postprocess: + docker: + image: registry.gitlab.collabora.com/lava/health-check-docker + steps: + - cat Image.gz my_dtb_filename.dtb > Image.gz+dtb + - mkbootimg --kernel Image.gz+dtb --cmdline "root=/dev/nfs rw nfsroot=$NFS_SERVER_IP:$NFS_ROOTFS,tcp,hard + rootwait ip=dhcp init=/init" --pagesize 4096 --base 0x80000000 -o boot.img +- deploy: + timeout: + minutes: 2 + to: fastboot + docker: + image: registry.gitlab.collabora.com/lava/health-check-docker + images: + boot: + url: downloads://boot.img +- boot: + timeout: + minutes: 2 + docker: + image: registry.gitlab.collabora.com/lava/health-check-docker + failure_retry: 3 + method: fastboot + prompts: + - 'lava-shell:' + commands: + - set_active a +- test: + timeout: + minutes: 10 + failure_retry: 1 + definitions: + - name: mesa + from: inline + lava-signal: kmsg + path: inline/mesa.yaml + repository: + metadata: + name: mesa + description: Mesa test plan + os: + - oe + scope: + - functional + format: Lava-Test Test Definition 1.0 + run: + steps: + - echo test FASTBOOT + - set -ex + - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 None | tar -xz + -C / + - mkdir -p /ci/project/dir + - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 None | tar --zstd + -x -C /ci/project/dir + - echo Could not find jwt file, disabling S3 requests... + - sed -i '/S3_RESULTS_UPLOAD/d' /set-job-env-vars.sh + - mkdir -p /ci/project/dir + - curl None | tar --zstd -x -C /ci/project/dir + - sleep 1 + - lava-test-case 'test-project_dut' --shell /init-stage2.sh diff --git a/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml b/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml new file mode 100644 index 00000000000..721eba24c2f --- /dev/null +++ b/.gitlab-ci/tests/data/UBOOT_force_uart=False_job_definition.yaml @@ -0,0 +1,114 @@ +job_name: 'test-project: my_pipeline_info' +device_type: my_uboot_device_type +visibility: + group: + - my_visibility_group +priority: 75 +context: + extra_nfsroot_args: ' init=/init rootwait usbcore.quirks=0bda:8153:k' +timeouts: + job: + minutes: 10 + actions: + depthcharge-retry: + minutes: 4 + depthcharge-start: + minutes: 1 + depthcharge-action: + minutes: 15 +actions: +- deploy: + timeout: + minutes: 10 + to: tftp + os: oe + kernel: + url: None/None + nfsrootfs: + url: None/lava-rootfs.tar.zst + compression: zstd + dtb: + url: None/my_dtb_filename.dtb + namespace: dut + failure_retry: 3 +- boot: + failure_retry: 3 + method: u-boot + prompts: + - 'lava-shell:' + commands: nfs + namespace: dut + auto_login: + login_commands: + - dropbear -R -B + - touch /dut_ready + login_prompt: 'ogin:' + username: '' +- test: + namespace: dut + definitions: + - from: inline + name: setup-ssh-server + path: inline-setup-ssh-server + repository: + metadata: + format: Lava-Test Test Definition 1.0 + name: dut-env-export + run: + steps: + - |- + echo test UBOOT + - export -p > /dut-env-vars.sh +- test: + namespace: container + timeout: + minutes: 10 + failure_retry: 3 + definitions: + - name: docker_ssh_client + from: inline + path: inline/docker_ssh_client.yaml + repository: + metadata: + name: mesa + description: Mesa test plan + format: Lava-Test Test Definition 1.0 + run: + steps: + - |- + set -ex + timeout 1m bash << EOF + while [ -z "$(lava-target-ip)" ]; do + echo Waiting for DUT to join LAN; + sleep 1; + done + EOF + + ping -c 5 -w 60 $(lava-target-ip) + + lava_ssh_test_case() { + set -x + local test_case="${1}" + shift + lava-test-case "${test_case}" --shell \ + ssh ${SSH_PTY_ARGS:--T} \ + -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ + root@$(lava-target-ip) "${@}" + } + - lava_ssh_test_case 'wait_for_dut_login' << EOF + - while [ ! -e /dut_ready ]; do sleep 1; done; + - EOF + - |- + lava_ssh_test_case 'artifact_download' 'bash --' << EOF + source /dut-env-vars.sh + set -ex + curl -L --retry 4 -f --retry-all-errors --retry-delay 60 None | tar -xz -C / + mkdir -p /ci/project/dir + curl -L --retry 4 -f --retry-all-errors --retry-delay 60 None | tar --zstd -x -C /ci/project/dir + echo Could not find jwt file, disabling S3 requests... + sed -i '/S3_RESULTS_UPLOAD/d' /set-job-env-vars.sh + EOF + - export SSH_PTY_ARGS=-tt + - lava_ssh_test_case 'test-project_dut' '"cd / && /init-stage2.sh"' + docker: + image: diff --git a/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml b/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml new file mode 100644 index 00000000000..38ae766b594 --- /dev/null +++ b/.gitlab-ci/tests/data/UBOOT_force_uart=True_job_definition.yaml @@ -0,0 +1,70 @@ +job_name: 'test-project: my_pipeline_info' +device_type: my_uboot_device_type +visibility: + group: + - my_visibility_group +priority: 75 +context: + extra_nfsroot_args: ' init=/init rootwait usbcore.quirks=0bda:8153:k' +timeouts: + job: + minutes: 10 + actions: + depthcharge-retry: + minutes: 4 + depthcharge-start: + minutes: 1 + depthcharge-action: + minutes: 15 +actions: +- deploy: + timeout: + minutes: 5 + to: tftp + os: oe + kernel: + url: None/None + nfsrootfs: + url: None/lava-rootfs.tar.zst + compression: zstd + dtb: + url: None/my_dtb_filename.dtb +- boot: + failure_retry: 3 + method: u-boot + prompts: + - 'lava-shell:' + commands: nfs +- test: + timeout: + minutes: 10 + failure_retry: 1 + definitions: + - name: mesa + from: inline + lava-signal: kmsg + path: inline/mesa.yaml + repository: + metadata: + name: mesa + description: Mesa test plan + os: + - oe + scope: + - functional + format: Lava-Test Test Definition 1.0 + run: + steps: + - echo test UBOOT + - set -ex + - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 None | tar -xz + -C / + - mkdir -p /ci/project/dir + - curl -L --retry 4 -f --retry-all-errors --retry-delay 60 None | tar --zstd + -x -C /ci/project/dir + - echo Could not find jwt file, disabling S3 requests... + - sed -i '/S3_RESULTS_UPLOAD/d' /set-job-env-vars.sh + - mkdir -p /ci/project/dir + - curl None | tar --zstd -x -C /ci/project/dir + - sleep 1 + - lava-test-case 'test-project_dut' --shell /init-stage2.sh diff --git a/.gitlab-ci/tests/utils/test_lava_job_definition.py b/.gitlab-ci/tests/utils/test_lava_job_definition.py new file mode 100644 index 00000000000..a02378a06d1 --- /dev/null +++ b/.gitlab-ci/tests/utils/test_lava_job_definition.py @@ -0,0 +1,197 @@ +import importlib +import os +import re +from itertools import chain +from pathlib import Path +from typing import Any, Iterable, Literal +from unittest import mock + +import lava.utils.constants +import pytest +from lava.lava_job_submitter import LAVAJobSubmitter +from lava.utils.lava_job_definition import LAVAJobDefinition +from ruamel.yaml import YAML + + +def flatten(iterable: Iterable[Iterable[Any]]) -> list[Any]: + return list(chain.from_iterable(iterable)) + + +# mock shell file +@pytest.fixture(scope="session") +def shell_file(tmp_path_factory): + def create_shell_file(content: str = "# test"): + shell_file = tmp_path_factory.mktemp("data") / "shell_file.sh" + shell_file.write_text(content) + return shell_file + + return create_shell_file + + +# fn to load the data file from $CWD/data using pathlib +def load_data_file(filename): + return Path(__file__).parent.parent / "data" / filename + + +def load_yaml_file(filename) -> dict: + with open(load_data_file(filename)) as f: + return YAML().load(f) + + +def job_submitter_factory(mode: Literal["UBOOT", "FASTBOOT"], shell_file): + if mode == "UBOOT": + boot_method = "u-boot" + device_type = "my_uboot_device_type" + elif mode == "FASTBOOT": + boot_method = "fastboot" + device_type = "my_fastboot_device_type" + + job_timeout_min = 10 + mesa_job_name = "dut test" + pipeline_info = "my_pipeline_info" + project_name = "test-project" + visibility_group = "my_visibility_group" + + return LAVAJobSubmitter( + boot_method=boot_method, + ci_project_dir="/ci/project/dir", + device_type=device_type, + dtb_filename="my_dtb_filename", + first_stage_init=shell_file, + job_timeout_min=job_timeout_min, + mesa_job_name=mesa_job_name, + pipeline_info=pipeline_info, + visibility_group=visibility_group, + project_name=project_name, + ) + + +@pytest.fixture +def clear_env_vars(autouse=True): + with mock.patch.dict(os.environ) as environ: + # Remove all LAVA-related environment variables to make the test more robust + # and deterministic, once a envvar is capable of overriding the default value + for key in environ: + if any(kw in key for kw in ("LAVA_", "CI_", "JOB_", "RUNNER_", "DEVICE_")): + del environ[key] + # reload lava.utils.constants to update the JOB_PRIORITY value + importlib.reload(lava.utils.constants) + importlib.reload(lava.utils.lava_job_definition) + yield + + +@pytest.fixture +def mock_collabora_farm(clear_env_vars, monkeypatch): + # Mock a Collabora farm-like device runner tag to enable SSH execution + monkeypatch.setenv("RUNNER_TAG", "mesa-ci-1234-lava-collabora") + + +@pytest.mark.parametrize("force_uart", [True, False], ids=["SSH", "UART"]) +@pytest.mark.parametrize("mode", ["UBOOT", "FASTBOOT"]) +def test_generate_lava_job_definition_sanity( + force_uart, mode, shell_file, mock_collabora_farm, monkeypatch +): + monkeypatch.setattr(lava.utils.lava_job_definition, "FORCE_UART", force_uart) + + init_script_content = f"echo test {mode}" + job_submitter = job_submitter_factory(mode, shell_file(init_script_content)) + job_definition = LAVAJobDefinition(job_submitter).generate_lava_job_definition() + + # Load the YAML output and check that it contains the expected keys and values + yaml = YAML() + job_dict = yaml.load(job_definition) + yaml.dump(job_dict, Path(f"/tmp/{mode}_force_uart={force_uart}_job_definition.yaml")) + assert job_dict["device_type"] == job_submitter.device_type + assert job_dict["visibility"]["group"] == [job_submitter.visibility_group] + assert job_dict["timeouts"]["job"]["minutes"] == job_submitter.job_timeout_min + assert job_dict["context"]["extra_nfsroot_args"] + assert job_dict["timeouts"]["actions"] + + assert len(job_dict["actions"]) == 3 if mode == "UART" else 5 + + last_test_action = job_dict["actions"][-1]["test"] + # TODO: Remove hardcoded "mesa" test name, as this submitter is being used by other projects + first_test_name = last_test_action["definitions"][0]["name"] + is_running_ssh = "ssh" in first_test_name + # if force_uart, is_ssh must be False. If is_ssh, force_uart must be False. Both can be False + assert not (is_running_ssh and force_uart) + assert last_test_action["failure_retry"] == 3 if is_running_ssh else 1 + + run_steps = "".join(last_test_action["definitions"][0]["repository"]["run"]["steps"]) + # Check for project name in lava-test-case + assert re.search(rf"lava.?\S*.test.case.*{job_submitter.project_name}", run_steps) + + action_names = flatten(j.keys() for j in job_dict["actions"]) + if is_running_ssh: + assert action_names == ( + [ + "deploy", + "boot", + "test", # DUT: SSH server + "test", # Docker: SSH client + ] + if mode == "UBOOT" + else [ + "deploy", # NFS + "deploy", # Image generation + "deploy", # Image deployment + "boot", + "test", # DUT: SSH server + "test", # Docker: SSH client + ] + ) + test_action_server = job_dict["actions"][-2]["test"] + # SSH server in the DUT + assert test_action_server["namespace"] == "dut" + # SSH client via docker + assert last_test_action["namespace"] == "container" + + boot_action = next(a["boot"] for a in job_dict["actions"] if "boot" in a) + assert boot_action["namespace"] == "dut" + + # SSH server bootstrapping + assert "dropbear" in "".join(boot_action["auto_login"]["login_commands"]) + return + + # ---- Not SSH job + assert action_names == ( + [ + "deploy", + "boot", + "test", + ] + if mode == "UBOOT" + else [ + "deploy", # NFS + "deploy", # Image generation + "deploy", # Image deployment + "boot", + "test", + ] + ) + assert init_script_content in run_steps + + +# use yaml files from tests/data/ to test the job definition generation +@pytest.mark.parametrize("force_uart", [False, True], ids=["SSH", "UART"]) +@pytest.mark.parametrize("mode", ["UBOOT", "FASTBOOT"]) +def test_lava_job_definition(mode, force_uart, shell_file, mock_collabora_farm, monkeypatch): + monkeypatch.setattr(lava.utils.lava_job_definition, "FORCE_UART", force_uart) + + yaml = YAML() + yaml.default_flow_style = False + + # Load the YAML output and check that it contains the expected keys and values + expected_job_dict = load_yaml_file(f"{mode}_force_uart={force_uart}_job_definition.yaml") + + init_script_content = f"echo test {mode}" + job_submitter = job_submitter_factory(mode, shell_file(init_script_content)) + job_definition = LAVAJobDefinition(job_submitter).generate_lava_job_definition() + + job_dict = yaml.load(job_definition) + + # Uncomment the following to update the expected YAML files + # yaml.dump(job_dict, Path(f"../../data/{mode}_force_uart={force_uart}_job_definition.yaml")) + + # Check that the generated job definition matches the expected one + assert job_dict == expected_job_dict
