This is an automated email from the ASF dual-hosted git repository. asorokoumov pushed a commit to branch e2e-tests in repository https://gitbox.apache.org/repos/asf/otava.git
commit 4078dcf0c1f588f898619c572a6d81a122c61869 Author: Alex Sorokoumov <[email protected]> AuthorDate: Wed Nov 26 21:41:05 2025 -0800 Add e2e tests covering CSV and PostgreSQL usage --- tests/csv_e2e_test.py | 205 +++++++++++++++++++++++++++ tests/postgres_e2e_test.py | 339 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 544 insertions(+) diff --git a/tests/csv_e2e_test.py b/tests/csv_e2e_test.py new file mode 100644 index 0000000..a1eff77 --- /dev/null +++ b/tests/csv_e2e_test.py @@ -0,0 +1,205 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import csv +import os +import subprocess +import tempfile +import textwrap +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + + +def test_analyze_csv(): + """ + End-to-end test for the CSV example from docs/CSV.md. + + Writes a temporary CSV and otava.yaml, runs: + uv run otava analyze local.sample + in the temporary directory, and compares stdout to the expected output. + """ + + now = datetime.now() + n = 10 + timestamps = [now - timedelta(days=i) for i in range(n)] + metrics1 = [154023, 138455, 143112, 149190, 132098, 151344, 155145, 148889, 149466, 148209] + metrics2 = [10.43, 10.23, 10.29, 10.91, 10.34, 10.69, 9.23, 9.11, 9.13, 9.03] + data_points = [] + for i in range(n): + data_points.append( + ( + timestamps[i].strftime("%Y.%m.%d %H:%M:%S %z"), # time + "aaa" + str(i), # commit + metrics1[i], + metrics2[i], + ) + ) + + config_content = textwrap.dedent( + """\ + tests: + local.sample: + type: csv + file: data/local_sample.csv + time_column: time + attributes: [commit] + metrics: [metric1, metric2] + csv_options: + delimiter: "," + quotechar: "'" + """ + ) + expected_output = textwrap.dedent( + """\ + time commit metric1 metric2 + ------------------------- -------- --------- --------- + {} aaa0 154023 10.43 + {} aaa1 138455 10.23 + {} aaa2 143112 10.29 + {} aaa3 149190 10.91 + {} aaa4 132098 10.34 + {} aaa5 151344 10.69 + ········· + -12.9% + ········· + {} aaa6 155145 9.23 + {} aaa7 148889 9.11 + {} aaa8 149466 9.13 + {} aaa9 148209 9.03 + """.format(*[ts.astimezone(timezone.utc).strftime("%Y-%m-%d %H:%M:%S +0000") for ts in timestamps]) + ) + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + # create data directory and write CSV + data_dir = td_path / "data" + data_dir.mkdir(parents=True, exist_ok=True) + csv_path = data_dir / "local_sample.csv" + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["time", "commit", "metric1", "metric2"]) + writer.writerows(data_points) + + # write otava.yaml in temp cwd + config_path = td_path / "otava.yaml" + config_path.write_text(config_content, encoding="utf-8") + + # run command + cmd = ["uv", "run", "otava", "analyze", "local.sample"] + proc = subprocess.run( + cmd, + cwd=str(td_path), + capture_output=True, + text=True, + timeout=120, + env=dict(os.environ, OTAVA_CONFIG=config_path) + ) + + if proc.returncode != 0: + pytest.fail( + "Command returned non-zero exit code.\n\n" + f"Command: {cmd!r}\n" + f"Exit code: {proc.returncode}\n\n" + f"Stdout:\n{proc.stdout}\n\n" + f"Stderr:\n{proc.stderr}\n" + ) + + assert proc.stdout.rstrip("\n") == expected_output.rstrip("\n") + + +def test_regressions_csv(): + """ + End-to-end test for the CSV example from docs/CSV.md. + + Writes a temporary CSV and otava.yaml, runs: + uv run otava analyze local.sample + in the temporary directory, and compares stdout to the expected output. + """ + + now = datetime.now() + n = 10 + timestamps = [now - timedelta(days=i) for i in range(n)] + metrics1 = [154023, 138455, 143112, 149190, 132098, 151344, 155145, 148889, 149466, 148209] + metrics2 = [10.43, 10.23, 10.29, 10.91, 10.34, 10.69, 9.23, 9.11, 9.13, 9.03] + data_points = [] + for i in range(n): + data_points.append( + ( + timestamps[i].strftime("%Y.%m.%d %H:%M:%S %z"), # time + "aaa" + str(i), # commit + metrics1[i], + metrics2[i], + ) + ) + + config_content = textwrap.dedent( + """\ + tests: + local.sample: + type: csv + file: data/local_sample.csv + time_column: time + attributes: [commit] + metrics: [metric1, metric2] + csv_options: + delimiter: "," + quotechar: "'" + """ + ) + expected_output = textwrap.dedent( + """\ + local.sample: + metric2 : 10.5 --> 9.12 ( -12.9%) + Regressions in 1 test found + """ + ) + with tempfile.TemporaryDirectory() as td: + td_path = Path(td) + # create data directory and write CSV + data_dir = td_path / "data" + data_dir.mkdir(parents=True, exist_ok=True) + csv_path = data_dir / "local_sample.csv" + with open(csv_path, "w", newline="") as f: + writer = csv.writer(f) + writer.writerow(["time", "commit", "metric1", "metric2"]) + writer.writerows(data_points) + + # write otava.yaml in temp cwd + config_path = td_path / "otava.yaml" + config_path.write_text(config_content, encoding="utf-8") + + # run command + cmd = ["uv", "run", "otava", "regressions", "local.sample"] + proc = subprocess.run( + cmd, + cwd=str(td_path), + capture_output=True, + text=True, + timeout=120, + env=dict(os.environ, OTAVA_CONFIG=config_path) + ) + + if proc.returncode != 0: + pytest.fail( + "Command returned non-zero exit code.\n\n" + f"Command: {cmd!r}\n" + f"Exit code: {proc.returncode}\n\n" + f"Stdout:\n{proc.stdout}\n\n" + f"Stderr:\n{proc.stderr}\n" + ) + assert proc.stdout == expected_output diff --git a/tests/postgres_e2e_test.py b/tests/postgres_e2e_test.py new file mode 100644 index 0000000..276a481 --- /dev/null +++ b/tests/postgres_e2e_test.py @@ -0,0 +1,339 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +import os +import shutil +import subprocess +import textwrap +import time +from contextlib import contextmanager +from pathlib import Path + +import pytest + + +def test_analyze(): + """ + End-to-end test for the PostgreSQL example. + + Starts the docker-compose stack from examples/postgresql/docker-compose.yaml, + waits for Postgres to be ready, runs the otava analysis in a one-off + container, and compares stdout to the expected output (seeded data uses + deterministic 2025 timestamps). + """ + with postgres_container() as postgres_container_id: + # Run the Otava analysis + proc = subprocess.run( + ["uv", "run", "otava","analyze", "aggregate_mem"], + capture_output=True, + text=True, + timeout=600, + env=dict( + os.environ, + OTAVA_CONFIG=Path("examples/postgresql/config/otava.yaml"), + POSTGRES_HOSTNAME="localhost", + POSTGRES_PORT="5432", + POSTGRES_USERNAME="exampleuser", + POSTGRES_PASSWORD="examplepassword", + POSTGRES_DATABASE="benchmark_results", + BRANCH="trunk") + ) + + if proc.returncode != 0: + pytest.fail( + "Command returned non-zero exit code.\n\n" + f"Command: {proc.args!r}\n" + f"Exit code: {proc.returncode}\n\n" + f"Stdout:\n{proc.stdout}\n\n" + f"Stderr:\n{proc.stderr}\n" + ) + + expected_output = textwrap.dedent( + """\ +time experiment_id commit config_id process_cumulative_rate_mean process_cumulative_rate_stderr process_cumulative_rate_diff +------------------------- ------------------ -------- ----------- ------------------------------ -------------------------------- ------------------------------ +2025-03-13 10:03:02 +0000 aggregate-36e5ccd2 36e5ccd2 1 61160 2052 13558 +2025-03-25 10:03:02 +0000 aggregate-d5460f38 d5460f38 1 60160 2142 13454 +2025-04-02 10:03:02 +0000 aggregate-bc9425cb bc9425cb 1 60960 2052 13053 + ······························ + -5.6% + ······························ +2025-04-06 10:03:02 +0000 aggregate-14df1b11 14df1b11 1 57123 2052 14052 +2025-04-13 10:03:02 +0000 aggregate-ac40c0d8 ac40c0d8 1 57980 2052 13521 +2025-04-27 10:03:02 +0000 aggregate-0af4ccbc 0af4ccbc 1 56950 2052 13532 + """ + ) + assert proc.stdout.rstrip("\n") == expected_output.rstrip("\n") + + # Verify the DB was updated with the detected change. + # Query the updated change metric at the detected change point. + query_proc = subprocess.run( + ["docker", "exec", postgres_container_id, + "psql", "-U", "exampleuser", "-d", "benchmark_results", + "-Atc", + """ + SELECT + process_cumulative_rate_mean_rel_forward_change, + process_cumulative_rate_mean_rel_backward_change, + process_cumulative_rate_mean_p_value + FROM results + WHERE experiment_id='aggregate-14df1b11' AND config_id=1; + """], + capture_output=True, + text=True, + timeout=60 + ) + if query_proc.returncode != 0: + pytest.fail( + "Command returned non-zero exit code.\n\n" + f"Command: {query_proc.args!r}\n" + f"Exit code: {query_proc.returncode}\n\n" + f"Stdout:\n{query_proc.stdout}\n\n" + f"Stderr:\n{query_proc.stderr}\n" + ) + + + # psql -Atc returns rows like: value|pvalue + forward_change, backward_change, p_value = query_proc.stdout.strip().split("|") + # --update-postgres was not specified, so no change point should be recorded + assert forward_change == backward_change == p_value == '' + + +def test_analyze_and_update_postgres(): + """ + End-to-end test for the PostgreSQL example. + + Starts the docker-compose stack from examples/postgresql/docker-compose.yaml, + waits for Postgres to be ready, runs the otava analysis in a one-off + container, and compares stdout to the expected output (seeded data uses + deterministic 2025 timestamps). + """ + with postgres_container() as postgres_container_id: + # Run the Otava analysis + proc = subprocess.run( + ["uv", "run", "otava","analyze", "aggregate_mem", "--update-postgres"], + capture_output=True, + text=True, + timeout=600, + env=dict( + os.environ, + OTAVA_CONFIG=Path("examples/postgresql/config/otava.yaml"), + POSTGRES_HOSTNAME="localhost", + POSTGRES_PORT="5432", + POSTGRES_USERNAME="exampleuser", + POSTGRES_PASSWORD="examplepassword", + POSTGRES_DATABASE="benchmark_results", + BRANCH="trunk") + ) + + if proc.returncode != 0: + pytest.fail( + "Command returned non-zero exit code.\n\n" + f"Command: {proc.args!r}\n" + f"Exit code: {proc.returncode}\n\n" + f"Stdout:\n{proc.stdout}\n\n" + f"Stderr:\n{proc.stderr}\n" + ) + + expected_output = textwrap.dedent( + """\ + time experiment_id commit config_id process_cumulative_rate_mean process_cumulative_rate_stderr process_cumulative_rate_diff + ------------------------- ------------------ -------- ----------- ------------------------------ -------------------------------- ------------------------------ + 2025-03-13 10:03:02 +0000 aggregate-36e5ccd2 36e5ccd2 1 61160 2052 13558 + 2025-03-25 10:03:02 +0000 aggregate-d5460f38 d5460f38 1 60160 2142 13454 + 2025-04-02 10:03:02 +0000 aggregate-bc9425cb bc9425cb 1 60960 2052 13053 + ······························ + -5.6% + ······························ + 2025-04-06 10:03:02 +0000 aggregate-14df1b11 14df1b11 1 57123 2052 14052 + 2025-04-13 10:03:02 +0000 aggregate-ac40c0d8 ac40c0d8 1 57980 2052 13521 + 2025-04-27 10:03:02 +0000 aggregate-0af4ccbc 0af4ccbc 1 56950 2052 13532 + """ + ) + assert proc.stdout.rstrip("\n") == expected_output.rstrip("\n") + + # Verify the DB was updated with the detected change. + # Query the updated change metric at the detected change point. + query_proc = subprocess.run( + ["docker", "exec", postgres_container_id, + "psql", "-U", "exampleuser", "-d", "benchmark_results", + "-Atc", + """ + SELECT + process_cumulative_rate_mean_rel_forward_change, + process_cumulative_rate_mean_rel_backward_change, + process_cumulative_rate_mean_p_value + FROM results + WHERE experiment_id='aggregate-14df1b11' AND config_id=1; + """], + capture_output=True, + text=True, + timeout=60 + ) + if query_proc.returncode != 0: + pytest.fail( + "Command returned non-zero exit code.\n\n" + f"Command: {query_proc.args!r}\n" + f"Exit code: {query_proc.returncode}\n\n" + f"Stdout:\n{query_proc.stdout}\n\n" + f"Stderr:\n{query_proc.stderr}\n" + ) + + + # psql -Atc returns rows like: value|pvalue + forward_change, backward_change, p_value = query_proc.stdout.strip().split("|") + forward_change = float(forward_change) + backward_change = float(backward_change) + p_value = float(p_value) + + if abs(forward_change - (-5.6)) > 0.2: + pytest.fail(f"DB change value {forward_change!r} not within tolerance of -5.6") + if abs(backward_change - 5.94) > 0.2: + pytest.fail(f"DB backward change {backward_change!r} not within tolerance of 5.94") + if p_value >= 0.001: + pytest.fail(f"DB p-value {p_value!r} not less than 0.01") + + +def test_regressions(): + """ + End-to-end test for the PostgreSQL regressions command. + + Starts the docker-compose stack from examples/postgresql/docker-compose.yaml, + waits for Postgres to be ready, runs the otava regressions command, + and compares stdout to the expected output. + """ + with postgres_container() as postgres_container_id: + # Run the Otava regressions command + proc = subprocess.run( + ["uv", "run", "otava", "regressions", "aggregate_mem"], + capture_output=True, + text=True, + timeout=600, + env=dict( + os.environ, + OTAVA_CONFIG=Path("examples/postgresql/config/otava.yaml"), + POSTGRES_HOSTNAME="localhost", + POSTGRES_PORT="5432", + POSTGRES_USERNAME="exampleuser", + POSTGRES_PASSWORD="examplepassword", + POSTGRES_DATABASE="benchmark_results", + BRANCH="trunk") + ) + + if proc.returncode != 0: + pytest.fail( + "Command returned non-zero exit code.\n\n" + f"Command: {proc.args!r}\n" + f"Exit code: {proc.returncode}\n\n" + f"Stdout:\n{proc.stdout}\n\n" + f"Stderr:\n{proc.stderr}\n" + ) + + expected_output = textwrap.dedent( + """\ + aggregate_mem: + process_cumulative_rate_mean: 6.08e+04 --> 5.74e+04 ( -5.6%) + Regressions in 1 test found + """ + ) + assert proc.stdout == expected_output + + # Verify the DB was NOT updated since --update-postgres was not specified + query_proc = subprocess.run( + ["docker", "exec", postgres_container_id, + "psql", "-U", "exampleuser", "-d", "benchmark_results", + "-Atc", + """ + SELECT + process_cumulative_rate_mean_rel_forward_change, + process_cumulative_rate_mean_rel_backward_change, + process_cumulative_rate_mean_p_value + FROM results + WHERE experiment_id='aggregate-14df1b11' AND config_id=1; + """], + capture_output=True, + text=True, + timeout=60 + ) + if query_proc.returncode != 0: + pytest.fail( + "Command returned non-zero exit code.\n\n" + f"Command: {query_proc.args!r}\n" + f"Exit code: {query_proc.returncode}\n\n" + f"Stdout:\n{query_proc.stdout}\n\n" + f"Stderr:\n{query_proc.stderr}\n" + ) + + # psql -Atc returns rows like: value|pvalue + forward_change, backward_change, p_value = query_proc.stdout.strip().split("|") + # --update-postgres was not specified, so no change point should be recorded + assert forward_change == backward_change == p_value == '' + + +@contextmanager +def postgres_container(): + """ + Context manager for running a PostgreSQL container. + Yields the container ID and ensures cleanup on exit. + """ + if not shutil.which("docker"): + pytest.fail("docker is not available on PATH") + + container_id = None + try: + # Start postgres container + cmd = [ + "docker", "run", "-d", + "--env", "POSTGRES_USER=exampleuser", + "--env", "POSTGRES_PASSWORD=examplepassword", + "--env", "POSTGRES_DB=benchmark_results", + "--volume", f"{Path('examples/postgresql/init-db').resolve()}:/docker-entrypoint-initdb.d", + "--publish", "5432:5432", + "postgres:latest" + ] + proc = subprocess.run(cmd, capture_output=True, text=True, timeout=60) + if proc.returncode != 0: + pytest.fail( + "Docker command returned non-zero exit code.\n\n" + f"Command: {cmd!r}\n" + f"Exit code: {proc.returncode}\n\n" + f"Stdout:\n{proc.stdout}\n\n" + f"Stderr:\n{proc.stderr}\n" + ) + container_id = proc.stdout.strip() + + # Wait until Postgres responds + deadline = time.time() + 60 + ready = False + while time.time() < deadline: + cmd = ["docker", "exec", container_id, + "pg_isready", "-U", "exampleuser", "-d", "benchmark_results"] + proc = subprocess.run(cmd, capture_output=True, text=True) + if proc.returncode == 0: + ready = True + break + time.sleep(1) + + if not ready: + pytest.fail("Postgres did not become ready within timeout.") + + yield container_id + finally: + if container_id: + res = subprocess.run(["docker", "stop", container_id], capture_output=True, text=True, timeout=60) + if res.returncode != 0: + pytest.fail(f"Docker command returned non-zero exit code: {res.returncode}\nStdout: {res.stdout}\nStderr: {res.stderr}") \ No newline at end of file
