This is an automated email from the ASF dual-hosted git repository. tuhaihe pushed a commit to branch main in repository https://gitbox.apache.org/repos/asf/cloudberry.git
commit c781604c5bad070971016dee37a7843fd7d7560a Author: liushengsong <[email protected]> AuthorDate: Tue Jun 2 15:54:38 2026 +0800 Fix vacuum_progress_column/row flaky tests by ensuring FTS detects mirror down The vacuum_progress tests simulate mirror failure during vacuum to test worker process change behavior. Previously, after stopping the mirror and resuming the walsender, the tests relied on passive FTS detection which could time out in CI. Add wait_for_mirror_down() helper that actively triggers FTS probe scans until mirror is confirmed down, ensuring the FTS version change propagates to QD and gang gets properly reset. Also update vacuum_progress_column expected output for post-cleanup phase progress values, which now reflect actual work done by the new vacuum worker (heap_blks_vacuumed, index_vacuum_count) after the worker change is reliably triggered. --- .../isolation2/expected/vacuum_progress_column.out | 17 +++++++++++---- .../isolation2/expected/vacuum_progress_row.out | 17 +++++++++++++++ src/test/isolation2/sql/vacuum_progress_column.sql | 21 ++++++++++++++++++ src/test/isolation2/sql/vacuum_progress_row.sql | 25 ++++++++++++++++++++++ 4 files changed, 76 insertions(+), 4 deletions(-) diff --git a/src/test/isolation2/expected/vacuum_progress_column.out b/src/test/isolation2/expected/vacuum_progress_column.out index 12a3bceb78f..fcc983f1e43 100644 --- a/src/test/isolation2/expected/vacuum_progress_column.out +++ b/src/test/isolation2/expected/vacuum_progress_column.out @@ -339,6 +339,15 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he -------------------------- Success: (1 row) +-- Trigger FTS probe repeatedly until mirror is marked as down, so that +-- FTS version change propagates to QD and gang gets reset. +CREATE OR REPLACE FUNCTION wait_for_mirror_down(content_id int) RETURNS bool AS $$ declare /* in func */ retries int; /* in func */ begin /* in func */ retries := 60; /* in func */ loop /* in func */ perform gp_request_fts_probe_scan(); /* in func */ if (select status = 'd' from gp_segment_configuration where content = content_id and role = 'm') then /* in func */ return true; /* in func */ end if; /* in func */ if retries <= 0 then /* in func */ return false; /* in func */ end if; /* in [...] +CREATE +2: SELECT wait_for_mirror_down(1); + wait_for_mirror_down +---------------------- + t +(1 row) -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -389,14 +398,14 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he select gp_segment_id, relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum where gp_segment_id > -1; gp_segment_id | relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------------+---------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - 2 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 - 0 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 - 1 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 + 2 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 9 | 2 | 0 | 0 + 0 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 9 | 2 | 0 | 0 + 1 | vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 9 | 2 | 0 | 0 (3 rows) select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, heap_blks_vacuumed, index_vacuum_count, max_dead_tuples, num_dead_tuples from gp_stat_progress_vacuum_summary; relname | phase | heap_blks_total | heap_blks_scanned | heap_blks_vacuumed | index_vacuum_count | max_dead_tuples | num_dead_tuples ---------------------------+-------------------------------+-----------------+-------------------+--------------------+--------------------+-----------------+----------------- - vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 0 | 0 | 0 | 0 + vacuum_progress_ao_column | append-optimized post-cleanup | 0 | 0 | 27 | 6 | 0 | 0 (1 row) 2: SELECT gp_inject_fault('vacuum_ao_post_cleanup_end', 'reset', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; diff --git a/src/test/isolation2/expected/vacuum_progress_row.out b/src/test/isolation2/expected/vacuum_progress_row.out index 414e2e0b7c4..29809709eab 100644 --- a/src/test/isolation2/expected/vacuum_progress_row.out +++ b/src/test/isolation2/expected/vacuum_progress_row.out @@ -292,6 +292,9 @@ SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuu -- Current behavior is it will clear previous compact phase num_dead_tuples in post-cleanup -- phase (at injecting point vacuum_ao_post_cleanup_end), which is different from above case -- in which vacuum worker isn't changed. + +CREATE OR REPLACE FUNCTION wait_for_mirror_down(content_id int) RETURNS bool AS $$ declare /* in func */ retries int; /* in func */ begin /* in func */ retries := 60; /* in func */ loop /* in func */ perform gp_request_fts_probe_scan(); /* in func */ if (select status = 'd' from gp_segment_configuration where content = content_id and role = 'm') then /* in func */ return true; /* in func */ end if; /* in func */ if retries <= 0 then /* in func */ return false; /* in func */ end if; /* in [...] +CREATE DROP TABLE IF EXISTS vacuum_progress_ao_row; DROP CREATE TABLE vacuum_progress_ao_row(i int, j int); @@ -392,6 +395,13 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he -------------------------- Success: (1 row) +-- Trigger FTS probe repeatedly until mirror is marked as down, so that +-- FTS version change propagates to QD and gang gets reset. +2: SELECT wait_for_mirror_down(1); + wait_for_mirror_down +---------------------- + t +(1 row) -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -596,6 +606,13 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he -------------------------- Success: (1 row) +-- Trigger FTS probe repeatedly until mirror is marked as down, so that +-- FTS version change propagates to QD and gang gets reset. +2: SELECT wait_for_mirror_down(1); + wait_for_mirror_down +---------------------- + t +(1 row) -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point diff --git a/src/test/isolation2/sql/vacuum_progress_column.sql b/src/test/isolation2/sql/vacuum_progress_column.sql index 60250368b46..519c3e37200 100644 --- a/src/test/isolation2/sql/vacuum_progress_column.sql +++ b/src/test/isolation2/sql/vacuum_progress_column.sql @@ -141,6 +141,27 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he 2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- resume walsender and let it exit so that mirror stop can be detected 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +-- Trigger FTS probe repeatedly until mirror is marked as down, so that +-- FTS version change propagates to QD and gang gets reset. +CREATE OR REPLACE FUNCTION wait_for_mirror_down(content_id int) RETURNS bool AS $$ +declare /* in func */ + retries int; /* in func */ +begin /* in func */ + retries := 60; /* in func */ + loop /* in func */ + perform gp_request_fts_probe_scan(); /* in func */ + if (select status = 'd' from gp_segment_configuration where content = content_id and role = 'm') then /* in func */ + return true; /* in func */ + end if; /* in func */ + if retries <= 0 then /* in func */ + return false; /* in func */ + end if; /* in func */ + perform pg_sleep(1); /* in func */ + retries := retries - 1; /* in func */ + end loop; /* in func */ +end; /* in func */ +$$ language plpgsql; +2: SELECT wait_for_mirror_down(1); -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point diff --git a/src/test/isolation2/sql/vacuum_progress_row.sql b/src/test/isolation2/sql/vacuum_progress_row.sql index c6a01b298c8..125368df72b 100644 --- a/src/test/isolation2/sql/vacuum_progress_row.sql +++ b/src/test/isolation2/sql/vacuum_progress_row.sql @@ -106,6 +106,25 @@ SELECT n_live_tup, n_dead_tup, last_vacuum is not null as has_last_vacuum, vacuu -- Current behavior is it will clear previous compact phase num_dead_tuples in post-cleanup -- phase (at injecting point vacuum_ao_post_cleanup_end), which is different from above case -- in which vacuum worker isn't changed. + +CREATE OR REPLACE FUNCTION wait_for_mirror_down(content_id int) RETURNS bool AS $$ +declare /* in func */ + retries int; /* in func */ +begin /* in func */ + retries := 60; /* in func */ + loop /* in func */ + perform gp_request_fts_probe_scan(); /* in func */ + if (select status = 'd' from gp_segment_configuration where content = content_id and role = 'm') then /* in func */ + return true; /* in func */ + end if; /* in func */ + if retries <= 0 then /* in func */ + return false; /* in func */ + end if; /* in func */ + perform pg_sleep(1); /* in func */ + retries := retries - 1; /* in func */ + end loop; /* in func */ +end; /* in func */ +$$ language plpgsql; DROP TABLE IF EXISTS vacuum_progress_ao_row; CREATE TABLE vacuum_progress_ao_row(i int, j int); CREATE INDEX on vacuum_progress_ao_row(i); @@ -145,6 +164,9 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he 2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- resume walsender and let it exit so that mirror stop can be detected 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +-- Trigger FTS probe repeatedly until mirror is marked as down, so that +-- FTS version change propagates to QD and gang gets reset. +2: SELECT wait_for_mirror_down(1); -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point @@ -219,6 +241,9 @@ select relid::regclass as relname, phase, heap_blks_total, heap_blks_scanned, he 2: SELECT gp_inject_fault('vacuum_worker_changed', 'suspend', dbid) FROM gp_segment_configuration WHERE content > -1 AND role = 'p'; -- resume walsender and let it exit so that mirror stop can be detected 2: SELECT gp_inject_fault_infinite('wal_sender_loop', 'reset', dbid) FROM gp_segment_configuration WHERE role = 'p' and content = 1; +-- Trigger FTS probe repeatedly until mirror is marked as down, so that +-- FTS version change propagates to QD and gang gets reset. +2: SELECT wait_for_mirror_down(1); -- Ensure we enter into the target logic which stops cumulative data but -- initializes a new vacrelstats at the beginning of post-cleanup phase. -- Also all segments should reach to the same "vacuum_worker_changed" point --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
