From ff3a8b82137723c025772051b65561d1bd469a54 Mon Sep 17 00:00:00 2001 From: nkey Date: Thu, 20 Feb 2025 14:50:10 +0300 Subject: [PATCH v9 1/4] Specs to reproduce the issues with CREATE INDEX CONCURRENTLY and REINDEX CONCURRENTLY in scenarios involving INSERT ON CONFLICT DO UPDATE. These tests reproduce different error cases related to "duplicate key value violates unique constraint" where this error should not occur by design. * REINDEX CONCURRENTLY and UPSERT with inferred index * CREATE INDEX CONCURRENTLY and UPSERT with inferred indexes * REINDEX CONCURRENTLY on a partitioned table * REINDEX CONCURRENTLY with specified constraint name * CREATE INDEX CONCURRENTLY with predicates In each of these scenarios, the expected behavior is that the INSERT ON CONFLICT DO UPDATE should handle conflicts gracefully without raising a "duplicate key value violates unique constraint" error. However, due to the concurrent operations on the indexes, this error is encountered. --- src/backend/commands/indexcmds.c | 4 +- src/backend/executor/execIndexing.c | 3 + src/backend/executor/nodeModifyTable.c | 2 + src/backend/utils/time/snapmgr.c | 2 + src/test/modules/injection_points/Makefile | 7 +- .../expected/index_concurrently_upsert.out | 84 ++++++ .../index_concurrently_upsert_predicate.out | 84 ++++++ .../expected/reindex_concurrently_upsert.out | 250 ++++++++++++++++++ ...ndex_concurrently_upsert_on_constraint.out | 250 ++++++++++++++++++ ...eindex_concurrently_upsert_partitioned.out | 250 ++++++++++++++++++ src/test/modules/injection_points/meson.build | 8 + .../specs/index_concurrently_upsert.spec | 72 +++++ .../index_concurrently_upsert_predicate.spec | 74 ++++++ .../specs/reindex_concurrently_upsert.spec | 90 +++++++ ...dex_concurrently_upsert_on_constraint.spec | 91 +++++++ ...index_concurrently_upsert_partitioned.spec | 92 +++++++ 16 files changed, 1361 insertions(+), 2 deletions(-) create mode 100644 src/test/modules/injection_points/expected/index_concurrently_upsert.out create mode 100644 src/test/modules/injection_points/expected/index_concurrently_upsert_predicate.out create mode 100644 src/test/modules/injection_points/expected/reindex_concurrently_upsert.out create mode 100644 src/test/modules/injection_points/expected/reindex_concurrently_upsert_on_constraint.out create mode 100644 src/test/modules/injection_points/expected/reindex_concurrently_upsert_partitioned.out create mode 100644 src/test/modules/injection_points/specs/index_concurrently_upsert.spec create mode 100644 src/test/modules/injection_points/specs/index_concurrently_upsert_predicate.spec create mode 100644 src/test/modules/injection_points/specs/reindex_concurrently_upsert.spec create mode 100644 src/test/modules/injection_points/specs/reindex_concurrently_upsert_on_constraint.spec create mode 100644 src/test/modules/injection_points/specs/reindex_concurrently_upsert_partitioned.spec diff --git a/src/backend/commands/indexcmds.c b/src/backend/commands/indexcmds.c index d962fe392cd..0f75debe7f1 100644 --- a/src/backend/commands/indexcmds.c +++ b/src/backend/commands/indexcmds.c @@ -1790,6 +1790,7 @@ DefineIndex(Oid tableId, * before the reference snap was taken, we have to wait out any * transactions that might have older snapshots. */ + INJECTION_POINT("define_index_before_set_valid", NULL); pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, PROGRESS_CREATEIDX_PHASE_WAIT_3); WaitForOlderSnapshots(limitXmin, true); @@ -4195,7 +4196,7 @@ ReindexRelationConcurrently(const ReindexStmt *stmt, Oid relationOid, const Rein * the same time to make sure we only get constraint violations from the * indexes with the correct names. */ - + INJECTION_POINT("reindex_relation_concurrently_before_swap", NULL); StartTransactionCommand(); /* @@ -4274,6 +4275,7 @@ ReindexRelationConcurrently(const ReindexStmt *stmt, Oid relationOid, const Rein * index_drop() for more details. */ + INJECTION_POINT("reindex_relation_concurrently_before_set_dead", NULL); pgstat_progress_update_param(PROGRESS_CREATEIDX_PHASE, PROGRESS_CREATEIDX_PHASE_WAIT_4); WaitForLockersMultiple(lockTags, AccessExclusiveLock, true); diff --git a/src/backend/executor/execIndexing.c b/src/backend/executor/execIndexing.c index bdf862b2406..499cba145dd 100644 --- a/src/backend/executor/execIndexing.c +++ b/src/backend/executor/execIndexing.c @@ -117,6 +117,7 @@ #include "utils/multirangetypes.h" #include "utils/rangetypes.h" #include "utils/snapmgr.h" +#include "utils/injection_point.h" /* waitMode argument to check_exclusion_or_unique_constraint() */ typedef enum @@ -942,6 +943,8 @@ retry: econtext->ecxt_scantuple = save_scantuple; ExecDropSingleTupleTableSlot(existing_slot); + if (!conflict) + INJECTION_POINT("check_exclusion_or_unique_constraint_no_conflict", NULL); return !conflict; } diff --git a/src/backend/executor/nodeModifyTable.c b/src/backend/executor/nodeModifyTable.c index 46d533b7288..566dbecb390 100644 --- a/src/backend/executor/nodeModifyTable.c +++ b/src/backend/executor/nodeModifyTable.c @@ -69,6 +69,7 @@ #include "utils/datum.h" #include "utils/rel.h" #include "utils/snapmgr.h" +#include "utils/injection_point.h" typedef struct MTTargetRelLookup @@ -1178,6 +1179,7 @@ ExecInsert(ModifyTableContext *context, return NULL; } } + INJECTION_POINT("exec_insert_before_insert_speculative", NULL); /* * Before we start insertion proper, acquire our "speculative diff --git a/src/backend/utils/time/snapmgr.c b/src/backend/utils/time/snapmgr.c index ea35f30f494..ad440ff024c 100644 --- a/src/backend/utils/time/snapmgr.c +++ b/src/backend/utils/time/snapmgr.c @@ -123,6 +123,7 @@ #include "utils/resowner.h" #include "utils/snapmgr.h" #include "utils/syscache.h" +#include "utils/injection_point.h" /* @@ -447,6 +448,7 @@ InvalidateCatalogSnapshot(void) pairingheap_remove(&RegisteredSnapshots, &CatalogSnapshot->ph_node); CatalogSnapshot = NULL; SnapshotResetXmin(); + INJECTION_POINT("invalidate_catalog_snapshot_end", NULL); } } diff --git a/src/test/modules/injection_points/Makefile b/src/test/modules/injection_points/Makefile index e680991f8d4..5aa53f03049 100644 --- a/src/test/modules/injection_points/Makefile +++ b/src/test/modules/injection_points/Makefile @@ -14,7 +14,12 @@ PGFILEDESC = "injection_points - facility for injection points" REGRESS = injection_points hashagg reindex_conc REGRESS_OPTS = --dlpath=$(top_builddir)/src/test/regress -ISOLATION = basic inplace syscache-update-pruned +ISOLATION = basic inplace syscache-update-pruned \ + reindex_concurrently_upsert \ + index_concurrently_upsert \ + reindex_concurrently_upsert_partitioned \ + reindex_concurrently_upsert_on_constraint \ + index_concurrently_upsert_predicate TAP_TESTS = 1 diff --git a/src/test/modules/injection_points/expected/index_concurrently_upsert.out b/src/test/modules/injection_points/expected/index_concurrently_upsert.out new file mode 100644 index 00000000000..e7612e065f4 --- /dev/null +++ b/src/test/modules/injection_points/expected/index_concurrently_upsert.out @@ -0,0 +1,84 @@ +Parsed test spec with 4 sessions + +starting permutation: s3_start_create_index s1_start_upsert s4_wakeup_define_index_before_set_valid s2_start_upsert s4_wakeup_s1_from_invalidate_catalog_snapshot s4_wakeup_s2 s4_wakeup_s1 +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_create_index: CREATE UNIQUE INDEX CONCURRENTLY tbl_pkey_duplicate ON test.tbl(i); +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_define_index_before_set_valid: + SELECT injection_points_detach('define_index_before_set_valid'); + SELECT injection_points_wakeup('define_index_before_set_valid'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_s1_from_invalidate_catalog_snapshot: + SELECT injection_points_detach('invalidate_catalog_snapshot_end'); + SELECT injection_points_wakeup('invalidate_catalog_snapshot_end'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s2_start_upsert: <... completed> +step s3_start_create_index: <... completed> diff --git a/src/test/modules/injection_points/expected/index_concurrently_upsert_predicate.out b/src/test/modules/injection_points/expected/index_concurrently_upsert_predicate.out new file mode 100644 index 00000000000..0ef2f3a681c --- /dev/null +++ b/src/test/modules/injection_points/expected/index_concurrently_upsert_predicate.out @@ -0,0 +1,84 @@ +Parsed test spec with 4 sessions + +starting permutation: s3_start_create_index s1_start_upsert s4_wakeup_define_index_before_set_valid s2_start_upsert s4_wakeup_s1_from_invalidate_catalog_snapshot s4_wakeup_s2 s4_wakeup_s1 +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_create_index: CREATE UNIQUE INDEX CONCURRENTLY tbl_pkey_special_duplicate ON test.tbl(abs(i)) WHERE i < 10000; +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(abs(i)) where i < 100 do update set updated_at = now(); + +step s4_wakeup_define_index_before_set_valid: + SELECT injection_points_detach('define_index_before_set_valid'); + SELECT injection_points_wakeup('define_index_before_set_valid'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(abs(i)) where i < 100 do update set updated_at = now(); + +step s4_wakeup_s1_from_invalidate_catalog_snapshot: + SELECT injection_points_detach('invalidate_catalog_snapshot_end'); + SELECT injection_points_wakeup('invalidate_catalog_snapshot_end'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s2_start_upsert: <... completed> +step s3_start_create_index: <... completed> diff --git a/src/test/modules/injection_points/expected/reindex_concurrently_upsert.out b/src/test/modules/injection_points/expected/reindex_concurrently_upsert.out new file mode 100644 index 00000000000..1bd8041289e --- /dev/null +++ b/src/test/modules/injection_points/expected/reindex_concurrently_upsert.out @@ -0,0 +1,250 @@ +Parsed test spec with 4 sessions + +starting permutation: s3_start_reindex s1_start_upsert s4_wakeup_to_swap s4_wakeup_to_set_dead s2_start_upsert s4_wakeup_s1 s4_wakeup_s2 +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_pkey; +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: <... completed> +step s3_start_reindex: <... completed> + +starting permutation: s3_start_reindex s1_start_upsert s4_wakeup_to_swap s2_start_upsert s4_wakeup_s2 s4_wakeup_s1 s4_wakeup_to_set_dead +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_pkey; +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s2_start_upsert: <... completed> +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_start_reindex: <... completed> + +starting permutation: s3_start_reindex s4_wakeup_to_swap s1_start_upsert s2_start_upsert s4_wakeup_s1 s4_wakeup_to_set_dead s4_wakeup_s2 +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_pkey; +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: <... completed> +step s3_start_reindex: <... completed> diff --git a/src/test/modules/injection_points/expected/reindex_concurrently_upsert_on_constraint.out b/src/test/modules/injection_points/expected/reindex_concurrently_upsert_on_constraint.out new file mode 100644 index 00000000000..68288812de2 --- /dev/null +++ b/src/test/modules/injection_points/expected/reindex_concurrently_upsert_on_constraint.out @@ -0,0 +1,250 @@ +Parsed test spec with 4 sessions + +starting permutation: s3_start_reindex s1_start_upsert s4_wakeup_to_swap s4_wakeup_to_set_dead s2_start_upsert s4_wakeup_s1 s4_wakeup_s2 +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_pkey; +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict on constraint tbl_pkey do update set updated_at = now(); + +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict on constraint tbl_pkey do update set updated_at = now(); + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: <... completed> +step s3_start_reindex: <... completed> + +starting permutation: s3_start_reindex s1_start_upsert s4_wakeup_to_swap s2_start_upsert s4_wakeup_s2 s4_wakeup_s1 s4_wakeup_to_set_dead +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_pkey; +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict on constraint tbl_pkey do update set updated_at = now(); + +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict on constraint tbl_pkey do update set updated_at = now(); + +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s2_start_upsert: <... completed> +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_start_reindex: <... completed> + +starting permutation: s3_start_reindex s4_wakeup_to_swap s1_start_upsert s2_start_upsert s4_wakeup_s1 s4_wakeup_to_set_dead s4_wakeup_s2 +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_pkey; +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict on constraint tbl_pkey do update set updated_at = now(); + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict on constraint tbl_pkey do update set updated_at = now(); + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: <... completed> +step s3_start_reindex: <... completed> diff --git a/src/test/modules/injection_points/expected/reindex_concurrently_upsert_partitioned.out b/src/test/modules/injection_points/expected/reindex_concurrently_upsert_partitioned.out new file mode 100644 index 00000000000..f8c81e2bea2 --- /dev/null +++ b/src/test/modules/injection_points/expected/reindex_concurrently_upsert_partitioned.out @@ -0,0 +1,250 @@ +Parsed test spec with 4 sessions + +starting permutation: s3_start_reindex s1_start_upsert s4_wakeup_to_swap s4_wakeup_to_set_dead s2_start_upsert s4_wakeup_s1 s4_wakeup_s2 +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_partition_pkey; +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: <... completed> +step s3_start_reindex: <... completed> + +starting permutation: s3_start_reindex s1_start_upsert s4_wakeup_to_swap s2_start_upsert s4_wakeup_s2 s4_wakeup_s1 s4_wakeup_to_set_dead +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_partition_pkey; +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s2_start_upsert: <... completed> +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s3_start_reindex: <... completed> + +starting permutation: s3_start_reindex s4_wakeup_to_swap s1_start_upsert s2_start_upsert s4_wakeup_s1 s4_wakeup_to_set_dead s4_wakeup_s2 +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +injection_points_attach +----------------------- + +(1 row) + +step s3_start_reindex: REINDEX INDEX CONCURRENTLY test.tbl_partition_pkey; +step s4_wakeup_to_swap: + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s2_start_upsert: + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); + +step s4_wakeup_s1: + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s1_start_upsert: <... completed> +step s4_wakeup_to_set_dead: + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s4_wakeup_s2: + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); + +injection_points_detach +----------------------- + +(1 row) + +injection_points_wakeup +----------------------- + +(1 row) + +step s2_start_upsert: <... completed> +step s3_start_reindex: <... completed> diff --git a/src/test/modules/injection_points/meson.build b/src/test/modules/injection_points/meson.build index d61149712fd..e1443664c70 100644 --- a/src/test/modules/injection_points/meson.build +++ b/src/test/modules/injection_points/meson.build @@ -47,8 +47,14 @@ tests += { 'basic', 'inplace', 'syscache-update-pruned', + 'reindex_concurrently_upsert', + 'index_concurrently_upsert', + 'reindex_concurrently_upsert_partitioned', + 'reindex_concurrently_upsert_on_constraint', + 'index_concurrently_upsert_predicate', ], 'runningcheck': false, # see syscache-update-pruned + 'runningcheck-parallel': false, # We waiting for all snapshots, so, avoid parallel test executions }, 'tap': { 'env': { @@ -57,5 +63,7 @@ tests += { 'tests': [ 't/001_stats.pl', ], + # The injection points are cluster-wide, so disable installcheck + 'runningcheck': false, }, } diff --git a/src/test/modules/injection_points/specs/index_concurrently_upsert.spec b/src/test/modules/injection_points/specs/index_concurrently_upsert.spec new file mode 100644 index 00000000000..473b0408f55 --- /dev/null +++ b/src/test/modules/injection_points/specs/index_concurrently_upsert.spec @@ -0,0 +1,72 @@ +# Test race conditions involving: +# - s1: UPSERT a tuple +# - s2: UPSERT the same tuple +# - s3: CREATE UNIQUE INDEX CONCURRENTLY +# - s4: operations with injection points + +setup +{ + CREATE EXTENSION injection_points; + CREATE SCHEMA test; + CREATE UNLOGGED TABLE test.tbl(i int primary key, updated_at timestamp); + ALTER TABLE test.tbl SET (parallel_workers=0); +} + +teardown +{ + DROP SCHEMA test CASCADE; + DROP EXTENSION injection_points; +} + +session s1 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('check_exclusion_or_unique_constraint_no_conflict', 'wait'); + SELECT injection_points_attach('invalidate_catalog_snapshot_end', 'wait'); +} +step s1_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); +} + +session s2 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('exec_insert_before_insert_speculative', 'wait'); +} +step s2_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); +} + +session s3 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('define_index_before_set_valid', 'wait'); +} +step s3_start_create_index { CREATE UNIQUE INDEX CONCURRENTLY tbl_pkey_duplicate ON test.tbl(i); } + +session s4 +step s4_wakeup_s1 { + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); +} +step s4_wakeup_s1_from_invalidate_catalog_snapshot { + SELECT injection_points_detach('invalidate_catalog_snapshot_end'); + SELECT injection_points_wakeup('invalidate_catalog_snapshot_end'); +} +step s4_wakeup_s2 { + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); +} +step s4_wakeup_define_index_before_set_valid { + SELECT injection_points_detach('define_index_before_set_valid'); + SELECT injection_points_wakeup('define_index_before_set_valid'); +} + +permutation + s3_start_create_index(s1_start_upsert, s2_start_upsert) + s1_start_upsert + s4_wakeup_define_index_before_set_valid + s2_start_upsert(s1_start_upsert) + s4_wakeup_s1_from_invalidate_catalog_snapshot + s4_wakeup_s2 + s4_wakeup_s1 \ No newline at end of file diff --git a/src/test/modules/injection_points/specs/index_concurrently_upsert_predicate.spec b/src/test/modules/injection_points/specs/index_concurrently_upsert_predicate.spec new file mode 100644 index 00000000000..c8644a82d57 --- /dev/null +++ b/src/test/modules/injection_points/specs/index_concurrently_upsert_predicate.spec @@ -0,0 +1,74 @@ +# Test race conditions involving: +# - s1: UPSERT a tuple +# - s2: UPSERT the same tuple +# - s3: CREATE UNIQUE INDEX CONCURRENTLY +# - s4: operations with injection points + +setup +{ + CREATE EXTENSION injection_points; + CREATE SCHEMA test; + CREATE UNLOGGED TABLE test.tbl(i int, updated_at timestamp); + + CREATE UNIQUE INDEX tbl_pkey_special ON test.tbl(abs(i)) WHERE i < 1000; + ALTER TABLE test.tbl SET (parallel_workers=0); +} + +teardown +{ + DROP SCHEMA test CASCADE; + DROP EXTENSION injection_points; +} + +session s1 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('check_exclusion_or_unique_constraint_no_conflict', 'wait'); + SELECT injection_points_attach('invalidate_catalog_snapshot_end', 'wait'); +} +step s1_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict(abs(i)) where i < 100 do update set updated_at = now(); +} + +session s2 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('exec_insert_before_insert_speculative', 'wait'); +} +step s2_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict(abs(i)) where i < 100 do update set updated_at = now(); +} + +session s3 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('define_index_before_set_valid', 'wait'); +} +step s3_start_create_index { CREATE UNIQUE INDEX CONCURRENTLY tbl_pkey_special_duplicate ON test.tbl(abs(i)) WHERE i < 10000;} + +session s4 +step s4_wakeup_s1 { + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); +} +step s4_wakeup_s1_from_invalidate_catalog_snapshot { + SELECT injection_points_detach('invalidate_catalog_snapshot_end'); + SELECT injection_points_wakeup('invalidate_catalog_snapshot_end'); +} +step s4_wakeup_s2 { + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); +} +step s4_wakeup_define_index_before_set_valid { + SELECT injection_points_detach('define_index_before_set_valid'); + SELECT injection_points_wakeup('define_index_before_set_valid'); +} + +permutation + s3_start_create_index(s1_start_upsert, s2_start_upsert) + s1_start_upsert + s4_wakeup_define_index_before_set_valid + s2_start_upsert(s1_start_upsert) + s4_wakeup_s1_from_invalidate_catalog_snapshot + s4_wakeup_s2 + s4_wakeup_s1 \ No newline at end of file diff --git a/src/test/modules/injection_points/specs/reindex_concurrently_upsert.spec b/src/test/modules/injection_points/specs/reindex_concurrently_upsert.spec new file mode 100644 index 00000000000..08d0deaf58b --- /dev/null +++ b/src/test/modules/injection_points/specs/reindex_concurrently_upsert.spec @@ -0,0 +1,90 @@ +# Test race conditions involving: +# - s1: UPSERT a tuple +# - s2: UPSERT the same tuple +# - s3: REINDEX concurrent primary key index +# - s4: operations with injection points + +setup +{ + CREATE EXTENSION injection_points; + CREATE SCHEMA test; + CREATE UNLOGGED TABLE test.tbl(i int primary key, updated_at timestamp); + ALTER TABLE test.tbl SET (parallel_workers=0); +} + +teardown +{ + DROP SCHEMA test CASCADE; + DROP EXTENSION injection_points; +} + +session s1 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('check_exclusion_or_unique_constraint_no_conflict', 'wait'); +} +step s1_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); +} + +session s2 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('exec_insert_before_insert_speculative', 'wait'); +} +step s2_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); +} + +session s3 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('reindex_relation_concurrently_before_set_dead', 'wait'); + SELECT injection_points_attach('reindex_relation_concurrently_before_swap', 'wait'); +} +step s3_start_reindex { REINDEX INDEX CONCURRENTLY test.tbl_pkey; } + +session s4 +step s4_wakeup_to_swap { + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); +} +step s4_wakeup_s1 { + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); +} +step s4_wakeup_s2 { + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); +} +step s4_wakeup_to_set_dead { + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); +} + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s1_start_upsert + s4_wakeup_to_swap + s4_wakeup_to_set_dead + s2_start_upsert(s1_start_upsert) + s4_wakeup_s1 + s4_wakeup_s2 + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s1_start_upsert + s4_wakeup_to_swap + s2_start_upsert(s1_start_upsert) + s4_wakeup_s2 + s4_wakeup_s1 + s4_wakeup_to_set_dead + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s4_wakeup_to_swap + s1_start_upsert + s2_start_upsert(s1_start_upsert) + s4_wakeup_s1 + s4_wakeup_to_set_dead + s4_wakeup_s2 diff --git a/src/test/modules/injection_points/specs/reindex_concurrently_upsert_on_constraint.spec b/src/test/modules/injection_points/specs/reindex_concurrently_upsert_on_constraint.spec new file mode 100644 index 00000000000..97622388e71 --- /dev/null +++ b/src/test/modules/injection_points/specs/reindex_concurrently_upsert_on_constraint.spec @@ -0,0 +1,91 @@ +# Test race conditions involving: +# - s1: UPSERT a tuple +# - s2: UPSERT the same tuple +# - s3: REINDEX concurrent primary key index +# - s4: operations with injection points + +setup +{ + CREATE EXTENSION injection_points; + CREATE SCHEMA test; + CREATE UNLOGGED TABLE test.tbl(i int primary key, updated_at timestamp); + ALTER TABLE test.tbl SET (parallel_workers=0); +} + +teardown +{ + DROP SCHEMA test CASCADE; + DROP EXTENSION injection_points; +} + +session s1 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('check_exclusion_or_unique_constraint_no_conflict', 'wait'); +} +step s1_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict on constraint tbl_pkey do update set updated_at = now(); +} + +session s2 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('exec_insert_before_insert_speculative', 'wait'); +} +step s2_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict on constraint tbl_pkey do update set updated_at = now(); +} + + +session s3 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('reindex_relation_concurrently_before_set_dead', 'wait'); + SELECT injection_points_attach('reindex_relation_concurrently_before_swap', 'wait'); +} +step s3_start_reindex { REINDEX INDEX CONCURRENTLY test.tbl_pkey; } + +session s4 +step s4_wakeup_to_swap { + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); +} +step s4_wakeup_s1 { + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); +} +step s4_wakeup_s2 { + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); +} +step s4_wakeup_to_set_dead { + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); +} + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s1_start_upsert + s4_wakeup_to_swap + s4_wakeup_to_set_dead + s2_start_upsert(s1_start_upsert) + s4_wakeup_s1 + s4_wakeup_s2 + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s1_start_upsert + s4_wakeup_to_swap + s2_start_upsert(s1_start_upsert) + s4_wakeup_s2 + s4_wakeup_s1 + s4_wakeup_to_set_dead + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s4_wakeup_to_swap + s1_start_upsert + s2_start_upsert(s1_start_upsert) + s4_wakeup_s1 + s4_wakeup_to_set_dead + s4_wakeup_s2 \ No newline at end of file diff --git a/src/test/modules/injection_points/specs/reindex_concurrently_upsert_partitioned.spec b/src/test/modules/injection_points/specs/reindex_concurrently_upsert_partitioned.spec new file mode 100644 index 00000000000..41f10c8736d --- /dev/null +++ b/src/test/modules/injection_points/specs/reindex_concurrently_upsert_partitioned.spec @@ -0,0 +1,92 @@ +# Test race conditions involving: +# - s1: UPSERT a tuple +# - s2: UPSERT the same tuple +# - s3: REINDEX concurrent primary key index +# - s4: operations with injection points + +setup +{ + CREATE EXTENSION injection_points; + CREATE SCHEMA test; + CREATE TABLE test.tbl(i int primary key, updated_at timestamp) PARTITION BY RANGE (i); + CREATE TABLE test.tbl_partition PARTITION OF test.tbl + FOR VALUES FROM (0) TO (10000) + WITH (parallel_workers = 0); +} + +teardown +{ + DROP SCHEMA test CASCADE; + DROP EXTENSION injection_points; +} + +session s1 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('check_exclusion_or_unique_constraint_no_conflict', 'wait'); +} +step s1_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); +} + +session s2 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('exec_insert_before_insert_speculative', 'wait'); +} +step s2_start_upsert { + INSERT INTO test.tbl VALUES(13,now()) on conflict(i) do update set updated_at = now(); +} + +session s3 +setup { + SELECT injection_points_set_local(); + SELECT injection_points_attach('reindex_relation_concurrently_before_set_dead', 'wait'); + SELECT injection_points_attach('reindex_relation_concurrently_before_swap', 'wait'); +} +step s3_start_reindex { REINDEX INDEX CONCURRENTLY test.tbl_partition_pkey; } + +session s4 +step s4_wakeup_to_swap { + SELECT injection_points_detach('reindex_relation_concurrently_before_swap'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_swap'); +} +step s4_wakeup_s1 { + SELECT injection_points_detach('check_exclusion_or_unique_constraint_no_conflict'); + SELECT injection_points_wakeup('check_exclusion_or_unique_constraint_no_conflict'); +} +step s4_wakeup_s2 { + SELECT injection_points_detach('exec_insert_before_insert_speculative'); + SELECT injection_points_wakeup('exec_insert_before_insert_speculative'); +} +step s4_wakeup_to_set_dead { + SELECT injection_points_detach('reindex_relation_concurrently_before_set_dead'); + SELECT injection_points_wakeup('reindex_relation_concurrently_before_set_dead'); +} + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s1_start_upsert + s4_wakeup_to_swap + s4_wakeup_to_set_dead + s2_start_upsert(s1_start_upsert) + s4_wakeup_s1 + s4_wakeup_s2 + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s1_start_upsert + s4_wakeup_to_swap + s2_start_upsert(s1_start_upsert) + s4_wakeup_s2 + s4_wakeup_s1 + s4_wakeup_to_set_dead + +permutation + s3_start_reindex(s1_start_upsert, s2_start_upsert) + s4_wakeup_to_swap + s1_start_upsert + s2_start_upsert(s1_start_upsert) + s4_wakeup_s1 + s4_wakeup_to_set_dead + s4_wakeup_s2 \ No newline at end of file -- 2.43.0