This is an automated email from the ASF dual-hosted git repository. michaelsmith pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/impala.git
commit f96ccbe962cb97994ec71ebd08a6d90bda836a35 Author: Michael Smith <[email protected]> AuthorDate: Mon Oct 7 13:47:23 2024 -0700 IMPALA-11761: Run large dir tests serially IMPALA-12442 removed duplicate labels for stress and execute_serially, as they resulted in running the tests twice in different suites. Most tests that had both labels expect to be run sequentially, which by definition cannot be run in our stress test suite (which runs many operations at once to stress the cluster). Updates all tests previously marked with both 'stress' and 'execute_serially' to run serially. The only test continuing to use 'stress' mode is test_ddl_stress.py which was designed for it using a separate 'test_index' parameter. Change-Id: I1f7d2017ae1bab0f2f8cb0b100c2c6cc8b4f3dcd Reviewed-on: http://gerrit.cloudera.org:8080/21905 Reviewed-by: Impala Public Jenkins <[email protected]> Tested-by: Impala Public Jenkins <[email protected]> --- tests/metadata/test_recursive_listing.py | 4 ++-- tests/pytest.ini | 2 +- tests/run-tests.py | 2 +- tests/stress/test_acid_stress.py | 10 +++++----- tests/stress/test_insert_stress.py | 4 ++-- tests/stress/test_merge_stress.py | 2 +- tests/stress/test_update_stress.py | 6 +++--- 7 files changed, 15 insertions(+), 15 deletions(-) diff --git a/tests/metadata/test_recursive_listing.py b/tests/metadata/test_recursive_listing.py index 78674b6ad..9ec8bc135 100644 --- a/tests/metadata/test_recursive_listing.py +++ b/tests/metadata/test_recursive_listing.py @@ -145,7 +145,7 @@ class TestRecursiveListing(ImpalaTestSuite): assert len(self._get_rows(fq_tbl_name)) == 1 @SkipIfFS.no_partial_listing - @pytest.mark.stress + @pytest.mark.execute_serially def test_large_staging_dirs(self, unique_database): """Regression test for IMPALA-11464: Test REFRESH survives with concurrent add/remove ops on large staging/tmp dirs @@ -168,7 +168,7 @@ class TestRecursiveListing(ImpalaTestSuite): refresh_should_fail=False) @SkipIfFS.no_partial_listing - @pytest.mark.stress + @pytest.mark.execute_serially def test_partition_dir_removed_inflight(self, unique_database): """Test REFRESH with concurrent add/remove ops on large partition dirs which contain more than 1000 files. Execute this serially since the sleep diff --git a/tests/pytest.ini b/tests/pytest.ini index fea3399fc..2b77ee8fa 100644 --- a/tests/pytest.ini +++ b/tests/pytest.ini @@ -2,4 +2,4 @@ addopts = -r xfE -v --tb=short --showlocals markers = execute_serially: tests that must not be executed in parallel - stress: stress tests, will be run independent of other tests + stress: stress tests running multiple clients with the test_index parameter, will be run independent of other tests diff --git a/tests/run-tests.py b/tests/run-tests.py index dce1eec6b..99088ded2 100755 --- a/tests/run-tests.py +++ b/tests/run-tests.py @@ -64,7 +64,7 @@ NUM_CONCURRENT_TESTS = multiprocessing.cpu_count() if 'NUM_CONCURRENT_TESTS' in os.environ: NUM_CONCURRENT_TESTS = int(os.environ['NUM_CONCURRENT_TESTS']) -# Default the number of stress clinets to 4x the number of CPUs (but not exceeding the +# Default the number of stress clients to 4x the number of CPUs (but not exceeding the # default max # of concurrent connections) # This can be overridden by setting the NUM_STRESS_CLIENTS environment variable. # TODO: fix the stress test so it can start more clients than available connections diff --git a/tests/stress/test_acid_stress.py b/tests/stress/test_acid_stress.py index c3f8dc295..5809cae06 100644 --- a/tests/stress/test_acid_stress.py +++ b/tests/stress/test_acid_stress.py @@ -164,7 +164,7 @@ class TestAcidInsertsBasic(TestAcidStress): @SkipIfHive2.acid @SkipIfFS.hive - @pytest.mark.stress + @pytest.mark.execute_serially def test_read_hive_inserts(self, unique_database): """Check that Impala can read partitioned and non-partitioned ACID tables written by Hive.""" @@ -172,7 +172,7 @@ class TestAcidInsertsBasic(TestAcidStress): self._run_test_read_hive_inserts(unique_database, is_partitioned) @SkipIfHive2.acid - @pytest.mark.stress + @pytest.mark.execute_serially def test_read_impala_inserts(self, unique_database): """Check that Impala can read partitioned and non-partitioned ACID tables written by Hive.""" @@ -190,7 +190,7 @@ class TestAcidInsertsBasic(TestAcidStress): finally: impalad_client.close() - @pytest.mark.stress + @pytest.mark.execute_serially @SkipIf.not_dfs @UniqueDatabase.parametrize(sync_ddl=True) def test_partitioned_inserts(self, unique_database): @@ -281,7 +281,7 @@ class TestConcurrentAcidInserts(TestAcidStress): @SkipIfFS.stress_insert_timeouts @SkipIfHive2.acid @SkipIfDockerizedCluster.jira(reason="IMPALA-11189") - @pytest.mark.stress + @pytest.mark.execute_serially @UniqueDatabase.parametrize(sync_ddl=True) def test_concurrent_inserts(self, unique_database): """Issues INSERT statements against multiple impalads in a way that some @@ -379,7 +379,7 @@ class TestFailingAcidInserts(TestAcidStress): @SkipIfFS.stress_insert_timeouts @SkipIfDockerizedCluster.jira(reason="IMPALA-11191") @SkipIfHive2.acid - @pytest.mark.stress + @pytest.mark.execute_serially @UniqueDatabase.parametrize(sync_ddl=True) def test_failing_inserts(self, unique_database): """Tests that failing INSERTs cannot be observed.""" diff --git a/tests/stress/test_insert_stress.py b/tests/stress/test_insert_stress.py index 85e7c8a38..5653cc9eb 100644 --- a/tests/stress/test_insert_stress.py +++ b/tests/stress/test_insert_stress.py @@ -82,7 +82,7 @@ class TestInsertStress(ImpalaTestSuite): finally: impalad_client.close() - @pytest.mark.stress + @pytest.mark.execute_serially @UniqueDatabase.parametrize(sync_ddl=True) def test_inserts(self, unique_database): """Issues INSERT statements against multiple impalads in a way that some @@ -104,7 +104,7 @@ class TestInsertStress(ImpalaTestSuite): for i in range(0, num_checkers)] run_tasks(writers + checkers) - @pytest.mark.stress + @pytest.mark.execute_serially @SkipIf.not_dfs @UniqueDatabase.parametrize(sync_ddl=True) def test_iceberg_inserts(self, unique_database): diff --git a/tests/stress/test_merge_stress.py b/tests/stress/test_merge_stress.py index b3e58e816..e948aa650 100644 --- a/tests/stress/test_merge_stress.py +++ b/tests/stress/test_merge_stress.py @@ -101,7 +101,7 @@ class TestIcebergConcurrentMergeStress(ImpalaTestSuite): time.sleep(random.random()) impalad_client.close() - @pytest.mark.stress + @pytest.mark.execute_serially @UniqueDatabase.parametrize(sync_ddl=True) def test_iceberg_merge_updates(self, unique_database): """Issues MERGE statements with UPDATE and INSERT clause against multiple diff --git a/tests/stress/test_update_stress.py b/tests/stress/test_update_stress.py index 704fbb5a3..1148f7ce4 100644 --- a/tests/stress/test_update_stress.py +++ b/tests/stress/test_update_stress.py @@ -119,7 +119,7 @@ class TestIcebergConcurrentUpdateStress(ImpalaTestSuite): time.sleep(random.random()) impalad_client.close() - @pytest.mark.stress + @pytest.mark.execute_serially @UniqueDatabase.parametrize(sync_ddl=True) def test_iceberg_updates(self, unique_database): """Issues UPDATE statements against multiple impalads in a way that some @@ -232,7 +232,7 @@ class TestIcebergConcurrentOperations(ImpalaTestSuite): time.sleep(random.random()) impalad_client.close() - @pytest.mark.stress + @pytest.mark.execute_serially @UniqueDatabase.parametrize(sync_ddl=True) def test_iceberg_deletes_and_updates(self, unique_database): """Issues DELETE and UPDATE statements in parallel in a way that some @@ -259,7 +259,7 @@ class TestIcebergConcurrentOperations(ImpalaTestSuite): result = self.client.execute("select count(*) from {}".format(tbl_name)) assert result.data == ['0'] - @pytest.mark.stress + @pytest.mark.execute_serially @UniqueDatabase.parametrize(sync_ddl=True) def test_iceberg_deletes_and_updates_and_optimize(self, unique_database): """Issues DELETE and UPDATE statements in parallel in a way that some
