[tvm] branch main updated: [Pytest] Sort unit tests before running. (#9188)

[areusch]

This is an automated email from the ASF dual-hosted git repository.

areusch pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git


The following commit(s) were added to refs/heads/main by this push:
     new 523eb12  [Pytest] Sort unit tests before running. (#9188)
523eb12 is described below

commit 523eb12a1a9ce92777afbe90e0b87f47712cad6a
Author: Lunderberg <lunderb...@users.noreply.github.com>
AuthorDate: Wed Oct 13 23:26:18 2021 -0500

    [Pytest] Sort unit tests before running. (#9188)
    
    * [Pytest] Sort unit tests before running.
    
    By default, pytest will sort tests to maximize the re-use of fixtures.
    However, this assumes that all fixtures have an equal cost to
    generate, and no caches outside of those managed by pytest.  A fixture
    for a `tvm.testing.parameter` is effectively free, while a fixture
    maintaining a cache of reference data
    `tvm.testing.utils._fixture_cache` be quite large.
    
    Since most of the TVM fixtures are specific to a python function, sort
    the test ordering by python function, so that
    tvm.testing.utils._fixture_cache can be cleared sooner rather than
    later.
    
    * Updated TestTargetAutoParametrization
    
    When sorting the tests, the order of parametrizations may change.
    Therefore, the tests checking for automatic target parametrization
    shouldn't depend on order.
---
 python/tvm/testing/plugin.py                       | 20 ++++++++++++++++++++
 tests/python/unittest/test_tvm_testing_features.py | 15 +++++++++------
 2 files changed, 29 insertions(+), 6 deletions(-)

diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py
index 0413c44..2cb228c 100644
--- a/python/tvm/testing/plugin.py
+++ b/python/tvm/testing/plugin.py
@@ -74,6 +74,7 @@ def pytest_collection_modifyitems(config, items):
     # pylint: disable=unused-argument
     _count_num_fixture_uses(items)
     _remove_global_fixture_definitions(items)
+    _sort_tests(items)
 
 
 @pytest.fixture
@@ -236,6 +237,25 @@ def _remove_global_fixture_definitions(items):
                 delattr(module, name)
 
 
+def _sort_tests(items):
+    """Sort tests by file/function.
+
+    By default, pytest will sort tests to maximize the re-use of
+    fixtures.  However, this assumes that all fixtures have an equal
+    cost to generate, and no caches outside of those managed by
+    pytest.  A tvm.testing.parameter is effectively free, while
+    reference data for testing may be quite large.  Since most of the
+    TVM fixtures are specific to a python function, sort the test
+    ordering by python function, so that
+    tvm.testing.utils._fixture_cache can be cleared sooner rather than
+    later.
+
+    Should be called from pytest_collection_modifyitems.
+
+    """
+    items.sort(key=lambda item: item.location)
+
+
 def _target_to_requirement(target):
     if isinstance(target, str):
         target = tvm.target.Target(target)
diff --git a/tests/python/unittest/test_tvm_testing_features.py 
b/tests/python/unittest/test_tvm_testing_features.py
index cbcdc43..c00fc02 100644
--- a/tests/python/unittest/test_tvm_testing_features.py
+++ b/tests/python/unittest/test_tvm_testing_features.py
@@ -46,8 +46,11 @@ class TestTargetAutoParametrization:
         self.devices_used.append(dev)
 
     def test_all_targets_used(self):
-        assert self.targets_used == self.enabled_targets
-        assert self.devices_used == self.enabled_devices
+        assert sorted(self.targets_used) == sorted(self.enabled_targets)
+
+    def test_all_devices_used(self):
+        sort_key = lambda dev: (dev.device_type, dev.device_id)
+        assert sorted(self.devices_used, key=sort_key) == 
sorted(self.enabled_devices, key=sort_key)
 
     targets_with_explicit_list = []
 
@@ -70,9 +73,9 @@ class TestTargetAutoParametrization:
         self.targets_with_exclusion.append(target)
 
     def test_all_nonexcluded_targets_ran(self):
-        assert self.targets_with_exclusion == [
-            target for target in self.enabled_targets if not 
target.startswith("llvm")
-        ]
+        assert sorted(self.targets_with_exclusion) == sorted(
+            [target for target in self.enabled_targets if not 
target.startswith("llvm")]
+        )
 
     run_targets_with_known_failure = []
 
@@ -85,7 +88,7 @@ class TestTargetAutoParametrization:
         assert "llvm" not in target
 
     def test_all_targets_ran(self):
-        assert self.run_targets_with_known_failure == self.enabled_targets
+        assert sorted(self.run_targets_with_known_failure) == 
sorted(self.enabled_targets)
 
     @tvm.testing.known_failing_targets("llvm")
     @tvm.testing.parametrize_targets("llvm")