spark-test commented on code in PR #60189: URL: https://github.com/apache/spark/pull/60189#discussion_r4500401147
########## dev/sparktestsupport/tests/test_utils.py: ########## @@ -0,0 +1,152 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import os +import unittest +from unittest import mock + +from sparktestsupport import modules +from sparktestsupport.utils import ( + determine_dangling_python_tests, + determine_modules_for_files, + determine_modules_to_test, + identify_changed_files_from_git_commits, +) + + +class DetermineModulesForFilesTests(unittest.TestCase): + def test_github_files_only_count_in_github_actions(self): + for environ, expected in (({}, set()), ({"GITHUB_ACTIONS": "true"}, {modules.root})): + with self.subTest(environ=environ), mock.patch.dict(os.environ, environ, clear=True): + self.assertEqual( + determine_modules_for_files([".github/workflows/build_main.yml"]), expected + ) + + +class IdentifyChangedFilesFromGitHistoryTests(unittest.TestCase): + def changed_modules(self, patch_sha, target_ref): + changed_files = identify_changed_files_from_git_commits(patch_sha, target_ref=target_ref) + return determine_modules_for_files(changed_files) + + def test_graphx_commit_selects_graphx_module(self): + self.assertEqual(self.changed_modules("fc0a1475ef", "5da21f07"), {modules.graphx}) + + def test_unmatched_commit_selects_root_module(self): + self.assertIn(modules.root, self.changed_modules("50a0496a43", "6765ef9")) + + +class DetermineModulesToTestTests(unittest.TestCase): + def test_root_short_circuits_dependents(self): + for module in (modules.root, modules.build, modules.core, modules.launcher): + with self.subTest(module=module.name): + self.assertEqual(determine_modules_to_test([module]), [modules.root]) + + def test_sql_dependents(self): + expected = { + "avro", + "connect", + "docker-integration-tests", + "examples", + "hive", + "hive-thriftserver", + "mllib", + "pipelines", + "protobuf", + "pyspark-connect", + "pyspark-ml", + "pyspark-ml-connect", + "pyspark-mllib", + "pyspark-pandas", + "pyspark-pandas-connect", + "pyspark-pandas-slow", + "pyspark-pandas-slow-connect", + "pyspark-pipelines", + "pyspark-sql", + "pyspark-structured-streaming", + "pyspark-structured-streaming-connect", + "pyspark-testing", + "repl", + "sparkr", + "sql", + "sql-kafka-0-10", + } + for changed, deduplicated in ( + ([modules.sql], True), + ([modules.sparkr, modules.sql], False), + ): + with self.subTest(changed=changed, deduplicated=deduplicated): + result = determine_modules_to_test(changed, deduplicated=deduplicated) + self.assertCountEqual([module.name for module in result], expected) + + def test_core_and_sql_without_deduplication(self): + expected = { + "avro", + "catalyst", + "connect", + "core", + "credential-aws", + "docker-integration-tests", + "examples", + "graphx", + "hive", + "hive-thriftserver", + "mllib", + "mllib-local", + "pipelines", + "protobuf", + "pyspark-connect", + "pyspark-core", + "pyspark-errors", + "pyspark-ml", + "pyspark-ml-connect", + "pyspark-mllib", + "pyspark-pandas", + "pyspark-pandas-connect", + "pyspark-pandas-slow", + "pyspark-pandas-slow-connect", + "pyspark-pipelines", + "pyspark-resource", + "pyspark-sql", + "pyspark-streaming", + "pyspark-structured-streaming", + "pyspark-structured-streaming-connect", + "pyspark-testing", + "repl", + "root", + "sparkr", + "sql", + "sql-kafka-0-10", + "streaming", + "streaming-kafka-0-10", + "streaming-kinesis-asl", + } + result = determine_modules_to_test([modules.sql, modules.core], deduplicated=False) + self.assertCountEqual([module.name for module in result], expected) + + +class DetermineDanglingPythonTestsTests(unittest.TestCase): + def setUp(self): + # Pretend every changed file exists so results do not depend on the working directory. + self.enterContext(mock.patch("os.path.exists", return_value=True)) Review Comment: **Non-blocking (P2):** `TestCase.enterContext` was added in Python 3.11, but the `dev/run-tests` wrapper still accepts any `python3` >= 3.10, and `dev/run-tests.py` now runs this suite before `main()` on every invocation. On Python 3.10 both tests in this class error in `setUp` (`AttributeError: ... has no attribute 'enterContext'`), so every `./dev/run-tests` run exits with status 255 before doing any work. That includes Scala-only runs, which get past the startup check on the base branch, and the error looks like a broken self-test rather than an unsupported Python version. Python 3.11+ and the CI jobs are unaffected. Deleting `unittest.TestCase.enterContext` on Python 3.12 to emulate 3.10 reproduces it: the startup discovery runs 23 tests with 2 errors. The smallest fix keeps the fixture 3.10-compatible with the same per-test cleanup: ```suggestion patcher = mock.patch("os.path.exists", return_value=True) patcher.start() self.addCleanup(patcher.stop) ``` Alternatively, since SPARK-56889 already dropped Python 3.10 from `python/run-tests` and packaging, the `dev/run-tests` check could move to 3.11 so that 3.10 gets a clear version message. **Verification:** - **Regression:** Discover the sparktestsupport suite the way dev/run-tests discovers it at startup, either on the oldest Python version the dev/run-tests wrapper accepts (3.10) or on a newer interpreter with unittest.TestCase.enterContext made unavailable. It runs with no errors or failures, so a dev/run-tests invocation would continue past its self-test. - **Behavior:** The dangling-test cases still report an unregistered test file under python/ as dangling and a test file outside python/ as not dangling, and the path-existence patch is active only while each of those tests runs. - **Compatibility:** On Python 3.11 and later, the full sparktestsupport suite still collects the same tests and passes, both through the CI precondition command and through the dev/run-tests startup discovery. -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected] --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
