This is an automated email from the ASF dual-hosted git repository. dongjoon pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new 24a9d25358f7 [SPARK-47027][PYTHON][TESTS] Use temporary directories for profiler test outputs 24a9d25358f7 is described below commit 24a9d25358f71e5634240aa29c600588b838edb2 Author: Takuya UESHIN <ues...@databricks.com> AuthorDate: Mon Feb 12 13:35:45 2024 -0800 [SPARK-47027][PYTHON][TESTS] Use temporary directories for profiler test outputs ### What changes were proposed in this pull request? Use temporary directories for profiler test outputs instead of `tempfile.gettempdir()`. ### Why are the changes needed? Directly using `tempfile.gettempdir()` can leave the files there after each test. ### Does this PR introduce _any_ user-facing change? No. ### How was this patch tested? Existing tests. ### Was this patch authored or co-authored using generative AI tooling? No. Closes #45087 from ueshin/issues/SPARK-47027/tempdir. Authored-by: Takuya UESHIN <ues...@databricks.com> Signed-off-by: Dongjoon Hyun <dh...@apple.com> --- python/pyspark/sql/tests/test_udf_profiler.py | 28 +++++++++++++-------------- python/pyspark/tests/test_memory_profiler.py | 6 +++--- python/pyspark/tests/test_profiler.py | 6 +++--- 3 files changed, 20 insertions(+), 20 deletions(-) diff --git a/python/pyspark/sql/tests/test_udf_profiler.py b/python/pyspark/sql/tests/test_udf_profiler.py index 4f767d274414..764a860026f8 100644 --- a/python/pyspark/sql/tests/test_udf_profiler.py +++ b/python/pyspark/sql/tests/test_udf_profiler.py @@ -82,20 +82,20 @@ class UDFProfilerTests(unittest.TestCase): finally: sys.stdout = old_stdout - d = tempfile.gettempdir() - self.sc.dump_profiles(d) - - for i, udf_name in enumerate(["add1", "add2", "add1", "add2"]): - id, profiler, _ = profilers[i] - with self.subTest(id=id, udf_name=udf_name): - stats = profiler.stats() - self.assertTrue(stats is not None) - width, stat_list = stats.get_print_list([]) - func_names = [func_name for fname, n, func_name in stat_list] - self.assertTrue(udf_name in func_names) - - self.assertTrue(udf_name in io.getvalue()) - self.assertTrue("udf_%d.pstats" % id in os.listdir(d)) + with tempfile.TemporaryDirectory() as d: + self.sc.dump_profiles(d) + + for i, udf_name in enumerate(["add1", "add2", "add1", "add2"]): + id, profiler, _ = profilers[i] + with self.subTest(id=id, udf_name=udf_name): + stats = profiler.stats() + self.assertTrue(stats is not None) + width, stat_list = stats.get_print_list([]) + func_names = [func_name for fname, n, func_name in stat_list] + self.assertTrue(udf_name in func_names) + + self.assertTrue(udf_name in io.getvalue()) + self.assertTrue("udf_%d.pstats" % id in os.listdir(d)) def test_custom_udf_profiler(self): class TestCustomProfiler(UDFBasicProfiler): diff --git a/python/pyspark/tests/test_memory_profiler.py b/python/pyspark/tests/test_memory_profiler.py index 536f38679c3e..aa3541620446 100644 --- a/python/pyspark/tests/test_memory_profiler.py +++ b/python/pyspark/tests/test_memory_profiler.py @@ -106,9 +106,9 @@ class MemoryProfilerTests(PySparkTestCase): self.sc.show_profiles() self.assertTrue("plus_one" in fake_out.getvalue()) - d = tempfile.gettempdir() - self.sc.dump_profiles(d) - self.assertTrue("udf_%d_memory.txt" % id in os.listdir(d)) + with tempfile.TemporaryDirectory() as d: + self.sc.dump_profiles(d) + self.assertTrue("udf_%d_memory.txt" % id in os.listdir(d)) def test_profile_pandas_udf(self): udfs = [self.exec_pandas_udf_ser_to_ser, self.exec_pandas_udf_ser_to_scalar] diff --git a/python/pyspark/tests/test_profiler.py b/python/pyspark/tests/test_profiler.py index b7797ead2adb..a12bc99c54ae 100644 --- a/python/pyspark/tests/test_profiler.py +++ b/python/pyspark/tests/test_profiler.py @@ -54,9 +54,9 @@ class ProfilerTests(PySparkTestCase): self.assertTrue("heavy_foo" in io.getvalue()) sys.stdout = old_stdout - d = tempfile.gettempdir() - self.sc.dump_profiles(d) - self.assertTrue("rdd_%d.pstats" % id in os.listdir(d)) + with tempfile.TemporaryDirectory() as d: + self.sc.dump_profiles(d) + self.assertTrue("rdd_%d.pstats" % id in os.listdir(d)) def test_custom_profiler(self): class TestCustomProfiler(BasicProfiler): --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org