dtenedor commented on code in PR #57691:
URL: https://github.com/apache/spark/pull/57691#discussion_r3716266933
##########
python/pyspark/sql/tests/df_golden/df_golden.py:
##########
@@ -744,3 +752,230 @@ def _compare_case(test_case, case, actual):
got.strip("\n"),
"[{}] mismatch in `{}`".format(name, key),
)
+
+
+def is_generating_golden():
+ """Whether this run regenerates the golden files instead of checking
them."""
+ return os.environ.get("SPARK_GENERATE_GOLDEN_FILES") is not None
+
+
+def check_cases_in_sync(test_file, declared, golden):
+ """
+ Check that the golden file describes exactly the declared cases, in order.
+
+ Each test asserts against its own block, so a case the golden file has
never
+ heard of (or one it still remembers after the method was deleted or
renamed)
+ would otherwise go unnoticed.
+ """
+ if declared == golden:
+ return
+ missing = [name for name in declared if name not in golden]
+ extra = [name for name in golden if name not in declared]
+ if missing:
+ detail = "cases with no block in the golden file: " + ",
".join(missing)
+ elif extra:
+ detail = "blocks in the golden file with no case method: " + ",
".join(extra)
+ else:
+ detail = "the golden file lists the cases in a different order"
+ raise AssertionError(
+ "{}: golden file is out of sync with the test class ({}); "
+ "regenerate the golden files".format(test_file, detail)
+ )
+
+
+# ---------------------------------------------------------------------------
+# Test class integration
+# ---------------------------------------------------------------------------
+
+
+class DFGoldenTestMixin:
+ """
+ Mixin turning a class of case methods into DataFrame golden file tests.
+
+ Mix into a session-providing test case, listing this class first so its
+ ``setUpClass`` runs once the session exists::
+
+ class GroupByGoldenTests(DFGoldenTestMixin, ReusedConnectTestCase):
+ golden_file = "group_by.test"
+
+ def _test_group_by_count(self, spark):
+ return
spark.table("testData").groupBy(col("a")).agg(count(col("b")))
+
+ Every ``_test_<case>`` method declares one case and returns the DataFrame
+ under test; a ``test_<case>`` method is registered for each, so cases run,
+ report and can be selected individually like any other unittest test.
+
+ The cases of a class share one Spark Connect session (``newSession()`` off
+ the session the test class provides, the Connect counterpart of
+ ``SQLQueryTestSuite``'s per-file ``newSession()``), prepared once by
+ :meth:`setup_session`. State created there -- temp views, UDFs, session
+ confs -- is discarded with the session and cannot leak into other classes.
+ """
+
+ #: Name of the ``.test`` golden file, resolved next to the test module.
+ golden_file = None
+
+ def __init_subclass__(cls, **kwargs):
+ super().__init_subclass__(**kwargs)
+ # This mixin's setUpClass must run after the session-providing class
has
+ # created the session, i.e. its setUpClass must be the outer one, which
+ # is only true when the mixin is listed first.
+ for base in cls.__mro__:
+ if base is DFGoldenTestMixin:
+ break
+ if base is not cls and "setUpClass" in vars(base):
Review Comment:
This is done.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
