Github user tdas commented on a diff in the pull request:
https://github.com/apache/spark/pull/16125#discussion_r90748529
--- Diff: python/pyspark/sql/streaming.py ---
@@ -132,6 +133,61 @@ def stop(self):
"""
self._jsq.stop()
+ @since(2.1)
+ def explain(self, extended=False):
+ """Prints the (logical and physical) plans to the console for
debugging purpose.
+
+ :param extended: boolean, default ``False``. If ``False``, prints
only the physical plan.
+
+ >>> sq =
sdf.writeStream.format('memory').queryName('query_explain').start()
+ >>> sq.processAllAvailable() # Wait a bit to generate the runtime
plans.
+ >>> sq.explain()
+ == Physical Plan ==
+ ...
+ >>> sq.explain(True)
+ == Parsed Logical Plan ==
+ ...
+ == Analyzed Logical Plan ==
+ ...
+ == Optimized Logical Plan ==
+ ...
+ == Physical Plan ==
+ ...
+ >>> sq.stop()
+ """
+ # Cannot call `_jsq.explain(...)` because it will print in the JVM
process.
+ # We should print it in the Python process.
+ print(self._jsq.explainInternal(extended))
+
+ @since(2.1)
+ def exception(self):
+ """
+ :return: the StreamingQueryException if the query was terminated
by an exception, or None.
+
+ >>> sq =
sdf.writeStream.format('memory').queryName('query_exception').start()
+ >>> str(sq.exception())
+ 'None'
+ >>> sq.stop()
+ >>> from pyspark.sql.functions import col, udf
+ >>> bad_udf = udf(lambda x: x / 0)
+ >>> sq =
sdf.select(bad_udf(col("value"))).writeStream.format('memory')\
+ .queryName('this_query').start()
+ >>> try:
+ ... sq.processAllAvailable() # Process some data to fail the
query
+ ... except:
+ ... pass # Ignore the error as we want to test the "exception"
method
+ >>> sq.exception()
+ StreamingQueryException()
+ >>> sq.stop()
--- End diff --
I am not sure if such a big test should be present as doc tests. please
add this to the sql/tests.py.
---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at [email protected] or file a JIRA ticket
with INFRA.
---
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
