This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new 57dcc373a1b [SPARK-44502][DOC][SS][PYTHON] Add missing versionchanged
field to streaming functions
57dcc373a1b is described below
commit 57dcc373a1bc02d63db2dc87f99bee45569735f7
Author: Wei Liu <[email protected]>
AuthorDate: Fri Jul 21 12:09:59 2023 +0900
[SPARK-44502][DOC][SS][PYTHON] Add missing versionchanged field to
streaming functions
Add missing doc field
Better doc
No
Don't need
Closes #42097 from WweiL/SPARK-44502-doc.
Authored-by: Wei Liu <[email protected]>
Signed-off-by: Hyukjin Kwon <[email protected]>
(cherry picked from commit f1e310b84d5f1d2bde1bbafbb737a9cc612d573a)
Signed-off-by: Hyukjin Kwon <[email protected]>
---
python/pyspark/sql/dataframe.py | 3 ++
python/pyspark/sql/streaming/query.py | 51 +++++++++++++++++++++++
python/pyspark/sql/streaming/readwriter.py | 66 ++++++++++++++++++++++++++++++
3 files changed, 120 insertions(+)
diff --git a/python/pyspark/sql/dataframe.py b/python/pyspark/sql/dataframe.py
index 99eb2a48bb2..8e655dc3a88 100644
--- a/python/pyspark/sql/dataframe.py
+++ b/python/pyspark/sql/dataframe.py
@@ -1052,6 +1052,9 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
eventTime : str
diff --git a/python/pyspark/sql/streaming/query.py
b/python/pyspark/sql/streaming/query.py
index 6d18c93f019..443e7dbee39 100644
--- a/python/pyspark/sql/streaming/query.py
+++ b/python/pyspark/sql/streaming/query.py
@@ -58,6 +58,9 @@ class StreamingQuery:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
str
@@ -85,6 +88,9 @@ class StreamingQuery:
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
str
@@ -114,6 +120,9 @@ class StreamingQuery:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
str
@@ -140,6 +149,9 @@ class StreamingQuery:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
bool
@@ -171,6 +183,9 @@ class StreamingQuery:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
timeout : int, optional
@@ -212,6 +227,9 @@ class StreamingQuery:
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
dict
@@ -240,6 +258,9 @@ class StreamingQuery:
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
list
@@ -268,6 +289,9 @@ class StreamingQuery:
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
dict, optional
@@ -297,6 +321,9 @@ class StreamingQuery:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Notes
-----
In the case of continually arriving data, this method may block
forever.
@@ -325,6 +352,9 @@ class StreamingQuery:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Examples
--------
>>> sdf = spark.readStream.format("rate").load()
@@ -347,6 +377,9 @@ class StreamingQuery:
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
extended : bool, optional
@@ -387,6 +420,9 @@ class StreamingQuery:
"""
.. versionadded:: 2.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
:class:`StreamingQueryException`
@@ -406,6 +442,9 @@ class StreamingQueryManager:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Notes
-----
This API is evolving.
@@ -421,6 +460,9 @@ class StreamingQueryManager:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Returns
-------
list
@@ -451,6 +493,9 @@ class StreamingQueryManager:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
id : str
@@ -513,6 +558,9 @@ class StreamingQueryManager:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
timeout : int, optional
@@ -554,6 +602,9 @@ class StreamingQueryManager:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Examples
--------
>>> spark.streams.resetTerminated()
diff --git a/python/pyspark/sql/streaming/readwriter.py
b/python/pyspark/sql/streaming/readwriter.py
index 3547dd060f6..08d7396ba86 100644
--- a/python/pyspark/sql/streaming/readwriter.py
+++ b/python/pyspark/sql/streaming/readwriter.py
@@ -87,6 +87,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
source : str
@@ -128,6 +131,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
schema : :class:`pyspark.sql.types.StructType` or str
@@ -177,6 +183,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Notes
-----
This API is evolving.
@@ -203,6 +212,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Notes
-----
This API is evolving.
@@ -238,6 +250,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
path : str, optional
@@ -324,6 +339,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
path : str
@@ -407,6 +425,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.3.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Other Parameters
----------------
Extra options
@@ -458,6 +479,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
path : str
@@ -521,6 +545,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
path : str or list
@@ -619,6 +646,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Other Parameters
----------------
Extra options
@@ -695,6 +725,9 @@ class DataStreamReader(OptionUtils):
.. versionadded:: 3.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
tableName : str
@@ -745,6 +778,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Notes
-----
This API is evolving.
@@ -776,6 +812,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Options include:
* `append`: Only the new rows in the streaming DataFrame/Dataset will
be written to
@@ -818,6 +857,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
source : str
@@ -857,6 +899,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Notes
-----
This API is evolving.
@@ -885,6 +930,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Notes
-----
This API is evolving.
@@ -925,6 +973,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
cols : str or list
@@ -968,6 +1019,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
queryName : str
@@ -1023,6 +1077,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
processingTime : str, optional
@@ -1280,6 +1337,9 @@ class DataStreamWriter:
.. versionadded:: 2.4.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Notes
-----
This API is evolving.
@@ -1388,6 +1448,9 @@ class DataStreamWriter:
.. versionadded:: 2.0.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
path : str, optional
@@ -1473,6 +1536,9 @@ class DataStreamWriter:
.. versionadded:: 3.1.0
+ .. versionchanged:: 3.5.0
+ Supports Spark Connect.
+
Parameters
----------
tableName : str
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
