This is an automated email from the ASF dual-hosted git repository.
kabhwan pushed a change to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
from e616b8404ff2 [SPARK-51836][PYTHON][CONNECT][TESTS][FOLLOW-UP] update
`test_connect_classification`
add ec430279b43d [SPARK-51814][SS][PYTHON] Introduce a new API
`transformWithState` in PySpark
No new revisions were added by this update.
Summary of changes:
.../org/apache/spark/api/python/PythonRunner.scala | 2 +
python/pyspark/sql/connect/group.py | 6 +-
python/pyspark/sql/pandas/_typing/__init__.pyi | 2 +
python/pyspark/sql/pandas/functions.py | 4 +
python/pyspark/sql/pandas/group_ops.py | 200 ++-
python/pyspark/sql/pandas/serializers.py | 229 ++-
python/pyspark/sql/streaming/stateful_processor.py | 56 +-
.../sql/streaming/stateful_processor_util.py | 60 +-
.../transform_with_state_driver_worker.py | 6 +-
.../test_parity_pandas_transform_with_state.py | 3 +
.../pyspark/sql/tests/pandas/helper}/__init__.py | 0
.../helper/helper_pandas_transform_with_state.py | 1478 ++++++++++++++++++++
.../pandas/test_pandas_transform_with_state.py | 1118 +++++----------
.../sql/tests/test_connect_compatibility.py | 3 +-
python/pyspark/util.py | 6 +
python/pyspark/worker.py | 125 +-
.../analysis/UnsupportedOperationChecker.scala | 2 +-
.../plans/logical/pythonLogicalOperators.scala | 13 +-
.../org/apache/spark/sql/internal/SQLConf.scala | 16 +-
.../sql/classic/RelationalGroupedDataset.scala | 50 +-
.../spark/sql/execution/SparkStrategies.scala | 24 +-
.../datasources/v2/state/StateDataSource.scala | 6 +-
... TransformWithStateInPySparkDeserializer.scala} | 6 +-
...scala => TransformWithStateInPySparkExec.scala} | 60 +-
... TransformWithStateInPySparkPythonRunner.scala} | 42 +-
...> TransformWithStateInPySparkStateServer.scala} | 22 +-
.../execution/streaming/IncrementalExecution.scala | 18 +-
.../streaming/BaseStreamingArrowWriterSuite.scala | 16 +-
...nsformWithStateInPySparkStateServerSuite.scala} | 48 +-
29 files changed, 2646 insertions(+), 975 deletions(-)
copy {examples/src/main/python =>
python/pyspark/sql/tests/pandas/helper}/__init__.py (100%)
create mode 100644
python/pyspark/sql/tests/pandas/helper/helper_pandas_transform_with_state.py
rename
sql/core/src/main/scala/org/apache/spark/sql/execution/python/streaming/{TransformWithStateInPandasDeserializer.scala
=> TransformWithStateInPySparkDeserializer.scala} (92%)
rename
sql/core/src/main/scala/org/apache/spark/sql/execution/python/streaming/{TransformWithStateInPandasExec.scala
=> TransformWithStateInPySparkExec.scala} (91%)
rename
sql/core/src/main/scala/org/apache/spark/sql/execution/python/streaming/{TransformWithStateInPandasPythonRunner.scala
=> TransformWithStateInPySparkPythonRunner.scala} (89%)
rename
sql/core/src/main/scala/org/apache/spark/sql/execution/python/streaming/{TransformWithStateInPandasStateServer.scala
=> TransformWithStateInPySparkStateServer.scala} (98%)
rename
sql/core/src/test/scala/org/apache/spark/sql/execution/python/streaming/{TransformWithStateInPandasStateServerSuite.scala
=> TransformWithStateInPySparkStateServerSuite.scala} (93%)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org
