rangadi commented on code in PR #41540:
URL: https://github.com/apache/spark/pull/41540#discussion_r1229991823
##########
python/pyspark/sql/streaming/listener.py:
##########
@@ -473,14 +603,14 @@ def json(self) -> str:
"""
The compact JSON representation of this progress.
"""
- return self._jprogress.json()
+ return self._json
@property
def prettyJson(self) -> str:
"""
The pretty (i.e. indented) JSON representation of this progress.
"""
- return self._jprogress.prettyJson()
+ return self._prettyJson
Review Comment:
Better to it from `_json` on the fly, rather than saving `_prettyJson`
##########
python/pyspark/sql/streaming/listener.py:
##########
@@ -563,14 +748,14 @@ def json(self) -> str:
"""
The compact JSON representation of this progress.
"""
- return self._jprogress.json()
+ return self._json
@property
def prettyJson(self) -> str:
"""
The pretty (i.e. indented) JSON representation of this progress.
"""
- return self._jprogress.prettyJson()
+ return self._prettyJson
Review Comment:
Same here. Remove `_prettyJson` class variable.
##########
python/pyspark/sql/streaming/listener.py:
##########
@@ -654,14 +882,14 @@ def json(self) -> str:
"""
The compact JSON representation of this progress.
"""
- return self._jprogress.json()
+ return self._json
@property
def prettyJson(self) -> str:
"""
The pretty (i.e. indented) JSON representation of this progress.
"""
- return self._jprogress.prettyJson()
+ return self._prettyJson
Review Comment:
Same.
##########
python/pyspark/sql/streaming/listener.py:
##########
@@ -155,11 +156,31 @@ class QueryStartedEvent:
This API is evolving.
"""
- def __init__(self, jevent: JavaObject) -> None:
- self._id: uuid.UUID = uuid.UUID(jevent.id().toString())
- self._runId: uuid.UUID = uuid.UUID(jevent.runId().toString())
- self._name: Optional[str] = jevent.name()
- self._timestamp: str = jevent.timestamp()
+ def __init__(
+ self, id: uuid.UUID, runId: uuid.UUID, name: Optional[str], timestamp:
str
+ ) -> None:
+ self._id: uuid.UUID = id
+ self._runId: uuid.UUID = runId
+ self._name: Optional[str] = name
+ self._timestamp: str = timestamp
+
+ @classmethod
+ def fromJObject(cls, jevent: JavaObject) -> "QueryStartedEvent":
+ return cls(
+ id=uuid.UUID(jevent.id().toString()),
+ runId=uuid.UUID(jevent.runId().toString()),
+ name=jevent.name(),
+ timestamp=jevent.timestamp(),
+ )
+
+ @classmethod
+ def fromJson(cls, j: Dict[str, Any]) -> "QueryStartedEvent":
Review Comment:
What is the context where these are used? Thanks for the detailed
description and you mentioned it for 'step 3'. Could you point to any WIP code
that uses this?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
