xinrong-databricks commented on a change in pull request #35410:
URL: https://github.com/apache/spark/pull/35410#discussion_r803491903
##########
File path: python/pyspark/sql/dataframe.py
##########
@@ -102,12 +104,34 @@ class DataFrame(PandasMapOpsMixin, PandasConversionMixin):
.groupBy(department.name, "gender").agg({"salary": "avg", "age":
"max"})
.. versionadded:: 1.3.0
+
+ .. note: A DataFrame should only be created as described above. It should
not be directly
+ created via using the constructor.
"""
- def __init__(self, jdf: JavaObject, sql_ctx: "SQLContext"):
- self._jdf = jdf
- self.sql_ctx = sql_ctx
- self._sc: SparkContext = cast(SparkContext, sql_ctx and sql_ctx._sc)
+ def __init__(
+ self,
+ jdf: JavaObject,
+ sql_ctx: Optional[Union["SQLContext", "SparkSession"]],
+ ):
+ from pyspark.sql.session import SparkSession
+
+ self._session: Optional["SparkSession"] = None
+ self._sql_ctx: Optional["SQLContext"] = None
+
+ if isinstance(sql_ctx, SparkSession):
+ self._session = sql_ctx
+ else:
+ assert not os.environ.get("SPARK_TESTING") # Sanity check for our
internal usage.
+ assert isinstance(sql_ctx, SQLContext)
+ # We should remove this if-else branch in the future release, and
rename
+ # sql_ctx to session in the constructor. This is an internal code
path but
+ # was kept with an warning because but used intensively by
third-party libraries.
Review comment:
nit: because `it's`? used intensively by third-party libraries
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
