amaliujia commented on code in PR #38086:
URL: https://github.com/apache/spark/pull/38086#discussion_r990932248
##########
python/pyspark/sql/connect/readwriter.py:
##########
@@ -31,6 +39,104 @@ class DataFrameReader:
def __init__(self, client: "RemoteSparkSession") -> None:
self._client = client
+ self._path: List[str] = []
+ self._format = ""
+ self._schema = ""
+ self._options: Dict[str, str] = {}
+
+ def format(self, source: str) -> "DataFrameReader":
+ """
+ Specifies the input data source format.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ source : str
+ string, name of the data source, e.g. 'json', 'parquet'.
+
+ """
+ self._format = source
+ return self
+
+ # TODO(SPARK-40539): support StructType in python client and support
schema as StructType.
+ def schema(self, schema: str) -> "DataFrameReader":
+ """
+ Specifies the input schema.
+
+ Some data sources (e.g. JSON) can infer the input schema automatically
from data.
+ By specifying the schema here, the underlying data source can skip the
schema
+ inference step, and thus speed up data loading.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ schema : str
+ a DDL-formatted string
+ (For example ``col0 INT, col1 DOUBLE``).
+
+ """
+ self._schema = schema
+ return self
+
+ def option(self, key: str, value: "OptionalPrimitiveType") ->
"DataFrameReader":
+ """
+ Adds an input option for the underlying data source.
+
+ .. versionadded:: 3.4.0
+
+ Parameters
+ ----------
+ key : str
+ The key for the option to set. key string is case-insensitive.
+ value
+ The value for the option to set.
+
+ """
+ self._options[key] = str(value)
Review Comment:
the key is case insensitive and the server side will convert it anyway.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
