HyukjinKwon commented on a change in pull request #27331:
URL: https://github.com/apache/spark/pull/27331#discussion_r445267780
##########
File path: python/pyspark/sql/tests/test_readwriter.py
##########
@@ -163,6 +163,43 @@ def test_insert_into(self):
self.assertEqual(6, self.spark.sql("select * from
test_table").count())
+class ReadwriterV2Tests(ReusedSQLTestCase):
+ def test_api(self):
+ from pyspark.sql.readwriter import DataFrameWriterV2
+ from pyspark.sql.functions import col
Review comment:
I think we can just import these on the top
##########
File path: python/pyspark/sql/dataframe.py
##########
@@ -2220,6 +2220,22 @@ def semanticHash(self):
sinceversion=1.4,
doc=":func:`drop_duplicates` is an alias for :func:`dropDuplicates`.")
+ @since(3.1)
+ def writeTo(self, table):
+ """
+ Create a write configuration builder for v2 sources.
+
+ This builder is used to configure and execute write operations.
+
+ For example, to append or create or replace existing tables.
+
+ >>> df.writeTo("catalog.db.table").append() # doctest: +SKIP
+ >>> df.writeTo( # doctest: +SKIP
+ ... "catalog.db.table"
+ ... ).partitionedBy($"col").createOrReplace()
Review comment:
I guess it shouldn't be `$"col"`
##########
File path: python/pyspark/sql/tests/test_readwriter.py
##########
@@ -163,6 +163,43 @@ def test_insert_into(self):
self.assertEqual(6, self.spark.sql("select * from
test_table").count())
+class ReadwriterV2Tests(ReusedSQLTestCase):
+ def test_api(self):
+ from pyspark.sql.readwriter import DataFrameWriterV2
+ from pyspark.sql.functions import col
+
+ df = self.df
+ writer = df.writeTo("testcat.t")
+ self.assertIsInstance(writer, DataFrameWriterV2)
+ self.assertIsInstance(writer.option("property", "value"),
DataFrameWriterV2)
+ self.assertIsInstance(writer.options(property="value"),
DataFrameWriterV2)
+ self.assertIsInstance(writer.using("source"), DataFrameWriterV2)
+ self.assertIsInstance(writer.partitionedBy("id"), DataFrameWriterV2)
+ self.assertIsInstance(writer.partitionedBy(col("id")),
DataFrameWriterV2)
+
+ def test_partitioning_functions(self):
+ import datetime
+ from pyspark.sql.readwriter import DataFrameWriterV2
+ from pyspark.sql.functions import col, years, months, days, hours,
bucket
+
+ df = self.spark.createDataFrame(
+ [(1, datetime.datetime.now(), "foo")],
Review comment:
I would avoid the indeterministic value in the test unless it's
necessary.
##########
File path: python/pyspark/sql/readwriter.py
##########
@@ -1048,6 +1048,128 @@ def jdbc(self, url, table, mode=None, properties=None):
self.mode(mode)._jwrite.jdbc(url, table, jprop)
+class DataFrameWriterV2(object):
+ """
+ Interface used to write a class:`pyspark.sql.dataframe.DataFrame`
+ to external storage using the v2 API.
+
+ .. versionadded:: 3.1.0
+ """
+
+ def __init__(self, df, table):
+ self._df = df
+ self._spark = df.sql_ctx
+ self._jwriter = df._jdf.writeTo(table)
+
+ @since(3.1)
+ def using(self, provider):
+ """
+ Specifies a provider for the underlying output data source.
+ Spark's default catalog supports "parquet", "json", etc.
+ """
+ self._jwriter.using(provider)
+ return self
+
+ @since(3.1)
+ def option(self, key, value):
+ """
+ Add a write option.
+ """
+ self._jwriter.option(key, to_str(value))
+ return self
+
+ @since(3.1)
+ def options(self, **options):
+ """
+ Add write options.
+ """
+ options = {k: to_str(v) for k, v in options.items()}
+ self._jwriter.options(options)
+ return self
+
+ @since(3.1)
+ def partitionedBy(self, col, *cols):
Review comment:
Maybe it's important to describe what are expected for `col`. Only
columns and the partition transform functions are allowed, not the regular
Spark Column. I still don't like it we made this API looks like it takes
regular Spark Columns, this was one of the reason why Pandas UDFs were separate
into two separate groups .. let's at least clarify it.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
