This is an automated email from the ASF dual-hosted git repository. gurwls223 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push: new a4ca577445d [SPARK-41305][CONNECT] Improve Documentation for Command proto a4ca577445d is described below commit a4ca577445d7867db777cbb668f36f44c99ac1ce Author: Rui Wang <rui.w...@databricks.com> AuthorDate: Mon Dec 5 09:59:56 2022 +0900 [SPARK-41305][CONNECT] Improve Documentation for Command proto ### What changes were proposed in this pull request? Improve Command.proto documentation by following https://github.com/apache/spark/blob/master/connector/connect/docs/adding-proto-messages.md. ### Why are the changes needed? Documentation qualify ### Does this PR introduce _any_ user-facing change? NO ### How was this patch tested? N/A Closes #38890 from amaliujia/improve_command_proto_documentation. Authored-by: Rui Wang <rui.w...@databricks.com> Signed-off-by: Hyukjin Kwon <gurwls...@apache.org> --- .../src/main/protobuf/spark/connect/commands.proto | 38 +++++++++++++++------- python/pyspark/sql/connect/proto/commands_pb2.pyi | 31 ++++++++++-------- 2 files changed, 45 insertions(+), 24 deletions(-) diff --git a/connector/connect/src/main/protobuf/spark/connect/commands.proto b/connector/connect/src/main/protobuf/spark/connect/commands.proto index 086d4d0cc92..650f4fb7fa1 100644 --- a/connector/connect/src/main/protobuf/spark/connect/commands.proto +++ b/connector/connect/src/main/protobuf/spark/connect/commands.proto @@ -43,11 +43,19 @@ message Command { // TODO(SPARK-40532) It is required to add the interpreter / language version to the command // parameters. message CreateScalarFunction { - // Fully qualified name of the function including the catalog / schema names. + // (Required) Fully qualified name of the function including the catalog / schema names. repeated string parts = 1; + + // (Required) the function language. FunctionLanguage language = 2; + + // (Required) if this is a temporary function. bool temporary = 3; + + // (Optional) A list of argument types. Can be empty when the function does not take an argument. repeated DataType argument_types = 4; + + // (Required) the return type of the function. DataType return_type = 5; // How the function body is defined: @@ -68,16 +76,16 @@ message CreateScalarFunction { // A command that can create DataFrame global temp view or local temp view. message CreateDataFrameViewCommand { - // Required. The relation that this view will be built on. + // (Required) The relation that this view will be built on. Relation input = 1; - // Required. View name. + // (Required) View name. string name = 2; - // Required. Whether this is global temp view or local temp view. + // (Required) Whether this is global temp view or local temp view. bool is_global = 3; - // Required. + // (Required) // // If true, and if the view already exists, updates it; if false, and if the view // already exists, throws exception. @@ -86,24 +94,32 @@ message CreateDataFrameViewCommand { // As writes are not directly handled during analysis and planning, they are modeled as commands. message WriteOperation { - // The output of the `input` relation will be persisted according to the options. + // (Required) The output of the `input` relation will be persisted according to the options. Relation input = 1; - // Format value according to the Spark documentation. Examples are: text, parquet, delta. + + // (Required) Format value according to the Spark documentation. Examples are: text, parquet, delta. string source = 2; + // The destination of the write operation must be either a path or a table. oneof save_type { string path = 3; string table_name = 4; } + + // (Required) the save mode. SaveMode mode = 5; - // List of columns to sort the output by. + + // (Optional) List of columns to sort the output by. repeated string sort_column_names = 6; - // List of columns for partitioning. + + // (Optional) List of columns for partitioning. repeated string partitioning_columns = 7; - // Optional bucketing specification. Bucketing must set the number of buckets and the columns + + // (Optional) Bucketing specification. Bucketing must set the number of buckets and the columns // to bucket by. BucketBy bucket_by = 8; - // Optional list of configuration options. + + // (Optional) A list of configuration options. map<string, string> options = 9; message BucketBy { diff --git a/python/pyspark/sql/connect/proto/commands_pb2.pyi b/python/pyspark/sql/connect/proto/commands_pb2.pyi index 9b9880e0b93..c3c7d31222b 100644 --- a/python/pyspark/sql/connect/proto/commands_pb2.pyi +++ b/python/pyspark/sql/connect/proto/commands_pb2.pyi @@ -153,17 +153,21 @@ class CreateScalarFunction(google.protobuf.message.Message): def parts( self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: - """Fully qualified name of the function including the catalog / schema names.""" + """(Required) Fully qualified name of the function including the catalog / schema names.""" language: global___CreateScalarFunction.FunctionLanguage.ValueType + """(Required) the function language.""" temporary: builtins.bool + """(Required) if this is a temporary function.""" @property def argument_types( self, ) -> google.protobuf.internal.containers.RepeatedCompositeFieldContainer[ pyspark.sql.connect.proto.types_pb2.DataType - ]: ... + ]: + """(Optional) A list of argument types. Can be empty when the function does not take an argument.""" @property - def return_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType: ... + def return_type(self) -> pyspark.sql.connect.proto.types_pb2.DataType: + """(Required) the return type of the function.""" serialized_function: builtins.bytes """As a raw string serialized:""" literal_string: builtins.str @@ -231,13 +235,13 @@ class CreateDataFrameViewCommand(google.protobuf.message.Message): REPLACE_FIELD_NUMBER: builtins.int @property def input(self) -> pyspark.sql.connect.proto.relations_pb2.Relation: - """Required. The relation that this view will be built on.""" + """(Required) The relation that this view will be built on.""" name: builtins.str - """Required. View name.""" + """(Required) View name.""" is_global: builtins.bool - """Required. Whether this is global temp view or local temp view.""" + """(Required) Whether this is global temp view or local temp view.""" replace: builtins.bool - """Required. + """(Required) If true, and if the view already exists, updates it; if false, and if the view already exists, throws exception. @@ -342,30 +346,31 @@ class WriteOperation(google.protobuf.message.Message): OPTIONS_FIELD_NUMBER: builtins.int @property def input(self) -> pyspark.sql.connect.proto.relations_pb2.Relation: - """The output of the `input` relation will be persisted according to the options.""" + """(Required) The output of the `input` relation will be persisted according to the options.""" source: builtins.str - """Format value according to the Spark documentation. Examples are: text, parquet, delta.""" + """(Required) Format value according to the Spark documentation. Examples are: text, parquet, delta.""" path: builtins.str table_name: builtins.str mode: global___WriteOperation.SaveMode.ValueType + """(Required) the save mode.""" @property def sort_column_names( self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: - """List of columns to sort the output by.""" + """(Optional) List of columns to sort the output by.""" @property def partitioning_columns( self, ) -> google.protobuf.internal.containers.RepeatedScalarFieldContainer[builtins.str]: - """List of columns for partitioning.""" + """(Optional) List of columns for partitioning.""" @property def bucket_by(self) -> global___WriteOperation.BucketBy: - """Optional bucketing specification. Bucketing must set the number of buckets and the columns + """(Optional) Bucketing specification. Bucketing must set the number of buckets and the columns to bucket by. """ @property def options(self) -> google.protobuf.internal.containers.ScalarMap[builtins.str, builtins.str]: - """Optional list of configuration options.""" + """(Optional) A list of configuration options.""" def __init__( self, *, --------------------------------------------------------------------- To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org For additional commands, e-mail: commits-h...@spark.apache.org