This is an automated email from the ASF dual-hosted git repository.
gurwls223 pushed a commit to branch branch-3.5
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-3.5 by this push:
new e6d5ddd2e846 Revert "[SPARK-48639][CONNECT][PYTHON] Add Origin to
Relation.RelationCommon"
e6d5ddd2e846 is described below
commit e6d5ddd2e8461b0c22f5bfabf334d59088d863cc
Author: Hyukjin Kwon <[email protected]>
AuthorDate: Thu Jun 27 12:24:05 2024 +0900
Revert "[SPARK-48639][CONNECT][PYTHON] Add Origin to
Relation.RelationCommon"
This reverts commit 0db5bdecfa6cbfff1be7690bb783a858989776b9.
---
.../main/protobuf/spark/connect/relations.proto | 4 +-
python/pyspark/sql/connect/proto/relations_pb2.py | 264 ++++++++++-----------
python/pyspark/sql/connect/proto/relations_pb2.pyi | 9 +-
3 files changed, 142 insertions(+), 135 deletions(-)
diff --git
a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
index 8b384728983b..f7f1315ede0f 100644
--- a/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
+++ b/connector/connect/common/src/main/protobuf/spark/connect/relations.proto
@@ -103,8 +103,8 @@ message Unknown {}
// Common metadata of all relations.
message RelationCommon {
- // (Optional) Shared relation metadata.
- reserved 1;
+ // (Required) Shared relation metadata.
+ string source_info = 1;
// (Optional) A per-client globally unique id for a given connect plan.
optional int64 plan_id = 2;
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.py
b/python/pyspark/sql/connect/proto/relations_pb2.py
index 7dd494db8695..3f7e57949373 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.py
+++ b/python/pyspark/sql/connect/proto/relations_pb2.py
@@ -35,7 +35,7 @@ from pyspark.sql.connect.proto import catalog_pb2 as
spark_dot_connect_dot_catal
DESCRIPTOR = _descriptor_pool.Default().AddSerializedFile(
-
b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto"\xe1\x18\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01
\x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02
\x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03
\x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66il [...]
+
b'\n\x1dspark/connect/relations.proto\x12\rspark.connect\x1a\x19google/protobuf/any.proto\x1a\x1fspark/connect/expressions.proto\x1a\x19spark/connect/types.proto\x1a\x1bspark/connect/catalog.proto"\xe1\x18\n\x08Relation\x12\x35\n\x06\x63ommon\x18\x01
\x01(\x0b\x32\x1d.spark.connect.RelationCommonR\x06\x63ommon\x12)\n\x04read\x18\x02
\x01(\x0b\x32\x13.spark.connect.ReadH\x00R\x04read\x12\x32\n\x07project\x18\x03
\x01(\x0b\x32\x16.spark.connect.ProjectH\x00R\x07project\x12/\n\x06\x66il [...]
)
_builder.BuildMessageAndEnumDescriptors(DESCRIPTOR, globals())
@@ -61,135 +61,135 @@ if _descriptor._USE_C_DESCRIPTORS == False:
_UNKNOWN._serialized_start = 3336
_UNKNOWN._serialized_end = 3345
_RELATIONCOMMON._serialized_start = 3347
- _RELATIONCOMMON._serialized_end = 3411
- _SQL._serialized_start = 3414
- _SQL._serialized_end = 3645
- _SQL_ARGSENTRY._serialized_start = 3555
- _SQL_ARGSENTRY._serialized_end = 3645
- _READ._serialized_start = 3648
- _READ._serialized_end = 4311
- _READ_NAMEDTABLE._serialized_start = 3826
- _READ_NAMEDTABLE._serialized_end = 4018
- _READ_NAMEDTABLE_OPTIONSENTRY._serialized_start = 3960
- _READ_NAMEDTABLE_OPTIONSENTRY._serialized_end = 4018
- _READ_DATASOURCE._serialized_start = 4021
- _READ_DATASOURCE._serialized_end = 4298
- _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 3960
- _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 4018
- _PROJECT._serialized_start = 4313
- _PROJECT._serialized_end = 4430
- _FILTER._serialized_start = 4432
- _FILTER._serialized_end = 4544
- _JOIN._serialized_start = 4547
- _JOIN._serialized_end = 5208
- _JOIN_JOINDATATYPE._serialized_start = 4886
- _JOIN_JOINDATATYPE._serialized_end = 4978
- _JOIN_JOINTYPE._serialized_start = 4981
- _JOIN_JOINTYPE._serialized_end = 5189
- _SETOPERATION._serialized_start = 5211
- _SETOPERATION._serialized_end = 5690
- _SETOPERATION_SETOPTYPE._serialized_start = 5527
- _SETOPERATION_SETOPTYPE._serialized_end = 5641
- _LIMIT._serialized_start = 5692
- _LIMIT._serialized_end = 5768
- _OFFSET._serialized_start = 5770
- _OFFSET._serialized_end = 5849
- _TAIL._serialized_start = 5851
- _TAIL._serialized_end = 5926
- _AGGREGATE._serialized_start = 5929
- _AGGREGATE._serialized_end = 6511
- _AGGREGATE_PIVOT._serialized_start = 6268
- _AGGREGATE_PIVOT._serialized_end = 6379
- _AGGREGATE_GROUPTYPE._serialized_start = 6382
- _AGGREGATE_GROUPTYPE._serialized_end = 6511
- _SORT._serialized_start = 6514
- _SORT._serialized_end = 6674
- _DROP._serialized_start = 6677
- _DROP._serialized_end = 6818
- _DEDUPLICATE._serialized_start = 6821
- _DEDUPLICATE._serialized_end = 7061
- _LOCALRELATION._serialized_start = 7063
- _LOCALRELATION._serialized_end = 7152
- _CACHEDLOCALRELATION._serialized_start = 7154
- _CACHEDLOCALRELATION._serialized_end = 7226
- _CACHEDREMOTERELATION._serialized_start = 7228
- _CACHEDREMOTERELATION._serialized_end = 7283
- _SAMPLE._serialized_start = 7286
- _SAMPLE._serialized_end = 7559
- _RANGE._serialized_start = 7562
- _RANGE._serialized_end = 7707
- _SUBQUERYALIAS._serialized_start = 7709
- _SUBQUERYALIAS._serialized_end = 7823
- _REPARTITION._serialized_start = 7826
- _REPARTITION._serialized_end = 7968
- _SHOWSTRING._serialized_start = 7971
- _SHOWSTRING._serialized_end = 8113
- _HTMLSTRING._serialized_start = 8115
- _HTMLSTRING._serialized_end = 8229
- _STATSUMMARY._serialized_start = 8231
- _STATSUMMARY._serialized_end = 8323
- _STATDESCRIBE._serialized_start = 8325
- _STATDESCRIBE._serialized_end = 8406
- _STATCROSSTAB._serialized_start = 8408
- _STATCROSSTAB._serialized_end = 8509
- _STATCOV._serialized_start = 8511
- _STATCOV._serialized_end = 8607
- _STATCORR._serialized_start = 8610
- _STATCORR._serialized_end = 8747
- _STATAPPROXQUANTILE._serialized_start = 8750
- _STATAPPROXQUANTILE._serialized_end = 8914
- _STATFREQITEMS._serialized_start = 8916
- _STATFREQITEMS._serialized_end = 9041
- _STATSAMPLEBY._serialized_start = 9044
- _STATSAMPLEBY._serialized_end = 9353
- _STATSAMPLEBY_FRACTION._serialized_start = 9245
- _STATSAMPLEBY_FRACTION._serialized_end = 9344
- _NAFILL._serialized_start = 9356
- _NAFILL._serialized_end = 9490
- _NADROP._serialized_start = 9493
- _NADROP._serialized_end = 9627
- _NAREPLACE._serialized_start = 9630
- _NAREPLACE._serialized_end = 9926
- _NAREPLACE_REPLACEMENT._serialized_start = 9785
- _NAREPLACE_REPLACEMENT._serialized_end = 9926
- _TODF._serialized_start = 9928
- _TODF._serialized_end = 10016
- _WITHCOLUMNSRENAMED._serialized_start = 10019
- _WITHCOLUMNSRENAMED._serialized_end = 10258
- _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_start = 10191
- _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_end = 10258
- _WITHCOLUMNS._serialized_start = 10260
- _WITHCOLUMNS._serialized_end = 10379
- _WITHWATERMARK._serialized_start = 10382
- _WITHWATERMARK._serialized_end = 10516
- _HINT._serialized_start = 10519
- _HINT._serialized_end = 10651
- _UNPIVOT._serialized_start = 10654
- _UNPIVOT._serialized_end = 10981
- _UNPIVOT_VALUES._serialized_start = 10911
- _UNPIVOT_VALUES._serialized_end = 10970
- _TOSCHEMA._serialized_start = 10983
- _TOSCHEMA._serialized_end = 11089
- _REPARTITIONBYEXPRESSION._serialized_start = 11092
- _REPARTITIONBYEXPRESSION._serialized_end = 11295
- _MAPPARTITIONS._serialized_start = 11298
- _MAPPARTITIONS._serialized_end = 11479
- _GROUPMAP._serialized_start = 11482
- _GROUPMAP._serialized_end = 12117
- _COGROUPMAP._serialized_start = 12120
- _COGROUPMAP._serialized_end = 12646
- _APPLYINPANDASWITHSTATE._serialized_start = 12649
- _APPLYINPANDASWITHSTATE._serialized_end = 13006
- _COMMONINLINEUSERDEFINEDTABLEFUNCTION._serialized_start = 13009
- _COMMONINLINEUSERDEFINEDTABLEFUNCTION._serialized_end = 13253
- _PYTHONUDTF._serialized_start = 13256
- _PYTHONUDTF._serialized_end = 13433
- _COLLECTMETRICS._serialized_start = 13436
- _COLLECTMETRICS._serialized_end = 13572
- _PARSE._serialized_start = 13575
- _PARSE._serialized_end = 13963
- _PARSE_OPTIONSENTRY._serialized_start = 3960
- _PARSE_OPTIONSENTRY._serialized_end = 4018
- _PARSE_PARSEFORMAT._serialized_start = 13864
- _PARSE_PARSEFORMAT._serialized_end = 13952
+ _RELATIONCOMMON._serialized_end = 3438
+ _SQL._serialized_start = 3441
+ _SQL._serialized_end = 3672
+ _SQL_ARGSENTRY._serialized_start = 3582
+ _SQL_ARGSENTRY._serialized_end = 3672
+ _READ._serialized_start = 3675
+ _READ._serialized_end = 4338
+ _READ_NAMEDTABLE._serialized_start = 3853
+ _READ_NAMEDTABLE._serialized_end = 4045
+ _READ_NAMEDTABLE_OPTIONSENTRY._serialized_start = 3987
+ _READ_NAMEDTABLE_OPTIONSENTRY._serialized_end = 4045
+ _READ_DATASOURCE._serialized_start = 4048
+ _READ_DATASOURCE._serialized_end = 4325
+ _READ_DATASOURCE_OPTIONSENTRY._serialized_start = 3987
+ _READ_DATASOURCE_OPTIONSENTRY._serialized_end = 4045
+ _PROJECT._serialized_start = 4340
+ _PROJECT._serialized_end = 4457
+ _FILTER._serialized_start = 4459
+ _FILTER._serialized_end = 4571
+ _JOIN._serialized_start = 4574
+ _JOIN._serialized_end = 5235
+ _JOIN_JOINDATATYPE._serialized_start = 4913
+ _JOIN_JOINDATATYPE._serialized_end = 5005
+ _JOIN_JOINTYPE._serialized_start = 5008
+ _JOIN_JOINTYPE._serialized_end = 5216
+ _SETOPERATION._serialized_start = 5238
+ _SETOPERATION._serialized_end = 5717
+ _SETOPERATION_SETOPTYPE._serialized_start = 5554
+ _SETOPERATION_SETOPTYPE._serialized_end = 5668
+ _LIMIT._serialized_start = 5719
+ _LIMIT._serialized_end = 5795
+ _OFFSET._serialized_start = 5797
+ _OFFSET._serialized_end = 5876
+ _TAIL._serialized_start = 5878
+ _TAIL._serialized_end = 5953
+ _AGGREGATE._serialized_start = 5956
+ _AGGREGATE._serialized_end = 6538
+ _AGGREGATE_PIVOT._serialized_start = 6295
+ _AGGREGATE_PIVOT._serialized_end = 6406
+ _AGGREGATE_GROUPTYPE._serialized_start = 6409
+ _AGGREGATE_GROUPTYPE._serialized_end = 6538
+ _SORT._serialized_start = 6541
+ _SORT._serialized_end = 6701
+ _DROP._serialized_start = 6704
+ _DROP._serialized_end = 6845
+ _DEDUPLICATE._serialized_start = 6848
+ _DEDUPLICATE._serialized_end = 7088
+ _LOCALRELATION._serialized_start = 7090
+ _LOCALRELATION._serialized_end = 7179
+ _CACHEDLOCALRELATION._serialized_start = 7181
+ _CACHEDLOCALRELATION._serialized_end = 7253
+ _CACHEDREMOTERELATION._serialized_start = 7255
+ _CACHEDREMOTERELATION._serialized_end = 7310
+ _SAMPLE._serialized_start = 7313
+ _SAMPLE._serialized_end = 7586
+ _RANGE._serialized_start = 7589
+ _RANGE._serialized_end = 7734
+ _SUBQUERYALIAS._serialized_start = 7736
+ _SUBQUERYALIAS._serialized_end = 7850
+ _REPARTITION._serialized_start = 7853
+ _REPARTITION._serialized_end = 7995
+ _SHOWSTRING._serialized_start = 7998
+ _SHOWSTRING._serialized_end = 8140
+ _HTMLSTRING._serialized_start = 8142
+ _HTMLSTRING._serialized_end = 8256
+ _STATSUMMARY._serialized_start = 8258
+ _STATSUMMARY._serialized_end = 8350
+ _STATDESCRIBE._serialized_start = 8352
+ _STATDESCRIBE._serialized_end = 8433
+ _STATCROSSTAB._serialized_start = 8435
+ _STATCROSSTAB._serialized_end = 8536
+ _STATCOV._serialized_start = 8538
+ _STATCOV._serialized_end = 8634
+ _STATCORR._serialized_start = 8637
+ _STATCORR._serialized_end = 8774
+ _STATAPPROXQUANTILE._serialized_start = 8777
+ _STATAPPROXQUANTILE._serialized_end = 8941
+ _STATFREQITEMS._serialized_start = 8943
+ _STATFREQITEMS._serialized_end = 9068
+ _STATSAMPLEBY._serialized_start = 9071
+ _STATSAMPLEBY._serialized_end = 9380
+ _STATSAMPLEBY_FRACTION._serialized_start = 9272
+ _STATSAMPLEBY_FRACTION._serialized_end = 9371
+ _NAFILL._serialized_start = 9383
+ _NAFILL._serialized_end = 9517
+ _NADROP._serialized_start = 9520
+ _NADROP._serialized_end = 9654
+ _NAREPLACE._serialized_start = 9657
+ _NAREPLACE._serialized_end = 9953
+ _NAREPLACE_REPLACEMENT._serialized_start = 9812
+ _NAREPLACE_REPLACEMENT._serialized_end = 9953
+ _TODF._serialized_start = 9955
+ _TODF._serialized_end = 10043
+ _WITHCOLUMNSRENAMED._serialized_start = 10046
+ _WITHCOLUMNSRENAMED._serialized_end = 10285
+ _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_start = 10218
+ _WITHCOLUMNSRENAMED_RENAMECOLUMNSMAPENTRY._serialized_end = 10285
+ _WITHCOLUMNS._serialized_start = 10287
+ _WITHCOLUMNS._serialized_end = 10406
+ _WITHWATERMARK._serialized_start = 10409
+ _WITHWATERMARK._serialized_end = 10543
+ _HINT._serialized_start = 10546
+ _HINT._serialized_end = 10678
+ _UNPIVOT._serialized_start = 10681
+ _UNPIVOT._serialized_end = 11008
+ _UNPIVOT_VALUES._serialized_start = 10938
+ _UNPIVOT_VALUES._serialized_end = 10997
+ _TOSCHEMA._serialized_start = 11010
+ _TOSCHEMA._serialized_end = 11116
+ _REPARTITIONBYEXPRESSION._serialized_start = 11119
+ _REPARTITIONBYEXPRESSION._serialized_end = 11322
+ _MAPPARTITIONS._serialized_start = 11325
+ _MAPPARTITIONS._serialized_end = 11506
+ _GROUPMAP._serialized_start = 11509
+ _GROUPMAP._serialized_end = 12144
+ _COGROUPMAP._serialized_start = 12147
+ _COGROUPMAP._serialized_end = 12673
+ _APPLYINPANDASWITHSTATE._serialized_start = 12676
+ _APPLYINPANDASWITHSTATE._serialized_end = 13033
+ _COMMONINLINEUSERDEFINEDTABLEFUNCTION._serialized_start = 13036
+ _COMMONINLINEUSERDEFINEDTABLEFUNCTION._serialized_end = 13280
+ _PYTHONUDTF._serialized_start = 13283
+ _PYTHONUDTF._serialized_end = 13460
+ _COLLECTMETRICS._serialized_start = 13463
+ _COLLECTMETRICS._serialized_end = 13599
+ _PARSE._serialized_start = 13602
+ _PARSE._serialized_end = 13990
+ _PARSE_OPTIONSENTRY._serialized_start = 3987
+ _PARSE_OPTIONSENTRY._serialized_end = 4045
+ _PARSE_PARSEFORMAT._serialized_start = 13891
+ _PARSE_PARSEFORMAT._serialized_end = 13979
# @@protoc_insertion_point(module_scope)
diff --git a/python/pyspark/sql/connect/proto/relations_pb2.pyi
b/python/pyspark/sql/connect/proto/relations_pb2.pyi
index 1c0036afbc43..007b92ef5f42 100644
--- a/python/pyspark/sql/connect/proto/relations_pb2.pyi
+++ b/python/pyspark/sql/connect/proto/relations_pb2.pyi
@@ -579,19 +579,26 @@ class RelationCommon(google.protobuf.message.Message):
DESCRIPTOR: google.protobuf.descriptor.Descriptor
+ SOURCE_INFO_FIELD_NUMBER: builtins.int
PLAN_ID_FIELD_NUMBER: builtins.int
+ source_info: builtins.str
+ """(Required) Shared relation metadata."""
plan_id: builtins.int
"""(Optional) A per-client globally unique id for a given connect plan."""
def __init__(
self,
*,
+ source_info: builtins.str = ...,
plan_id: builtins.int | None = ...,
) -> None: ...
def HasField(
self, field_name: typing_extensions.Literal["_plan_id", b"_plan_id",
"plan_id", b"plan_id"]
) -> builtins.bool: ...
def ClearField(
- self, field_name: typing_extensions.Literal["_plan_id", b"_plan_id",
"plan_id", b"plan_id"]
+ self,
+ field_name: typing_extensions.Literal[
+ "_plan_id", b"_plan_id", "plan_id", b"plan_id", "source_info",
b"source_info"
+ ],
) -> None: ...
def WhichOneof(
self, oneof_group: typing_extensions.Literal["_plan_id", b"_plan_id"]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
