This is an automated email from the ASF dual-hosted git repository.
ruifengz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new 1f32721cad53 [SPARK-55395][SQL][FOLLOW-UP] Delete obsolete
`withSequenceColumn`
1f32721cad53 is described below
commit 1f32721cad53306d6f43269af9e9ce01a4455a53
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Wed Feb 11 08:47:44 2026 +0800
[SPARK-55395][SQL][FOLLOW-UP] Delete obsolete `withSequenceColumn`
### What changes were proposed in this pull request?
Delete obsolete `withSequenceColumn` from `Dataset`
### Why are the changes needed?
this method was originally added for pandas api on spark, but after python
side refactoring, this method has been unused for a while
### Does this PR introduce _any_ user-facing change?
no
### How was this patch tested?
ci
### Was this patch authored or co-authored using generative AI tooling?
no
Closes #54244 from zhengruifeng/cleanup_legacy.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../src/main/scala/org/apache/spark/sql/classic/Dataset.scala | 8 --------
sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala | 2 +-
2 files changed, 1 insertion(+), 9 deletions(-)
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
b/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
index 17d4640f22fa..84b356855710 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/classic/Dataset.scala
@@ -2057,14 +2057,6 @@ class Dataset[T] private[sql](
// For Python API
////////////////////////////////////////////////////////////////////////////
- /**
- * It adds a new long column with the name `name` that increases one by one.
- * This is for 'distributed-sequence' default index in pandas API on Spark.
- */
- private[sql] def withSequenceColumn(name: String) = {
- select(Column(DistributedSequenceID(Literal(true))).alias(name), col("*"))
- }
-
/**
* Converts a JavaRDD to a PythonRDD.
*/
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
index 5b88eeefeca7..fdeed60e1929 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/DataFrameSuite.scala
@@ -2406,7 +2406,7 @@ class DataFrameSuite extends QueryTest
}
}
- test("SPARK-36338: DataFrame.withSequenceColumn should append unique
sequence IDs") {
+ test("SPARK-36338: AttachDistributedSequence should append unique sequence
IDs") {
val ids = spark.range(10).repartition(5).select(
Column.internalFn("distributed_sequence_id").alias("default_index"),
col("id"))
assert(ids.collect().map(_.getLong(0)).toSet === Range(0, 10).toSet)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
