This is an automated email from the ASF dual-hosted git repository.
zhengruifeng pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new fe84f648e55b [SPARK-59521][ML][TEST][FOLLOWUP] Add a compatibility
test for Word2VecModel saved by Spark 4.0.0
fe84f648e55b is described below
commit fe84f648e55b57c6cfbb91121c9b4bdb02be4a6e
Author: Ruifeng Zheng <[email protected]>
AuthorDate: Thu Oct 29 08:08:33 2026 +0800
[SPARK-59521][ML][TEST][FOLLOWUP] Add a compatibility test for
Word2VecModel saved by Spark 4.0.0
### What changes were proposed in this pull request?
Adds a compatibility fixture and a load test for `Word2VecModel`:
- `mllib/src/test/resources/ml-models/w2v-4.0.0`: a `Word2VecModel` saved
by released Spark 4.0.0 (PySpark 4.0.0 `Word2Vec`, `vectorSize=4`, 10-word
vocabulary).
- `Word2VecSuite`: "Load Word2VecModel saved by Spark 4.0.0" loads the
fixture with the current code and checks the vector size, the vocabulary, the
vectors for two words, and the `sparkVersion` in the metadata.
This is a follow-up to SPARK-59521 (commit `1be3a3aae39`), which changed
the `Word2VecModel` constructor and its load path. Existing fixtures in
`ml-models` cover only older versions, so nothing tested the saved format of a
released Spark against the new code.
### Why are the changes needed?
SPARK-59521 changed how `Word2VecModel` stores and loads its word vectors.
A saved model from a released Spark should keep loading after such a change.
This test catches a regression of that kind.
### Does this PR introduce _any_ user-facing change?
No. Test and test-resource changes only.
### How was this patch tested?
Added the new test case and fixture. I did not run the test in this PR: I
did not compile or run `Word2VecSuite` locally. The expected values come from
the PySpark 4.0.0 run that wrote the fixture.
### Was this patch authored or co-authored using generative AI tooling?
Generated-by: Claude (system.ai.claude-haiku-5-5) via Claude Code
This pull request and its description were written by Isaac.
Closes #59856 from zhengruifeng/ml-model-fixtures-compat.
Authored-by: Ruifeng Zheng <[email protected]>
Signed-off-by: Ruifeng Zheng <[email protected]>
---
.../resources/ml-models/w2v-4.0.0/data/._SUCCESS.crc | Bin 0 -> 8 bytes
...1d-4a73-afb5-09d4f5461440-c000.snappy.parquet.crc | Bin 0 -> 20 bytes
.../test/resources/ml-models/w2v-4.0.0/data/_SUCCESS | 0
...5-7b1d-4a73-afb5-09d4f5461440-c000.snappy.parquet | Bin 0 -> 1070 bytes
.../ml-models/w2v-4.0.0/metadata/._SUCCESS.crc | Bin 0 -> 8 bytes
...da33aa45-4cfc-4650-9739-09949b61d451-c000.txt.crc | Bin 0 -> 12 bytes
.../resources/ml-models/w2v-4.0.0/metadata/_SUCCESS | 0
...000-da33aa45-4cfc-4650-9739-09949b61d451-c000.txt | 1 +
.../org/apache/spark/ml/feature/Word2VecSuite.scala | 19 +++++++++++++++++++
9 files changed, 20 insertions(+)
diff --git a/mllib/src/test/resources/ml-models/w2v-4.0.0/data/._SUCCESS.crc
b/mllib/src/test/resources/ml-models/w2v-4.0.0/data/._SUCCESS.crc
new file mode 100644
index 000000000000..3b7b044936a8
Binary files /dev/null and
b/mllib/src/test/resources/ml-models/w2v-4.0.0/data/._SUCCESS.crc differ
diff --git
a/mllib/src/test/resources/ml-models/w2v-4.0.0/data/.part-00000-340fe065-7b1d-4a73-afb5-09d4f5461440-c000.snappy.parquet.crc
b/mllib/src/test/resources/ml-models/w2v-4.0.0/data/.part-00000-340fe065-7b1d-4a73-afb5-09d4f5461440-c000.snappy.parquet.crc
new file mode 100644
index 000000000000..134ec28210d6
Binary files /dev/null and
b/mllib/src/test/resources/ml-models/w2v-4.0.0/data/.part-00000-340fe065-7b1d-4a73-afb5-09d4f5461440-c000.snappy.parquet.crc
differ
diff --git a/mllib/src/test/resources/ml-models/w2v-4.0.0/data/_SUCCESS
b/mllib/src/test/resources/ml-models/w2v-4.0.0/data/_SUCCESS
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git
a/mllib/src/test/resources/ml-models/w2v-4.0.0/data/part-00000-340fe065-7b1d-4a73-afb5-09d4f5461440-c000.snappy.parquet
b/mllib/src/test/resources/ml-models/w2v-4.0.0/data/part-00000-340fe065-7b1d-4a73-afb5-09d4f5461440-c000.snappy.parquet
new file mode 100644
index 000000000000..1b7bb7d735bf
Binary files /dev/null and
b/mllib/src/test/resources/ml-models/w2v-4.0.0/data/part-00000-340fe065-7b1d-4a73-afb5-09d4f5461440-c000.snappy.parquet
differ
diff --git
a/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/._SUCCESS.crc
b/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/._SUCCESS.crc
new file mode 100644
index 000000000000..3b7b044936a8
Binary files /dev/null and
b/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/._SUCCESS.crc differ
diff --git
a/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/.part-00000-da33aa45-4cfc-4650-9739-09949b61d451-c000.txt.crc
b/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/.part-00000-da33aa45-4cfc-4650-9739-09949b61d451-c000.txt.crc
new file mode 100644
index 000000000000..63cd822431dc
Binary files /dev/null and
b/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/.part-00000-da33aa45-4cfc-4650-9739-09949b61d451-c000.txt.crc
differ
diff --git a/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/_SUCCESS
b/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/_SUCCESS
new file mode 100644
index 000000000000..e69de29bb2d1
diff --git
a/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/part-00000-da33aa45-4cfc-4650-9739-09949b61d451-c000.txt
b/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/part-00000-da33aa45-4cfc-4650-9739-09949b61d451-c000.txt
new file mode 100644
index 000000000000..f86a283a765f
--- /dev/null
+++
b/mllib/src/test/resources/ml-models/w2v-4.0.0/metadata/part-00000-da33aa45-4cfc-4650-9739-09949b61d451-c000.txt
@@ -0,0 +1 @@
+{"class":"org.apache.spark.ml.feature.Word2VecModel","timestamp":1793153464413,"sparkVersion":"4.0.0","uid":"Word2Vec_ff7f21470063","paramMap":{"numPartitions":1,"outputCol":"out","maxIter":3,"seed":42,"minCount":0,"vectorSize":4,"inputCol":"text"},"defaultParamMap":{"numPartitions":1,"windowSize":5,"maxSentenceLength":1000,"outputCol":"Word2Vec_ff7f21470063__output","maxIter":1,"seed":-3477684905227644199,"minCount":5,"vectorSize":100,"stepSize":0.025}}
diff --git
a/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
b/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
index e92a7380dcb4..42d0625529f1 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/feature/Word2VecSuite.scala
@@ -240,4 +240,23 @@ class Word2VecSuite extends MLTest with
DefaultReadWriteTest {
testTransformerByGlobalCheckFunc[(Seq[String], Seq[String])](ngramDF,
model, "result") { _ => }
}
+ test("Load Word2VecModel saved by Spark 4.0.0") {
+ val modelPath = testFile("ml-models/w2v-4.0.0")
+
+ val loadedModel = Word2VecModel.load(modelPath)
+ assert(loadedModel.getVectorSize === 4)
+ val vectors = loadedModel.getVectors.collect().map { row =>
+ row.getString(0) -> row.getAs[Vector](1).toArray
+ }.toMap
+ assert(vectors.keySet === Set("a", "cluster", "engine", "fast", "is",
"learning",
+ "machine", "on", "runs", "spark"))
+ assert(vectors("a") === Array(-0.04335077852010727, -0.10706974565982819,
+ -0.11131945252418518, 0.11506903171539307))
+ assert(vectors("spark") === Array(-0.007363073527812958,
0.09850209951400757,
+ -0.11606200784444809, -0.0377626046538353))
+
+ val metadata = spark.read.json(s"$modelPath/metadata")
+ val sparkVersionStr = metadata.select("sparkVersion").first().getString(0)
+ assert(sparkVersionStr === "4.0.0")
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
