[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user asfgit closed the pull request at: https://github.com/apache/spark/pull/20583 --- - To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167740107
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.schema("image").dataType == columnSchema, "data do not fit
ImageSchema")
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
--- End diff --
Thanks, done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user srowen commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167567793
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.schema("image").dataType == columnSchema, "data do not fit
ImageSchema")
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
--- End diff --
I think these kinds of assertions end up too brittle. I might focus on
asserting the message is about "sampleRatio" only or something
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167545871
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio = 0") {
+val df = readImages(imagePath, null, true, 3, true, 0.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: with sparkSession") {
+val df = readImages(imagePath, sparkSession = spark, true, 3, true,
1.0, 0)
--- End diff --
Can you check it? This PR not merge into branch2.3:
https://github.com/apache/spark/pull/20389
I fetch the code of branch 2.3 before.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167481154
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio = 0") {
+val df = readImages(imagePath, null, true, 3, true, 0.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: with sparkSession") {
+val df = readImages(imagePath, sparkSession = spark, true, 3, true,
1.0, 0)
--- End diff --
It should be. I try it now.
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167479724
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio = 0") {
+val df = readImages(imagePath, null, true, 3, true, 0.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: with sparkSession") {
+val df = readImages(imagePath, sparkSession = spark, true, 3, true,
1.0, 0)
+assert(df.count() === 7)
+ }
+
test("readImages partition test") {
val df = readImages(imagePath, null, true, 3, true, 1.0, 0)
assert(df.rdd.getNumPartitions === 3)
}
+ test("readImages partition test: < 0") {
+val df = readImages(imagePath, null, true, -3, true, 1.0, 0)
+assert(df.rdd.getNumPartitions ===
spark.sparkContext.defaultParallelism)
+ }
+
+ test("readImages partition test: = 0") {
+val df = readImages(imagePath, null, true, 0, true, 1.0, 0)
+assert(df.rdd.getNumPartitions != 0)
--- End diff --
Ok, done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167479658
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio = 0") {
+val df = readImages(imagePath, null, true, 3, true, 0.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: with sparkSession") {
+val df = readImages(imagePath, sparkSession = spark, true, 3, true,
1.0, 0)
+assert(df.count() === 7)
+ }
+
test("readImages partition test") {
val df = readImages(imagePath, null, true, 3, true, 1.0, 0)
assert(df.rdd.getNumPartitions === 3)
}
+ test("readImages partition test: < 0") {
+val df = readImages(imagePath, null, true, -3, true, 1.0, 0)
+assert(df.rdd.getNumPartitions ===
spark.sparkContext.defaultParallelism)
+ }
+
+ test("readImages partition test: = 0") {
--- End diff --
test
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user xubo245 commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167479566
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
--- End diff --
Ok, done
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user imatiach-msft commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167471442
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio = 0") {
+val df = readImages(imagePath, null, true, 3, true, 0.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: with sparkSession") {
+val df = readImages(imagePath, sparkSession = spark, true, 3, true,
1.0, 0)
--- End diff --
it looks like this test is failing in the latest build? I'm not sure why,
the error message says the count is 8. This test, which is similar, seems to
have a count of 7 ("readImages count test"):
df = readImages(imagePath, null, true, -1, true, 1.0, 0)
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user imatiach-msft commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167470547
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio = 0") {
+val df = readImages(imagePath, null, true, 3, true, 0.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: with sparkSession") {
+val df = readImages(imagePath, sparkSession = spark, true, 3, true,
1.0, 0)
+assert(df.count() === 7)
--- End diff --
I think this should be spark.sparkContext.defaultParallelism instead of 7
(?)
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user imatiach-msft commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167470380
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: sampleRatio > 1") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, 1.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio < 0") {
+val e = intercept[IllegalArgumentException] {
+ readImages(imagePath, null, true, 3, true, -0.1, 0)
+}
+assert(e.getMessage.equals("requirement failed: sampleRatio should be
between 0 and 1"))
+ }
+
+ test("readImages test: sampleRatio = 0") {
+val df = readImages(imagePath, null, true, 3, true, 0.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: with sparkSession") {
+val df = readImages(imagePath, sparkSession = spark, true, 3, true,
1.0, 0)
+assert(df.count() === 7)
+ }
+
test("readImages partition test") {
val df = readImages(imagePath, null, true, 3, true, 1.0, 0)
assert(df.rdd.getNumPartitions === 3)
}
+ test("readImages partition test: < 0") {
+val df = readImages(imagePath, null, true, -3, true, 1.0, 0)
+assert(df.rdd.getNumPartitions ===
spark.sparkContext.defaultParallelism)
+ }
+
+ test("readImages partition test: = 0") {
+val df = readImages(imagePath, null, true, 0, true, 1.0, 0)
+assert(df.rdd.getNumPartitions != 0)
--- End diff --
the check here seems redundant because of the additional check below, maybe
remove it (?)
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
[GitHub] spark pull request #20583: [SPARK-23392][TEST] Add some test cases for image...
Github user imatiach-msft commented on a diff in the pull request:
https://github.com/apache/spark/pull/20583#discussion_r167470275
--- Diff:
mllib/src/test/scala/org/apache/spark/ml/image/ImageSchemaSuite.scala ---
@@ -65,11 +65,71 @@ class ImageSchemaSuite extends SparkFunSuite with
MLlibTestSparkContext {
assert(count50 > 0 && count50 < countTotal)
}
+ test("readImages test: recursive = false") {
+val df = readImages(imagePath, null, false, 3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read jpg image") {
+val df = readImages(imagePath + "/kittens/DP153539.jpg", null, false,
3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read png image") {
+val df = readImages(imagePath + "/multi-channel/BGRA.png", null,
false, 3, true, 1.0, 0)
+assert(df.count() === 1)
+ }
+
+ test("readImages test: read non image") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, true, 1.0, 0)
+assert(df.count() === 0)
+ }
+
+ test("readImages test: read non image and dropImageFailures is false") {
+val df = readImages(imagePath + "/kittens/not-image.txt", null, false,
3, false, 1.0, 0)
+assert(df.count() === 1)
--- End diff --
(optional) you can also validate here that the entry has the correct
structure for an invalid image
---
-
To unsubscribe, e-mail: reviews-unsubscr...@spark.apache.org
For additional commands, e-mail: reviews-h...@spark.apache.org
