AngersZhuuuu commented on a change in pull request #29966:
URL: https://github.com/apache/spark/pull/29966#discussion_r533872086
##########
File path: core/src/test/scala/org/apache/spark/SparkContextSuite.scala
##########
@@ -955,6 +978,121 @@ class SparkContextSuite extends SparkFunSuite with
LocalSparkContext with Eventu
.set(EXECUTOR_ALLOW_SPARK_CONTEXT, true)).stop()
}
}
+
+ test("SPARK-33084: Add jar support ivy url -- default transitive = false") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+
assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")
+
assert(sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+ }
+
+ test("SPARK-33084: Add jar support ivy url -- invalid transitive use default
false") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=foo")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+ assert(!sc.listJars().exists(_.contains("org.slf4j_slf4j-api-1.7.10.jar")))
+
assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+ }
+
+ test("SPARK-33084: Add jar support ivy url -- transitive=true will download
dependency jars") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+ assert(sc.listJars().exists(_.contains("org.slf4j_slf4j-api-1.7.10.jar")))
+
assert(sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+ }
+
+ test("SPARK-33084: Add jar support ivy url -- test exclude param when
transitive=true") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0" +
+ "?exclude=commons-lang:commons-lang&transitive=true")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+ assert(sc.listJars().exists(_.contains("org.slf4j_slf4j-api-1.7.10.jar")))
+
assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+ }
+
+ test("SPARK-33084: Add jar support ivy url -- test different version") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.6.0")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.6.0.jar")))
+ }
+
+ test("SPARK-33084: Add jar support ivy url -- test invalid param") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?invalidParam=foo")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+ }
+
+ test("SPARK-33084: Add jar support ivy url -- test multiple transitive
params") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?" +
+ "transitive=true&transitive=false&transitive=invalidValue")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+
assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?" +
+ "transitive=false&transitive=invalidValue&transitive=true")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+
assert(sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+ }
+
+ test("SPARK-33084: Add jar support ivy url -- test param case sensitive") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?TRANSITIVE=true")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+
assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+
assert(sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+ }
+
+ test("SPARK-33084: Add jar support ivy url -- test transitive value case
sensitive") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=TRUE")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+
assert(!sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+
+ sc.addJar("ivy://org.apache.hive:hive-storage-api:2.7.0?transitive=true")
+
assert(sc.listJars().exists(_.contains("org.apache.hive_hive-storage-api-2.7.0.jar")))
+
assert(sc.listJars().exists(_.contains("commons-lang_commons-lang-2.6.jar")))
+ }
+
+
+ test("SPARK-33084: Add jar support ivy url -- test invalid ivy URI") {
+ sc = new SparkContext(new
SparkConf().setAppName("test").setMaster("local-cluster[3, 1, 1024]"))
+ val e1 = intercept[IllegalArgumentException] {
+ sc.addJar("ivy://")
+ }.getMessage
+ assert(e1.contains("Expected authority at index 6: ivy://"))
+
+ val e2 = intercept[URISyntaxException] {
+ sc.addJar("ivy://org.apache.hive:hive-contrib")
+ }.getMessage
+ assert(e2.contains("Invalid url: Expected 'org:module:version'," +
+ " found org.apache.hive:hive-contrib"))
+
+ val e3 = intercept[URISyntaxException] {
+ sc.addJar("ivy://org.apache.hive:hive-contrib:2.3.7?foo=")
+ }.getMessage
+ assert(e3.contains("Invalid query string: foo="))
+
+ val e4 = intercept[URISyntaxException] {
+ sc.addJar("ivy://org.apache.hive:hive-contrib:2.3.7?bar=&baz=foo")
+ }.getMessage
+ assert(e4.contains("Invalid query string: bar=&baz=foo"))
+
+ val e5 = intercept[URISyntaxException] {
Review comment:
> IllegalArgumentException
Yea, catch and re-throw updated
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
