This is an automated email from the ASF dual-hosted git repository.

dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git


The following commit(s) were added to refs/heads/master by this push:
     new 9942548  [SPARK-34487][K8S][TESTS] Use the runtime Hadoop version in 
K8s IT
9942548 is described below

commit 9942548c37ee6b08b6e29332c1e42407f4026fd3
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Sun Feb 21 08:57:02 2021 -0800

    [SPARK-34487][K8S][TESTS] Use the runtime Hadoop version in K8s IT
    
    ### What changes were proposed in this pull request?
    
    This PR aims to use the runtime Hadoop version in K8s integration test.
    
    ### Why are the changes needed?
    
    SPARK-33212 upgrades Hadoop dependency from 3.2.0 to 3.2.2 and we will 
upgrade to 3.3.x+.
    We had better use the runtime Hadoop version instead of having a static 
string.
    
    ### Does this PR introduce _any_ user-facing change?
    
    No.
    
    ### How was this patch tested?
    
    Pass the K8s IT.
    
    This is tested locally like the following.
    ```
    KubernetesSuite:
    ...
    - Launcher client dependencies
    - SPARK-33615: Launcher client archives
    - SPARK-33748: Launcher python client respecting PYSPARK_PYTHON
    - SPARK-33748: Launcher python client respecting spark.pyspark.python and 
spark.pyspark.driver.python
    - Launcher python client dependencies using a zip file
    ...
    ```
    
    Closes #31604 from dongjoon-hyun/SPARK-34487.
    
    Authored-by: Dongjoon Hyun <dh...@apple.com>
    Signed-off-by: Dongjoon Hyun <dh...@apple.com>
---
 .../org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala   | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git 
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
 
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
index c5c4007..812fd21 100644
--- 
a/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
+++ 
b/resource-managers/kubernetes/integration-tests/src/test/scala/org/apache/spark/deploy/k8s/integrationtest/DepsTestsSuite.scala
@@ -24,6 +24,7 @@ import com.amazonaws.auth.BasicAWSCredentials
 import com.amazonaws.services.s3.AmazonS3Client
 import io.fabric8.kubernetes.api.model._
 import io.fabric8.kubernetes.api.model.apps.StatefulSetBuilder
+import org.apache.hadoop.util.VersionInfo
 import org.scalatest.concurrent.{Eventually, PatienceConfiguration}
 import org.scalatest.time.{Minutes, Span}
 
@@ -294,7 +295,7 @@ private[spark] trait DepsTestsSuite { k8sSuite: 
KubernetesSuite =>
       minioUrlStr: String): Unit = {
     val (minioHost, minioPort) = getServiceHostAndPort(minioUrlStr)
     val packages = if (Utils.isHadoop3) {
-      "org.apache.hadoop:hadoop-aws:3.2.0"
+      s"org.apache.hadoop:hadoop-aws:${VersionInfo.getVersion}"
     } else {
       "com.amazonaws:aws-java-sdk:1.7.4,org.apache.hadoop:hadoop-aws:2.7.6"
     }


---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscr...@spark.apache.org
For additional commands, e-mail: commits-h...@spark.apache.org

Reply via email to