[ https://issues.apache.org/jira/browse/SPARK-24227?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Felipe Cavalcanti updated SPARK-24227: -------------------------------------- Description: Hi, I'm trying to submit a spark job to kubernetes with no success, I followed the steps @ [https://spark.apache.org/docs/latest/running-on-kubernetes.html] with no success, when I run: {code:java} bin/spark-submit \ --master k8s://https://${host}:${port} \ --deploy-mode cluster \ --name jaeger-spark \ --class io.jaegertracing.spark.dependencies.DependenciesSparkJob \ --conf spark.executor.instances=5 \ --conf spark.kubernetes.container.image=bla/jaeger-deps-spark:latest\ --conf spark.kubernetes.namespace=spark \ local:///opt/spark/jars/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar {code} Im getting the following stack trace: {code:java} 2018-05-09 17:06:02 WARN WatchConnectionManager:192 - Exec Failure javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.ssl.Alerts.getSSLException(Alerts.java:192) at sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1949) at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:302) at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:296) at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1514) at sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:216) at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1026) at sun.security.ssl.Handshaker.process_record(Handshaker.java:961) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1062) at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387) at okhttp3.internal.connection.RealConnection.connectTls(RealConnection.java:281) at okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:251) at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:151) at okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:195) at okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:121) at okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:100) at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at io.fabric8.kubernetes.client.utils.HttpClientUtils$2.intercept(HttpClientUtils.java:90) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:135) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:387) at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:292) at sun.security.validator.Validator.validate(Validator.java:260) at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) ... 33 more Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141) at sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126) at java.security.cert.CertPathBuilder.build(CertPathBuilder.java:280) at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:382) ... 39 more Exception in thread "main" io.fabric8.kubernetes.client.KubernetesClientException: Failed to start websocket at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2.onFailure(WatchConnectionManager.java:194) at okhttp3.internal.ws.RealWebSocket.failWebSocket(RealWebSocket.java:543) at okhttp3.internal.ws.RealWebSocket$2.onFailure(RealWebSocket.java:208) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:148) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Caused by: javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.ssl.Alerts.getSSLException(Alerts.java:192) at sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1949) at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:302) at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:296) at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1514) at sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:216) at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1026) at sun.security.ssl.Handshaker.process_record(Handshaker.java:961) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1062) at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387) at okhttp3.internal.connection.RealConnection.connectTls(RealConnection.java:281) at okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:251) at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:151) at okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:195) at okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:121) at okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:100) at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at io.fabric8.kubernetes.client.utils.HttpClientUtils$2.intercept(HttpClientUtils.java:90) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:135) ... 4 more Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:387) at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:292) at sun.security.validator.Validator.validate(Validator.java:260) at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) ... 33 more Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141) at sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126) at java.security.cert.CertPathBuilder.build(CertPathBuilder.java:280) at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:382) ... 39 more 2018-05-09 17:06:02 INFO ShutdownHookManager:54 - Shutdown hook called 2018-05-09 17:06:02 INFO ShutdownHookManager:54 - Deleting directory /private/var/folders/w1/xrzml0dx4k77kc68x1pjbj0h0000gn/T/spark-0aa4041a-b65b-445f-a129-fd2983ff1cc4 {code} Because I thought it's a unsecure certificate error, I tried again by proxying kube-api: {code:java} kubectl proxy {code} and then: {code:java} bin/spark-submit \ --master k8s://http://localhost:8001 \ --deploy-mode cluster \ --name jaeger-spark \ --class io.jaegertracing.spark.dependencies.DependenciesSparkJob \ --conf spark.executor.instances=5 \ --conf spark.kubernetes.container.image=bla/jaeger-deps-spark:latest\ --conf spark.kubernetes.namespace=spark \ local:///opt/spark/jars/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar {code} but then I get this stack trace: {code:java} 2018-05-09 17:28:16 WARN Utils:66 - Kubernetes master URL uses HTTP instead of HTTPS. 2018-05-09 17:28:19 WARN WatchConnectionManager:185 - Exec Failure: HTTP 401, Status: 401 - Unauthorized java.net.ProtocolException: Expected HTTP 101 response but was '401 Unauthorized' at okhttp3.internal.ws.RealWebSocket.checkResponse(RealWebSocket.java:216) at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:183) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:141) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Exception in thread "main" io.fabric8.kubernetes.client.KubernetesClientException: Unauthorized at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2.onFailure(WatchConnectionManager.java:188) at okhttp3.internal.ws.RealWebSocket.failWebSocket(RealWebSocket.java:543) at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:185) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:141) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 2018-05-09 17:28:19 INFO ShutdownHookManager:54 - Shutdown hook called 2018-05-09 17:28:19 INFO ShutdownHookManager:54 - Deleting directory /private/var/folders/w1/xrzml0dx4k77kc68x1pjbj0h0000gn/T/spark-7ff928c3-b4a0-4ce3-aba2-56a50d341b97 {code} I run kubernetes on AWS and brought the cluster up with Kops, I have RBAC enabled with oid2 auth. was: Hi, I'm trying to submit a spark job to kubernetes with no success, I followed the steps @ [https://spark.apache.org/docs/latest/running-on-kubernetes.html] with no success, when I run: {code:java} bin/spark-submit \ --master k8s://https://${host}:${port} \ --deploy-mode cluster \ --name jaeger-spark \ --class io.jaegertracing.spark.dependencies.DependenciesSparkJob \ --conf spark.executor.instances=5 \ --conf spark.kubernetes.container.image=bla/jaeger-deps-spark:latest\ --conf spark.kubernetes.namespace=spark \ local:///opt/spark/jars/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar {code} Im getting the following stack trace: {code:java} 2018-05-09 17:06:02 WARN WatchConnectionManager:192 - Exec Failure javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.ssl.Alerts.getSSLException(Alerts.java:192) at sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1949) at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:302) at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:296) at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1514) at sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:216) at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1026) at sun.security.ssl.Handshaker.process_record(Handshaker.java:961) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1062) at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387) at okhttp3.internal.connection.RealConnection.connectTls(RealConnection.java:281) at okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:251) at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:151) at okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:195) at okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:121) at okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:100) at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at io.fabric8.kubernetes.client.utils.HttpClientUtils$2.intercept(HttpClientUtils.java:90) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:135) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:387) at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:292) at sun.security.validator.Validator.validate(Validator.java:260) at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) ... 33 more Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141) at sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126) at java.security.cert.CertPathBuilder.build(CertPathBuilder.java:280) at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:382) ... 39 more Exception in thread "main" io.fabric8.kubernetes.client.KubernetesClientException: Failed to start websocket at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2.onFailure(WatchConnectionManager.java:194) at okhttp3.internal.ws.RealWebSocket.failWebSocket(RealWebSocket.java:543) at okhttp3.internal.ws.RealWebSocket$2.onFailure(RealWebSocket.java:208) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:148) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Caused by: javax.net.ssl.SSLHandshakeException: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.ssl.Alerts.getSSLException(Alerts.java:192) at sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1949) at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:302) at sun.security.ssl.Handshaker.fatalSE(Handshaker.java:296) at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1514) at sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:216) at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1026) at sun.security.ssl.Handshaker.process_record(Handshaker.java:961) at sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1062) at sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403) at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387) at okhttp3.internal.connection.RealConnection.connectTls(RealConnection.java:281) at okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:251) at okhttp3.internal.connection.RealConnection.connect(RealConnection.java:151) at okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:195) at okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:121) at okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:100) at okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at io.fabric8.kubernetes.client.utils.HttpClientUtils$2.intercept(HttpClientUtils.java:90) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) at okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:135) ... 4 more Caused by: sun.security.validator.ValidatorException: PKIX path building failed: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:387) at sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:292) at sun.security.validator.Validator.validate(Validator.java:260) at sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) at sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) at sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) at sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) ... 33 more Caused by: sun.security.provider.certpath.SunCertPathBuilderException: unable to find valid certification path to requested target at sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141) at sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126) at java.security.cert.CertPathBuilder.build(CertPathBuilder.java:280) at sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:382) ... 39 more 2018-05-09 17:06:02 INFO ShutdownHookManager:54 - Shutdown hook called 2018-05-09 17:06:02 INFO ShutdownHookManager:54 - Deleting directory /private/var/folders/w1/xrzml0dx4k77kc68x1pjbj0h0000gn/T/spark-0aa4041a-b65b-445f-a129-fd2983ff1cc4 {code} Because I thought it's a unsecure certificate error, I tried again by proxying kube-proxy: {code:java} kubectl proxy {code} and then: {code:java} bin/spark-submit \ --master k8s://http://localhost:8001 \ --deploy-mode cluster \ --name jaeger-spark \ --class io.jaegertracing.spark.dependencies.DependenciesSparkJob \ --conf spark.executor.instances=5 \ --conf spark.kubernetes.container.image=bla/jaeger-deps-spark:latest\ --conf spark.kubernetes.namespace=spark \ local:///opt/spark/jars/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar {code} but then I get this stack trace: {code:java} 2018-05-09 17:28:16 WARN Utils:66 - Kubernetes master URL uses HTTP instead of HTTPS. 2018-05-09 17:28:19 WARN WatchConnectionManager:185 - Exec Failure: HTTP 401, Status: 401 - Unauthorized java.net.ProtocolException: Expected HTTP 101 response but was '401 Unauthorized' at okhttp3.internal.ws.RealWebSocket.checkResponse(RealWebSocket.java:216) at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:183) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:141) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) Exception in thread "main" io.fabric8.kubernetes.client.KubernetesClientException: Unauthorized at io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2.onFailure(WatchConnectionManager.java:188) at okhttp3.internal.ws.RealWebSocket.failWebSocket(RealWebSocket.java:543) at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:185) at okhttp3.RealCall$AsyncCall.execute(RealCall.java:141) at okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) at java.lang.Thread.run(Thread.java:748) 2018-05-09 17:28:19 INFO ShutdownHookManager:54 - Shutdown hook called 2018-05-09 17:28:19 INFO ShutdownHookManager:54 - Deleting directory /private/var/folders/w1/xrzml0dx4k77kc68x1pjbj0h0000gn/T/spark-7ff928c3-b4a0-4ce3-aba2-56a50d341b97 {code} I run kubernetes on AWS and brought the cluster up with Kops, I have RBAC enabled with oid2 auth. > Not able to submit spark job to kubernetes on 2.3 > ------------------------------------------------- > > Key: SPARK-24227 > URL: https://issues.apache.org/jira/browse/SPARK-24227 > Project: Spark > Issue Type: Bug > Components: Spark Core, Spark Submit > Affects Versions: 2.3.0 > Reporter: Felipe Cavalcanti > Priority: Major > Labels: kubernetes, spark > > Hi, I'm trying to submit a spark job to kubernetes with no success, I > followed the steps @ > [https://spark.apache.org/docs/latest/running-on-kubernetes.html] with no > success, when I run: > > {code:java} > bin/spark-submit \ > --master k8s://https://${host}:${port} \ > --deploy-mode cluster \ > --name jaeger-spark \ > --class io.jaegertracing.spark.dependencies.DependenciesSparkJob \ > --conf spark.executor.instances=5 \ > --conf spark.kubernetes.container.image=bla/jaeger-deps-spark:latest\ > --conf spark.kubernetes.namespace=spark \ > local:///opt/spark/jars/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar > {code} > > Im getting the following stack trace: > {code:java} > 2018-05-09 17:06:02 WARN WatchConnectionManager:192 - Exec Failure > javax.net.ssl.SSLHandshakeException: > sun.security.validator.ValidatorException: PKIX path building failed: > sun.security.provider.certpath.SunCertPathBuilderException: unable to find > valid certification path to requested target at > sun.security.ssl.Alerts.getSSLException(Alerts.java:192) at > sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1949) at > sun.security.ssl.Handshaker.fatalSE(Handshaker.java:302) at > sun.security.ssl.Handshaker.fatalSE(Handshaker.java:296) at > sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1514) > at > sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:216) > at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1026) at > sun.security.ssl.Handshaker.process_record(Handshaker.java:961) at > sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1062) at > sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375) > at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403) at > sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387) at > okhttp3.internal.connection.RealConnection.connectTls(RealConnection.java:281) > at > okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:251) > at > okhttp3.internal.connection.RealConnection.connect(RealConnection.java:151) > at > okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:195) > at > okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:121) > at > okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:100) > at > okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) > at > okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) > at > okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) > at > io.fabric8.kubernetes.client.utils.HttpClientUtils$2.intercept(HttpClientUtils.java:90) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) > at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185) at > okhttp3.RealCall$AsyncCall.execute(RealCall.java:135) at > okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:748) Caused by: > sun.security.validator.ValidatorException: PKIX path building failed: > sun.security.provider.certpath.SunCertPathBuilderException: unable to find > valid certification path to requested target at > sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:387) at > sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:292) > at sun.security.validator.Validator.validate(Validator.java:260) at > sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) > at > sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) > at > sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) > at > sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) > ... 33 more Caused by: > sun.security.provider.certpath.SunCertPathBuilderException: unable to find > valid certification path to requested target at > sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141) > at > sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126) > at java.security.cert.CertPathBuilder.build(CertPathBuilder.java:280) at > sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:382) ... 39 > more Exception in thread "main" > io.fabric8.kubernetes.client.KubernetesClientException: Failed to start > websocket at > io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2.onFailure(WatchConnectionManager.java:194) > at okhttp3.internal.ws.RealWebSocket.failWebSocket(RealWebSocket.java:543) > at okhttp3.internal.ws.RealWebSocket$2.onFailure(RealWebSocket.java:208) at > okhttp3.RealCall$AsyncCall.execute(RealCall.java:148) at > okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:748) Caused by: > javax.net.ssl.SSLHandshakeException: > sun.security.validator.ValidatorException: PKIX path building failed: > sun.security.provider.certpath.SunCertPathBuilderException: unable to find > valid certification path to requested target at > sun.security.ssl.Alerts.getSSLException(Alerts.java:192) at > sun.security.ssl.SSLSocketImpl.fatal(SSLSocketImpl.java:1949) at > sun.security.ssl.Handshaker.fatalSE(Handshaker.java:302) at > sun.security.ssl.Handshaker.fatalSE(Handshaker.java:296) at > sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1514) > at > sun.security.ssl.ClientHandshaker.processMessage(ClientHandshaker.java:216) > at sun.security.ssl.Handshaker.processLoop(Handshaker.java:1026) at > sun.security.ssl.Handshaker.process_record(Handshaker.java:961) at > sun.security.ssl.SSLSocketImpl.readRecord(SSLSocketImpl.java:1062) at > sun.security.ssl.SSLSocketImpl.performInitialHandshake(SSLSocketImpl.java:1375) > at sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1403) at > sun.security.ssl.SSLSocketImpl.startHandshake(SSLSocketImpl.java:1387) at > okhttp3.internal.connection.RealConnection.connectTls(RealConnection.java:281) > at > okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:251) > at > okhttp3.internal.connection.RealConnection.connect(RealConnection.java:151) > at > okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:195) > at > okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:121) > at > okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:100) > at > okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) > at > okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) > at > okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:120) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) > at > io.fabric8.kubernetes.client.utils.HttpClientUtils$2.intercept(HttpClientUtils.java:90) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:92) > at > okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:67) > at okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:185) at > okhttp3.RealCall$AsyncCall.execute(RealCall.java:135) ... 4 more Caused by: > sun.security.validator.ValidatorException: PKIX path building failed: > sun.security.provider.certpath.SunCertPathBuilderException: unable to find > valid certification path to requested target at > sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:387) at > sun.security.validator.PKIXValidator.engineValidate(PKIXValidator.java:292) > at sun.security.validator.Validator.validate(Validator.java:260) at > sun.security.ssl.X509TrustManagerImpl.validate(X509TrustManagerImpl.java:324) > at > sun.security.ssl.X509TrustManagerImpl.checkTrusted(X509TrustManagerImpl.java:229) > at > sun.security.ssl.X509TrustManagerImpl.checkServerTrusted(X509TrustManagerImpl.java:124) > at > sun.security.ssl.ClientHandshaker.serverCertificate(ClientHandshaker.java:1496) > ... 33 more Caused by: > sun.security.provider.certpath.SunCertPathBuilderException: unable to find > valid certification path to requested target at > sun.security.provider.certpath.SunCertPathBuilder.build(SunCertPathBuilder.java:141) > at > sun.security.provider.certpath.SunCertPathBuilder.engineBuild(SunCertPathBuilder.java:126) > at java.security.cert.CertPathBuilder.build(CertPathBuilder.java:280) at > sun.security.validator.PKIXValidator.doBuild(PKIXValidator.java:382) ... 39 > more 2018-05-09 17:06:02 INFO ShutdownHookManager:54 - Shutdown hook called > 2018-05-09 17:06:02 INFO ShutdownHookManager:54 - Deleting directory > /private/var/folders/w1/xrzml0dx4k77kc68x1pjbj0h0000gn/T/spark-0aa4041a-b65b-445f-a129-fd2983ff1cc4 > {code} > > Because I thought it's a unsecure certificate error, I tried again by > proxying kube-api: > > {code:java} > kubectl proxy > {code} > and then: > {code:java} > bin/spark-submit \ > --master k8s://http://localhost:8001 \ > --deploy-mode cluster \ > --name jaeger-spark \ > --class io.jaegertracing.spark.dependencies.DependenciesSparkJob \ > --conf spark.executor.instances=5 \ > --conf spark.kubernetes.container.image=bla/jaeger-deps-spark:latest\ > --conf spark.kubernetes.namespace=spark \ > local:///opt/spark/jars/jaeger-spark-dependencies-0.0.1-SNAPSHOT.jar > {code} > > but then I get this stack trace: > > {code:java} > 2018-05-09 17:28:16 WARN Utils:66 - Kubernetes master URL uses HTTP instead > of HTTPS. 2018-05-09 17:28:19 WARN WatchConnectionManager:185 - Exec Failure: > HTTP 401, Status: 401 - Unauthorized java.net.ProtocolException: Expected > HTTP 101 response but was '401 Unauthorized' at > okhttp3.internal.ws.RealWebSocket.checkResponse(RealWebSocket.java:216) at > okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:183) at > okhttp3.RealCall$AsyncCall.execute(RealCall.java:141) at > okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:748) Exception in thread "main" > io.fabric8.kubernetes.client.KubernetesClientException: Unauthorized at > io.fabric8.kubernetes.client.dsl.internal.WatchConnectionManager$2.onFailure(WatchConnectionManager.java:188) > at okhttp3.internal.ws.RealWebSocket.failWebSocket(RealWebSocket.java:543) > at okhttp3.internal.ws.RealWebSocket$2.onResponse(RealWebSocket.java:185) at > okhttp3.RealCall$AsyncCall.execute(RealCall.java:141) at > okhttp3.internal.NamedRunnable.run(NamedRunnable.java:32) at > java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142) > at > java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617) > at java.lang.Thread.run(Thread.java:748) 2018-05-09 17:28:19 INFO > ShutdownHookManager:54 - Shutdown hook called 2018-05-09 17:28:19 INFO > ShutdownHookManager:54 - Deleting directory > /private/var/folders/w1/xrzml0dx4k77kc68x1pjbj0h0000gn/T/spark-7ff928c3-b4a0-4ce3-aba2-56a50d341b97 > {code} > I run kubernetes on AWS and brought the cluster up with Kops, I have RBAC > enabled with oid2 auth. -- This message was sent by Atlassian JIRA (v7.6.3#76005) --------------------------------------------------------------------- To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org For additional commands, e-mail: issues-h...@spark.apache.org