[
https://issues.apache.org/jira/browse/FLINK-17416?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17100700#comment-17100700
]
Robert Metzger edited comment on FLINK-17416 at 5/6/20, 11:37 AM:
------------------------------------------------------------------
Actually, we have the issue on Azure as well?
https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=674&view=logs&j=c88eea3b-64a0-564d-0031-9fdcd7b8abee&t=1e2bbe5b-4657-50be-1f07-d84bfce5b1f5
For better discoverability:
{code}
2020-05-06T09:43:38.0213229Z 2020-05-06 09:43:38,021 INFO
org.apache.flink.configuration.GlobalConfiguration [] - Loading
configuration property: jobmanager.execution.failover-strategy, region
2020-05-06T09:43:39.1410368Z 2020-05-06 09:43:38,949 ERROR
org.apache.flink.kubernetes.cli.KubernetesSessionCli [] - Error while
running the Flink session.
2020-05-06T09:43:39.1412057Z
io.fabric8.kubernetes.client.KubernetesClientException: Operation: [get] for
kind: [Service] with name: [flink-native-k8s-session-1-rest] in namespace:
[default] failed.
2020-05-06T09:43:39.1413357Z at
io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:64)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1414777Z at
io.fabric8.kubernetes.client.KubernetesClientException.launderThrowable(KubernetesClientException.java:72)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1416398Z at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:231)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1417559Z at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.get(BaseOperation.java:164)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1418948Z at
org.apache.flink.kubernetes.kubeclient.Fabric8FlinkKubeClient.getRestService(Fabric8FlinkKubeClient.java:201)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1420222Z at
org.apache.flink.kubernetes.cli.KubernetesSessionCli.run(KubernetesSessionCli.java:104)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1422055Z at
org.apache.flink.kubernetes.cli.KubernetesSessionCli.lambda$main$0(KubernetesSessionCli.java:185)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1423731Z at
org.apache.flink.runtime.security.contexts.NoOpSecurityContext.runSecured(NoOpSecurityContext.java:30)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1424868Z at
org.apache.flink.kubernetes.cli.KubernetesSessionCli.main(KubernetesSessionCli.java:185)
[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1425544Z Caused by: java.net.SocketException: Broken pipe
(Write failed)
2020-05-06T09:43:39.1426133Z at
java.net.SocketOutputStream.socketWrite0(Native Method) ~[?:1.8.0_252]
2020-05-06T09:43:39.1426567Z at
java.net.SocketOutputStream.socketWrite(SocketOutputStream.java:111)
~[?:1.8.0_252]
2020-05-06T09:43:39.1427091Z at
java.net.SocketOutputStream.write(SocketOutputStream.java:155) ~[?:1.8.0_252]
2020-05-06T09:43:39.1478206Z at
sun.security.ssl.OutputRecord.writeBuffer(OutputRecord.java:431) ~[?:1.8.0_252]
2020-05-06T09:43:39.1478717Z at
sun.security.ssl.OutputRecord.write(OutputRecord.java:417) ~[?:1.8.0_252]
2020-05-06T09:43:39.1479815Z at
sun.security.ssl.SSLSocketImpl.writeRecordInternal(SSLSocketImpl.java:894)
~[?:1.8.0_252]
2020-05-06T09:43:39.1480353Z at
sun.security.ssl.SSLSocketImpl.writeRecord(SSLSocketImpl.java:865)
~[?:1.8.0_252]
2020-05-06T09:43:39.1481024Z at
sun.security.ssl.AppOutputStream.write(AppOutputStream.java:123) ~[?:1.8.0_252]
2020-05-06T09:43:39.1482062Z at
org.apache.flink.kubernetes.shaded.okio.Okio$1.write(Okio.java:79)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1483535Z at
org.apache.flink.kubernetes.shaded.okio.AsyncTimeout$1.write(AsyncTimeout.java:180)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1484750Z at
org.apache.flink.kubernetes.shaded.okio.RealBufferedSink.flush(RealBufferedSink.java:224)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1485790Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http2.Http2Writer.settings(Http2Writer.java:203)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1486874Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http2.Http2Connection.start(Http2Connection.java:515)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1488261Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http2.Http2Connection.start(Http2Connection.java:505)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1490124Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.connection.RealConnection.startHttp2(RealConnection.java:298)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1491667Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.connection.RealConnection.establishProtocol(RealConnection.java:287)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1493131Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.connection.RealConnection.connect(RealConnection.java:168)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1494682Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.connection.StreamAllocation.findConnection(StreamAllocation.java:257)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1496363Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.connection.StreamAllocation.findHealthyConnection(StreamAllocation.java:135)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1497783Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.connection.StreamAllocation.newStream(StreamAllocation.java:114)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1499260Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.connection.ConnectInterceptor.intercept(ConnectInterceptor.java:42)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1501504Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1504529Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1506765Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1508818Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1510932Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1512523Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1515317Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1517038Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:126)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1518474Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1521417Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1522997Z at
io.fabric8.kubernetes.client.utils.BackwardsCompatibilityInterceptor.intercept(BackwardsCompatibilityInterceptor.java:119)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1524899Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1525970Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1527187Z at
io.fabric8.kubernetes.client.utils.ImpersonatorInterceptor.intercept(ImpersonatorInterceptor.java:68)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1530414Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1532488Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1533927Z at
io.fabric8.kubernetes.client.utils.HttpClientUtils.lambda$createHttpClient$3(HttpClientUtils.java:112)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1535766Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1536906Z at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1539160Z at
org.apache.flink.kubernetes.shaded.okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:254)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1540772Z at
org.apache.flink.kubernetes.shaded.okhttp3.RealCall.execute(RealCall.java:92)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1542910Z at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:411)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1545053Z at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:372)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1546526Z at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:337)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1547709Z at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:318)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1549744Z at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:812)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1551119Z at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:220)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
2020-05-06T09:43:39.1551712Z ... 6 more
{code}
was (Author: rmetzger):
Actually, we have the issue on Azure as well?
https://dev.azure.com/apache-flink/apache-flink/_build/results?buildId=674&view=logs&j=c88eea3b-64a0-564d-0031-9fdcd7b8abee&t=1e2bbe5b-4657-50be-1f07-d84bfce5b1f5
For better discoverability:
{code}
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.cache.CacheInterceptor.intercept(CacheInterceptor.java:93)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.BridgeInterceptor.intercept(BridgeInterceptor.java:93)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RetryAndFollowUpInterceptor.intercept(RetryAndFollowUpInterceptor.java:126)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.utils.BackwardsCompatibilityInterceptor.intercept(BackwardsCompatibilityInterceptor.java:119)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.utils.ImpersonatorInterceptor.intercept(ImpersonatorInterceptor.java:68)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.utils.HttpClientUtils.lambda$createHttpClient$3(HttpClientUtils.java:112)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:147)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.internal.http.RealInterceptorChain.proceed(RealInterceptorChain.java:121)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.RealCall.getResponseWithInterceptorChain(RealCall.java:254)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
org.apache.flink.kubernetes.shaded.okhttp3.RealCall.execute(RealCall.java:92)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:411)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleResponse(OperationSupport.java:372)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:337)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.OperationSupport.handleGet(OperationSupport.java:318)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.handleGet(BaseOperation.java:812)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
at
io.fabric8.kubernetes.client.dsl.base.BaseOperation.getMandatory(BaseOperation.java:220)
~[flink-dist_2.11-1.11-SNAPSHOT.jar:1.11-SNAPSHOT]
... 6 more
{code}
> Flink-kubernetes doesn't work on java 8 8u252
> ---------------------------------------------
>
> Key: FLINK-17416
> URL: https://issues.apache.org/jira/browse/FLINK-17416
> Project: Flink
> Issue Type: Bug
> Components: Deployment / Kubernetes
> Affects Versions: 1.10.0
> Reporter: wangxiyuan
> Priority: Minor
>
> When using java-8-8u252 version, the flink container end-to-end failed. The
> test `Running 'Run kubernetes session test'` fails with the `Broken pipe`
> error.
> See:
> [https://logs.openlabtesting.org/logs/periodic-20-flink-mail/github.com/apache/flink/master/flink-end-to-end-test-arm64-container/fcfdd47/job-output.txt.gz]
>
> Flink Azure CI doesn't hit this problem because it runs under jdk-8-8u242
>
> The reason is that the okhttp library which flink using doesn't work on
> java-8-8u252:
> [https://github.com/square/okhttp/issues/5970]
>
> The problem has been with the PR:
> [https://github.com/square/okhttp/pull/5977]
>
> Maybe we can wait for a new 3.12.x release and bump the okhttp version in
> Flink later.
>
>
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
