tillrohrmann commented on a change in pull request #18220: URL: https://github.com/apache/flink/pull/18220#discussion_r782229073
########## File path: flink-yarn/src/main/java/org/apache/flink/yarn/ApplicationReportProviderImpl.java ########## @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.util.FlinkException; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import java.io.IOException; + +/** The implementation of {@link ApplicationReportProvider}. */ +@Internal +public class ApplicationReportProviderImpl implements ApplicationReportProvider { + private final YarnClientRetriever yarnClientRetriever; + private final ApplicationId appId; + + private ApplicationReportProviderImpl( + YarnClientRetriever yarnClientRetriever, ApplicationId applicationId) { + this.yarnClientRetriever = yarnClientRetriever; + this.appId = applicationId; + } + + @Override + public ApplicationReport waitTillSubmissionFinish() throws FlinkException { + // A new yarn client need to be created in order to avoid the yarn client has been closed by + // YarnClusterDescriptor + try (final YarnClientRetriever retriever = yarnClientRetriever) { Review comment: I think it would be a bit nicer if we don't reuse the `yarnClientRetriever` after it has been closed (for subsequent calls). I have the mental model that a closed resource shouldn't be reused. ########## File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnClientRetrieverImpl.java ########## @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn; + +import org.apache.flink.annotation.Internal; + +import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.yarn.client.api.YarnClient.createYarnClient; + +/** + * The implementation of {@link YarnClientRetriever} which is used to get yarn client for {@link + * ApplicationReportProviderImpl}. When external yarn client is closed, it will create new yarn + * client that its lifecycle will be managed in this retriever. + */ +@Internal +public final class YarnClientRetrieverImpl implements YarnClientRetriever { + private static final Logger LOG = LoggerFactory.getLogger(YarnClientRetrieverImpl.class); + + private final YarnClient externalYarnClient; + private final YarnConfiguration yarnConfiguration; + private YarnClient createdYarnClient; Review comment: `@Nullable` ########## File path: flink-yarn-tests/src/test/java/org/apache/flink/yarn/YARNITCase.java ########## @@ -133,6 +140,42 @@ public void testPerJobWithArchive() throws Exception { runTest(() -> deployPerJob(flinkConfig, archiveJobGraph, true)); } + /** + * Once YARN app is accepted, it will return cluster client provider. Till invoking + * getClusterClient method, it will wait app state from accepted to running. + */ + @Test + public void testPerJobOnceAppAccepted() throws Exception { + runTest( + () -> { + Configuration configuration = + createDefaultConfiguration(YarnConfigOptions.UserJarInclusion.FIRST); + JobGraph jobGraph = getTestingJobGraph(); + jobGraph.setJobType(JobType.STREAMING); + try (final YarnClusterDescriptor yarnClusterDescriptor = + createYarnClusterDescriptor(configuration)) { + + ClusterClientProvider<ApplicationId> clusterClientProvider = + deployJobCluster(yarnClusterDescriptor, jobGraph); + + ApplicationId applicationId = yarnClusterDescriptor.getApplicationId(); + YarnClient yarnClient = getYarnClient(); + ApplicationReport report = yarnClient.getApplicationReport(applicationId); + + // Only test when app master state is accepted in YarnClusterDescriptor. + if (ACCEPTED == report.getYarnApplicationState()) { Review comment: Why can't the application state be in `RUNNING`? ########## File path: flink-yarn/src/main/java/org/apache/flink/yarn/ApplicationReportProviderImpl.java ########## @@ -0,0 +1,67 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn; + +import org.apache.flink.annotation.Internal; +import org.apache.flink.util.FlinkException; + +import org.apache.hadoop.yarn.api.records.ApplicationId; +import org.apache.hadoop.yarn.api.records.ApplicationReport; +import org.apache.hadoop.yarn.api.records.YarnApplicationState; +import org.apache.hadoop.yarn.exceptions.YarnException; + +import java.io.IOException; + +/** The implementation of {@link ApplicationReportProvider}. */ +@Internal +public class ApplicationReportProviderImpl implements ApplicationReportProvider { + private final YarnClientRetriever yarnClientRetriever; + private final ApplicationId appId; + + private ApplicationReportProviderImpl( + YarnClientRetriever yarnClientRetriever, ApplicationId applicationId) { + this.yarnClientRetriever = yarnClientRetriever; + this.appId = applicationId; + } + + @Override + public ApplicationReport waitTillSubmissionFinish() throws FlinkException { + // A new yarn client need to be created in order to avoid the yarn client has been closed by + // YarnClusterDescriptor Review comment: I think we can remove this comment. Otherwise we somehow link this implementation to the `YarnClusterDescriptor` that is not necessary. ########## File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnClientRetrieverImpl.java ########## @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn; + +import org.apache.flink.annotation.Internal; + +import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.yarn.client.api.YarnClient.createYarnClient; + +/** + * The implementation of {@link YarnClientRetriever} which is used to get yarn client for {@link + * ApplicationReportProviderImpl}. When external yarn client is closed, it will create new yarn + * client that its lifecycle will be managed in this retriever. + */ +@Internal +public final class YarnClientRetrieverImpl implements YarnClientRetriever { + private static final Logger LOG = LoggerFactory.getLogger(YarnClientRetrieverImpl.class); + + private final YarnClient externalYarnClient; + private final YarnConfiguration yarnConfiguration; + private YarnClient createdYarnClient; + + private YarnClientRetrieverImpl(YarnClient yarnClient, YarnConfiguration yarnConfiguration) { + this.externalYarnClient = yarnClient; + this.yarnConfiguration = yarnConfiguration; + } + + @Override + public YarnClient getYarnClient() { + if (externalYarnClient != null && !externalYarnClient.isInState(Service.STATE.STOPPED)) { + return externalYarnClient; + } + + if (createdYarnClient != null) { + return createdYarnClient; Review comment: What if this client is stopped? ########## File path: flink-yarn/src/main/java/org/apache/flink/yarn/YarnClientRetrieverImpl.java ########## @@ -0,0 +1,77 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.flink.yarn; + +import org.apache.flink.annotation.Internal; + +import org.apache.hadoop.service.Service; +import org.apache.hadoop.yarn.client.api.YarnClient; +import org.apache.hadoop.yarn.conf.YarnConfiguration; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import static org.apache.hadoop.yarn.client.api.YarnClient.createYarnClient; + +/** + * The implementation of {@link YarnClientRetriever} which is used to get yarn client for {@link + * ApplicationReportProviderImpl}. When external yarn client is closed, it will create new yarn + * client that its lifecycle will be managed in this retriever. + */ +@Internal +public final class YarnClientRetrieverImpl implements YarnClientRetriever { + private static final Logger LOG = LoggerFactory.getLogger(YarnClientRetrieverImpl.class); + + private final YarnClient externalYarnClient; + private final YarnConfiguration yarnConfiguration; + private YarnClient createdYarnClient; + + private YarnClientRetrieverImpl(YarnClient yarnClient, YarnConfiguration yarnConfiguration) { + this.externalYarnClient = yarnClient; + this.yarnConfiguration = yarnConfiguration; + } + + @Override + public YarnClient getYarnClient() { + if (externalYarnClient != null && !externalYarnClient.isInState(Service.STATE.STOPPED)) { Review comment: Can `externalYarnClient` ever be `null`? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
