zentol commented on a change in pull request #16286:
URL: https://github.com/apache/flink/pull/16286#discussion_r663778409
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/deployment/application/FromJarEntryClassInformationProvider.java
##########
@@ -68,11 +68,33 @@ private FromJarEntryClassInformationProvider(
this.jobClassName = jobClassName;
}
+ /**
+ * Returns the specified {@code jarFile}. If no {@code jarFile} was
specified it can only return
+ * an empty {@code Optional} if the {@code jobClassName} is specified
instead. In that case, the
+ * caller should assume that the passed job class can be found on the
system classpath.
+ *
+ * @return The specified {@code jarFile}. If returning an empty {@code
Optional} the caller
+ * should assume to find the job class on the system classpath.
+ * @see #getJobClassName()
+ */
@Override
public Optional<File> getJarFile() {
return Optional.ofNullable(jarFile);
}
+ /**
+ * Returns the specified job class name that is either available in the
corresponding {@code
+ * jarFile} or, if the {@code jarFile} is not set, should be assumed to be
contained in the
+ * system classpath. It can return an emtpy {@code Optional} if the
respective {@code jarFile}
Review comment:
emtpy -> empty
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/deployment/application/FromJarEntryClassInformationProvider.java
##########
@@ -68,11 +68,33 @@ private FromJarEntryClassInformationProvider(
this.jobClassName = jobClassName;
}
+ /**
+ * Returns the specified {@code jarFile}. If no {@code jarFile} was
specified it can only return
+ * an empty {@code Optional} if the {@code jobClassName} is specified
instead. In that case, the
Review comment:
The condition when it returns an empty optional is unnecessary and just
creates confusion. The jar _and_ classname not being specified is a case that a
consumer of this APIn ever has to be concerned with because the constructor
rejects it.
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/deployment/application/FromJarEntryClassInformationProvider.java
##########
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.deployment.application;
+
+import org.apache.flink.client.program.PackagedProgramUtils;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.util.Optional;
+
+/**
+ * {@code FromJarEntryClassInformationProvider} is used for cases where the
Jar archive is
+ * explicitly specified.
Review comment:
The jar doesn't have to be explicitly specified.
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/deployment/application/FromJarEntryClassInformationProvider.java
##########
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.deployment.application;
+
+import org.apache.flink.client.program.PackagedProgramUtils;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.util.Optional;
+
+/**
+ * {@code FromJarEntryClassInformationProvider} is used for cases where the
Jar archive is
+ * explicitly specified.
+ */
+public class FromJarEntryClassInformationProvider implements
EntryClassInformationProvider {
+
+ private final File jarFile;
+ private final String jobClassName;
+
+ /**
+ * Creates a {@code FromJarEntryClassInformationProvider} for a custom Jar
archive. At least the
+ * {@code jarFile} or the {@code jobClassName} has to be set.
+ *
+ * @param jarFile The Jar archive.
+ * @param jobClassName The name of the job class.
+ * @return The {@code FromJarEntryClassInformationProvider} referring to
the passed information.
+ */
+ public static FromJarEntryClassInformationProvider createFromCustomJar(
+ @Nullable File jarFile, @Nullable String jobClassName) {
+ return new FromJarEntryClassInformationProvider(jarFile, jobClassName);
+ }
+
+ /**
+ * Creates a {@code FromJarEntryClassInformationProvider} for a job
implemented in Python.
+ *
+ * @return A {@code FromJarEntryClassInformationProvider} for a job
implemented in Python
+ */
+ public static FromJarEntryClassInformationProvider createFromPythonJar() {
+ return new FromJarEntryClassInformationProvider(
+ new File(PackagedProgramUtils.getPythonJar().getPath()),
+ PackagedProgramUtils.getPythonDriverClassName());
+ }
+
+ private FromJarEntryClassInformationProvider(
+ @Nullable File jarFile, @Nullable String jobClassName) {
+ Preconditions.checkArgument(
+ jarFile != null || jobClassName != null,
+ "Either the jar file or the job class name, or both need to be
set.");
+ this.jarFile = jarFile;
+ this.jobClassName = jobClassName;
+ }
+
+ /**
+ * Returns the specified {@code jarFile}. If no {@code jarFile} was
specified it can only return
+ * an empty {@code Optional} if the {@code jobClassName} is specified
instead. In that case, the
+ * caller should assume that the passed job class can be found on the
system classpath.
+ *
+ * @return The specified {@code jarFile}. If returning an empty {@code
Optional} the caller
+ * should assume to find the job class on the system classpath.
+ * @see #getJobClassName()
+ */
+ @Override
+ public Optional<File> getJarFile() {
+ return Optional.ofNullable(jarFile);
+ }
+
+ /**
+ * Returns the specified job class name that is either available in the
corresponding {@code
+ * jarFile} or, if the {@code jarFile} is not set, should be assumed to be
contained in the
+ * system classpath. It can return an emtpy {@code Optional} if the
respective {@code jarFile}
+ * is set. In that case one should assume the job class being the entry
class of the respective
Review comment:
```suggestion
* system classpath. It can return an empty {@code Optional} despite the
respective {@code jarFile}
* being set if the job class is the entry class of the jar.
```
The descriptions are bit difficult to understand :/
Note also that this behavior is super weird and we should just return the
entry class imo.
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/deployment/application/FromJarEntryClassInformationProvider.java
##########
@@ -0,0 +1,102 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.deployment.application;
+
+import org.apache.flink.client.program.PackagedProgramUtils;
+import org.apache.flink.util.Preconditions;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.util.Optional;
+
+/**
+ * {@code FromJarEntryClassInformationProvider} is used for cases where the
Jar archive is
+ * explicitly specified.
+ */
+public class FromJarEntryClassInformationProvider implements
EntryClassInformationProvider {
+
+ private final File jarFile;
+ private final String jobClassName;
+
+ /**
+ * Creates a {@code FromJarEntryClassInformationProvider} for a custom Jar
archive. At least the
+ * {@code jarFile} or the {@code jobClassName} has to be set.
+ *
+ * @param jarFile The Jar archive.
+ * @param jobClassName The name of the job class.
+ * @return The {@code FromJarEntryClassInformationProvider} referring to
the passed information.
+ */
+ public static FromJarEntryClassInformationProvider createFromCustomJar(
+ @Nullable File jarFile, @Nullable String jobClassName) {
+ return new FromJarEntryClassInformationProvider(jarFile, jobClassName);
+ }
+
+ /**
+ * Creates a {@code FromJarEntryClassInformationProvider} for a job
implemented in Python.
+ *
+ * @return A {@code FromJarEntryClassInformationProvider} for a job
implemented in Python
+ */
+ public static FromJarEntryClassInformationProvider createFromPythonJar() {
+ return new FromJarEntryClassInformationProvider(
+ new File(PackagedProgramUtils.getPythonJar().getPath()),
+ PackagedProgramUtils.getPythonDriverClassName());
+ }
+
+ private FromJarEntryClassInformationProvider(
+ @Nullable File jarFile, @Nullable String jobClassName) {
+ Preconditions.checkArgument(
+ jarFile != null || jobClassName != null,
+ "Either the jar file or the job class name, or both need to be
set.");
+ this.jarFile = jarFile;
+ this.jobClassName = jobClassName;
+ }
+
+ /**
+ * Returns the specified {@code jarFile}. If no {@code jarFile} was
specified it can only return
+ * an empty {@code Optional} if the {@code jobClassName} is specified
instead. In that case, the
+ * caller should assume that the passed job class can be found on the
system classpath.
+ *
+ * @return The specified {@code jarFile}. If returning an empty {@code
Optional} the caller
+ * should assume to find the job class on the system classpath.
+ * @see #getJobClassName()
+ */
+ @Override
+ public Optional<File> getJarFile() {
+ return Optional.ofNullable(jarFile);
+ }
+
+ /**
+ * Returns the specified job class name that is either available in the
corresponding {@code
+ * jarFile} or, if the {@code jarFile} is not set, should be assumed to be
contained in the
+ * system classpath. It can return an emtpy {@code Optional} if the
respective {@code jarFile}
+ * is set. In that case one should assume the job class being the entry
class of the respective
+ * {@code jarFile}.
+ *
+ * @return Returns the job class that can either be found in the
respective {@code jarFile} or
+ * on the system classpath. It can also return an empty {@code
Optional} if the {@code
+ * jarFile} is specified. In that case the caller should assume the
job class being provided
+ * by the {@code jarFile} as the entry class.
Review comment:
similar suggestion to above
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgramRetrieverImpl.java
##########
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import
org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import
org.apache.flink.client.deployment.application.FromClasspathEntryClassInformationProvider;
+import
org.apache.flink.client.deployment.application.FromJarEntryClassInformationProvider;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.function.FunctionUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.stream.Collectors;
+
+/** {@code PackageProgramRetrieverImpl} implements {@link
PackagedProgramRetriever}. */
+public class PackagedProgramRetrieverImpl implements PackagedProgramRetriever {
+
+ private final EntryClassInformationProvider entryClassInformationProvider;
+ private final String[] programArguments;
+ private final List<URL> userClasspath;
+ private final Configuration configuration;
+
+ /**
+ * Creates a {@code PackageProgramRetrieverImpl} with the given parameters.
+ *
+ * @param userLibDir The user library directory that is used for
generating the user classpath
+ * if specified. The system classpath is used if not specified.
+ * @param jobClassName The job class that will be used if specified. The
classpath is used to
+ * detect any main class if not specified.
+ * @param programArgs The program arguments.
+ * @param configuration The Flink configuration for the given job.
+ * @return The {@code PackageProgramRetrieverImpl} that can be used to
create a {@link
+ * PackagedProgram} instance.
+ * @throws FlinkException If something goes wrong during instantiation.
+ */
+ public static PackagedProgramRetrieverImpl create(
+ @Nullable File userLibDir,
+ @Nullable String jobClassName,
+ String[] programArgs,
+ Configuration configuration)
+ throws FlinkException {
+ return create(userLibDir, null, jobClassName, programArgs,
configuration);
+ }
+
+ /**
+ * Creates a {@code PackageProgramRetrieverImpl} with the given parameters.
+ *
+ * @param userLibDir The user library directory that is used for
generating the user classpath
+ * if specified. The system classpath is used if not specified.
+ * @param jarFile The jar archive used expecting to have the job class
included.
+ * @param jobClassName The job class that will be used if specified. The
classpath is used to
+ * detect any main class if not specified.
Review comment:
```suggestion
* @param jobClassName The job class to use; if null the system
classpath will be scanned for possible main classes.
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgramRetrieverImpl.java
##########
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import
org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import
org.apache.flink.client.deployment.application.FromClasspathEntryClassInformationProvider;
+import
org.apache.flink.client.deployment.application.FromJarEntryClassInformationProvider;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.function.FunctionUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.stream.Collectors;
+
+/** {@code PackageProgramRetrieverImpl} implements {@link
PackagedProgramRetriever}. */
Review comment:
Well this is quite a non-comment isn't it. It could say something like
"default X implementation that can retrieve a packaged program from a specific
jar or the system classpath."
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgramRetrieverImpl.java
##########
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import
org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import
org.apache.flink.client.deployment.application.FromClasspathEntryClassInformationProvider;
+import
org.apache.flink.client.deployment.application.FromJarEntryClassInformationProvider;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.function.FunctionUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.stream.Collectors;
+
+/** {@code PackageProgramRetrieverImpl} implements {@link
PackagedProgramRetriever}. */
+public class PackagedProgramRetrieverImpl implements PackagedProgramRetriever {
Review comment:
Recently we used `Default*` for the default interface implementations.
##########
File path:
flink-clients/src/main/java/org/apache/flink/client/program/PackagedProgramRetrieverImpl.java
##########
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.client.program;
+
+import org.apache.flink.annotation.VisibleForTesting;
+import
org.apache.flink.client.deployment.application.EntryClassInformationProvider;
+import
org.apache.flink.client.deployment.application.FromClasspathEntryClassInformationProvider;
+import
org.apache.flink.client.deployment.application.FromJarEntryClassInformationProvider;
+import org.apache.flink.configuration.Configuration;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.FlinkException;
+import org.apache.flink.util.Preconditions;
+import org.apache.flink.util.function.FunctionUtils;
+
+import javax.annotation.Nullable;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Path;
+import java.util.Collections;
+import java.util.List;
+import java.util.NoSuchElementException;
+import java.util.stream.Collectors;
+
+/** {@code PackageProgramRetrieverImpl} implements {@link
PackagedProgramRetriever}. */
+public class PackagedProgramRetrieverImpl implements PackagedProgramRetriever {
+
+ private final EntryClassInformationProvider entryClassInformationProvider;
+ private final String[] programArguments;
+ private final List<URL> userClasspath;
+ private final Configuration configuration;
+
+ /**
+ * Creates a {@code PackageProgramRetrieverImpl} with the given parameters.
+ *
+ * @param userLibDir The user library directory that is used for
generating the user classpath
+ * if specified. The system classpath is used if not specified.
+ * @param jobClassName The job class that will be used if specified. The
classpath is used to
+ * detect any main class if not specified.
+ * @param programArgs The program arguments.
+ * @param configuration The Flink configuration for the given job.
+ * @return The {@code PackageProgramRetrieverImpl} that can be used to
create a {@link
+ * PackagedProgram} instance.
+ * @throws FlinkException If something goes wrong during instantiation.
+ */
+ public static PackagedProgramRetrieverImpl create(
+ @Nullable File userLibDir,
+ @Nullable String jobClassName,
+ String[] programArgs,
+ Configuration configuration)
+ throws FlinkException {
+ return create(userLibDir, null, jobClassName, programArgs,
configuration);
+ }
+
+ /**
+ * Creates a {@code PackageProgramRetrieverImpl} with the given parameters.
+ *
+ * @param userLibDir The user library directory that is used for
generating the user classpath
+ * if specified. The system classpath is used if not specified.
+ * @param jarFile The jar archive used expecting to have the job class
included.
Review comment:
```suggestion
* @param jarFile The jar archive expected to contain the job class;
null if the job class is on the system classpath.
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
