aljoscha commented on a change in pull request #11991:
URL: https://github.com/apache/flink/pull/11991#discussion_r419947262
##########
File path:
flink-yarn/src/main/java/org/apache/flink/yarn/YarnClusterDescriptor.java
##########
@@ -1105,160 +1076,6 @@ private ApplicationReport startAppMaster(
return report;
}
- /**
- * Returns the Path where the YARN application files should be uploaded
to.
- *
- * @param appId YARN application id
- */
- private Path getYarnFilesDir(final ApplicationId appId) throws
IOException {
- final FileSystem fileSystem = FileSystem.get(yarnConfiguration);
- final Path homeDir = fileSystem.getHomeDirectory();
- return new Path(homeDir, ".flink/" + appId + '/');
- }
-
- /**
- * Uploads and registers a single resource and adds it to
<tt>localResources</tt>.
- *
- * @param key
- * the key to add the resource under
- * @param fs
- * the remote file system to upload to
- * @param appId
- * application ID
- * @param localSrcPath
- * local path to the file
- * @param localResources
- * map of resources
- * @param replication
- * number of replications of a remote file to be created
- *
- * @return the remote path to the uploaded resource
- */
- private static Path setupSingleLocalResource(
- String key,
- FileSystem fs,
- ApplicationId appId,
- Path localSrcPath,
- Map<String, LocalResource> localResources,
- Path targetHomeDir,
- String relativeTargetPath,
- int replication) throws IOException {
- Tuple2<Path, LocalResource> resource = Utils.setupLocalResource(
- fs,
- appId.toString(),
- localSrcPath,
- targetHomeDir,
- relativeTargetPath,
- replication);
-
- localResources.put(key, resource.f1);
-
- return resource.f0;
- }
-
- /**
- * Match file name for "<tt>flink-dist*.jar</tt>" pattern.
- *
- * @param fileName file name to check
- * @return true if file is a dist jar
- */
- private static boolean isDistJar(String fileName) {
- return fileName.startsWith("flink-dist") &&
fileName.endsWith("jar");
- }
-
- /**
- * Recursively uploads (and registers) any (user and system) files in
<tt>shipFiles</tt> except
- * for files matching "<tt>flink-dist*.jar</tt>" which should be
uploaded separately.
- *
- * @param shipFiles
- * files to upload
- * @param fs
- * file system to upload to
- * @param targetHomeDir
- * remote home directory to upload to
- * @param appId
- * application ID
- * @param remotePaths
- * paths of the remote resources (uploaded resources will
be added)
- * @param localResources
- * map of resources (uploaded resources will be added)
- * @param localResourcesDirectory
- * the directory the localResources are uploaded to
- * @param envShipFileList
- * list of shipped files in a format understood by {@link
Utils#createTaskExecutorContext}
- * @param replication
- * number of replications of a remote file to be created
- *
- * @return list of class paths with the the proper resource keys from
the registration
- */
- static List<String> uploadAndRegisterFiles(
- Collection<File> shipFiles,
- FileSystem fs,
- Path targetHomeDir,
- ApplicationId appId,
- List<Path> remotePaths,
- Map<String, LocalResource> localResources,
- String localResourcesDirectory,
- StringBuilder envShipFileList,
- int replication) throws IOException {
-
- checkArgument(replication >= 1);
- final List<Path> localPaths = new ArrayList<>();
- final List<Path> relativePaths = new ArrayList<>();
- for (File shipFile : shipFiles) {
- if (shipFile.isDirectory()) {
- // add directories to the classpath
- final java.nio.file.Path shipPath =
shipFile.toPath();
- final java.nio.file.Path parentPath =
shipPath.getParent();
- Files.walkFileTree(shipPath, new
SimpleFileVisitor<java.nio.file.Path>() {
- @Override
- public FileVisitResult
visitFile(java.nio.file.Path file, BasicFileAttributes attrs) {
- localPaths.add(new
Path(file.toUri()));
- relativePaths.add(new
Path(localResourcesDirectory, parentPath.relativize(file).toString()));
- return FileVisitResult.CONTINUE;
- }
- });
- } else {
- localPaths.add(new Path(shipFile.toURI()));
- relativePaths.add(new
Path(localResourcesDirectory, shipFile.getName()));
- }
- }
-
- final Set<String> archives = new HashSet<>();
- final Set<String> resources = new HashSet<>();
- for (int i = 0; i < localPaths.size(); i++) {
- final Path localPath = localPaths.get(i);
- final Path relativePath = relativePaths.get(i);
- if (!isDistJar(relativePath.getName())) {
Review comment:
Ah, I know saw that you renamed the method in the moved code. I was
looking for the same method name.
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
