This is an automated email from the ASF dual-hosted git repository.
zhouquan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/submarine.git
The following commit(s) were added to refs/heads/master by this push:
new bdaf829 SUBMARINE-68. Add tests to FileSystemOperations class
bdaf829 is described below
commit bdaf829db77f390f58b5824714acacf3b36d18d8
Author: Adam Antal <[email protected]>
AuthorDate: Tue Dec 10 10:11:13 2019 +0100
SUBMARINE-68. Add tests to FileSystemOperations class
### What is this PR for?
Adding tests to `FileSystemOperations` class and also performed some minor
refactor in the test infrastructure.
### What type of PR is it?
Refactoring
### Todos
* [ ] - new UTs should pass
### What is the Jira issue?
[SUBMARINE-68](https://issues.apache.org/jira/browse/SUBMARINE-68)
### How should this be tested?
* Only the new UTs should pass.
### Screenshots (if appropriate)
### Questions:
* Does the licenses files need update? No
* Is there breaking changes for older versions? No
* Does this needs documentation? No
Author: Adam Antal <[email protected]>
Closes #111 from adamantal/SUBMARINE-68 and squashes the following commits:
7ac717c [Adam Antal] Fix checkstyle error.
33a43ca [Adam Antal] SUBMARINE-68. Add tests to FileSystemOperations class
---
.../client/cli/ShowJobCliParsingTest.java | 8 +-
.../cli/runjob/RunJobCliParsingCommonTest.java | 2 +-
.../runjob/RunJobCliParsingParameterizedTest.java | 5 +-
.../commons/runtime/MockClientContext.java | 9 +-
.../runtime/fs/MockRemoteDirectoryManager.java | 97 +++++---
.../src/test/java/YarnUtilsTest.java | 2 +-
.../yarnservice/FileSystemOperations.java | 203 ++++++++++++-----
.../submitter/yarnservice/utils/Localizer.java | 7 +-
.../apache/submarine/FileUtilitiesForTests.java | 13 ++
.../cli/yarnservice/YarnServiceCliTestUtils.java | 8 +-
.../YarnServiceRunJobCliCommonsTest.java | 2 +-
.../YarnServiceRunJobCliLocalizationTest.java | 3 +-
.../cli/yarnservice/YarnServiceRunJobCliTest.java | 2 +-
.../yarnservice/FileSystemOperationsTest.java | 246 +++++++++++++++++++++
.../command/AbstractTFLaunchCommandTestHelper.java | 3 +-
.../pytorch/PyTorchServiceSpecTest.java | 3 +-
.../command/TensorBoardLaunchCommandTest.java | 6 +-
.../command/TensorFlowLaunchCommandTest.java | 13 +-
.../tensorflow/component/ComponentTestCommons.java | 11 +-
.../component/TensorBoardComponentTest.java | 2 +-
.../utils/KerberosPrincipalFactoryTest.java | 11 +-
.../submitter/yarnservice/utils/LocalizerTest.java | 6 +-
22 files changed, 535 insertions(+), 127 deletions(-)
diff --git
a/submarine-client/src/test/java/org/apache/submarine/client/cli/ShowJobCliParsingTest.java
b/submarine-client/src/test/java/org/apache/submarine/client/cli/ShowJobCliParsingTest.java
index b95132c..9bbbbe3 100644
---
a/submarine-client/src/test/java/org/apache/submarine/client/cli/ShowJobCliParsingTest.java
+++
b/submarine-client/src/test/java/org/apache/submarine/client/cli/ShowJobCliParsingTest.java
@@ -48,8 +48,8 @@ public class ShowJobCliParsingTest {
}
@Test
- public void testPrintHelp() {
- MockClientContext mockClientContext = new MockClientContext();
+ public void testPrintHelp() throws IOException {
+ MockClientContext mockClientContext = new MockClientContext("testJob");
ShowJobCli showJobCli = new ShowJobCli(mockClientContext);
showJobCli.printUsages();
}
@@ -58,7 +58,7 @@ public class ShowJobCliParsingTest {
public void testShowJob()
throws InterruptedException, SubmarineException, YarnException,
ParseException, IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new MockClientContext("testJob");
ShowJobCli showJobCli = new ShowJobCli(mockClientContext) {
@Override
protected void getAndPrintJobInfo() {
@@ -84,7 +84,7 @@ public class ShowJobCliParsingTest {
throws InterruptedException, SubmarineException, YarnException,
ParseException, IOException {
SubmarineStorage storage = new MemorySubmarineStorage();
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new MockClientContext("testJob");
RuntimeFactory runtimeFactory = mock(RuntimeFactory.class);
when(runtimeFactory.getSubmarineStorage()).thenReturn(storage);
mockClientContext.setRuntimeFactory(runtimeFactory);
diff --git
a/submarine-client/src/test/java/org/apache/submarine/client/cli/runjob/RunJobCliParsingCommonTest.java
b/submarine-client/src/test/java/org/apache/submarine/client/cli/runjob/RunJobCliParsingCommonTest.java
index 0889030..d796574 100644
---
a/submarine-client/src/test/java/org/apache/submarine/client/cli/runjob/RunJobCliParsingCommonTest.java
+++
b/submarine-client/src/test/java/org/apache/submarine/client/cli/runjob/RunJobCliParsingCommonTest.java
@@ -59,7 +59,7 @@ public class RunJobCliParsingCommonTest {
public static MockClientContext getMockClientContext()
throws IOException, YarnException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new MockClientContext("testJob");
JobSubmitter mockJobSubmitter = mock(JobSubmitter.class);
when(mockJobSubmitter.submitJob(any(ParametersHolder.class)))
.thenReturn(ApplicationId.newInstance(1235L, 1));
diff --git
a/submarine-client/src/test/java/org/apache/submarine/client/cli/runjob/RunJobCliParsingParameterizedTest.java
b/submarine-client/src/test/java/org/apache/submarine/client/cli/runjob/RunJobCliParsingParameterizedTest.java
index 310979f..5e5ab09 100644
---
a/submarine-client/src/test/java/org/apache/submarine/client/cli/runjob/RunJobCliParsingParameterizedTest.java
+++
b/submarine-client/src/test/java/org/apache/submarine/client/cli/runjob/RunJobCliParsingParameterizedTest.java
@@ -36,6 +36,7 @@ import org.junit.rules.ExpectedException;
import org.junit.runner.RunWith;
import org.junit.runners.Parameterized;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
@@ -80,8 +81,8 @@ public class RunJobCliParsingParameterizedTest {
}
@Test
- public void testPrintHelp() {
- MockClientContext mockClientContext = new MockClientContext();
+ public void testPrintHelp() throws IOException {
+ MockClientContext mockClientContext = new MockClientContext("testJob");
JobSubmitter mockJobSubmitter = mock(JobSubmitter.class);
JobMonitor mockJobMonitor = mock(JobMonitor.class);
RunJobCli runJobCli = new RunJobCli(mockClientContext, mockJobSubmitter,
diff --git
a/submarine-commons/commons-runtime/src/test/java/org/apache/submarine/commons/runtime/MockClientContext.java
b/submarine-commons/commons-runtime/src/test/java/org/apache/submarine/commons/runtime/MockClientContext.java
index 4370875..4f55bf7 100644
---
a/submarine-commons/commons-runtime/src/test/java/org/apache/submarine/commons/runtime/MockClientContext.java
+++
b/submarine-commons/commons-runtime/src/test/java/org/apache/submarine/commons/runtime/MockClientContext.java
@@ -22,10 +22,15 @@ package org.apache.submarine.commons.runtime;
import org.apache.submarine.commons.runtime.fs.MockRemoteDirectoryManager;
import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager;
+import java.io.IOException;
+
public class MockClientContext extends ClientContext {
- private RemoteDirectoryManager remoteDirectoryMgr =
- new MockRemoteDirectoryManager();
+ private RemoteDirectoryManager remoteDirectoryMgr;
+
+ public MockClientContext(String jobName) throws IOException {
+ this.remoteDirectoryMgr = new MockRemoteDirectoryManager(jobName);
+ }
@Override
public RemoteDirectoryManager getRemoteDirectoryManager() {
diff --git
a/submarine-commons/commons-runtime/src/test/java/org/apache/submarine/commons/runtime/fs/MockRemoteDirectoryManager.java
b/submarine-commons/commons-runtime/src/test/java/org/apache/submarine/commons/runtime/fs/MockRemoteDirectoryManager.java
index 00bfc02..71331d5 100644
---
a/submarine-commons/commons-runtime/src/test/java/org/apache/submarine/commons/runtime/fs/MockRemoteDirectoryManager.java
+++
b/submarine-commons/commons-runtime/src/test/java/org/apache/submarine/commons/runtime/fs/MockRemoteDirectoryManager.java
@@ -19,39 +19,86 @@
package org.apache.submarine.commons.runtime.fs;
+import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.util.Time;
import java.io.File;
import java.io.IOException;
import java.util.Objects;
public class MockRemoteDirectoryManager implements RemoteDirectoryManager {
- private File jobsParentDir = null;
- private File modelParentDir = null;
- private File jobDir = null;
+ private static final String FAILED_TO_CREATE_DIRS_FORMAT_STRING =
+ "Failed to create directories under path: %s";
+ private static final String JOB_NAME_MUST_NOT_BE_NULL =
+ "Job name must not be null!";
+ private static final File STAGING_AREA = new File("target/_staging_area_");
+
+ private File jobsParentDir;
+ private File jobDir;
+ private File modelParentDir;
+
+ public MockRemoteDirectoryManager(String jobName) throws IOException {
+ Objects.requireNonNull(jobName, JOB_NAME_MUST_NOT_BE_NULL);
+ this.cleanup();
+ this.jobsParentDir = initializeJobParentDir();
+ this.jobDir = initializeJobDir(jobName);
+ this.modelParentDir = initializeModelParentDir();
+ }
+
+ private void cleanup() throws IOException {
+ FileUtils.deleteDirectory(STAGING_AREA);
+ }
+
+ private File initializeJobParentDir() throws IOException {
+ File dir = new File(STAGING_AREA, String.valueOf(Time.monotonicNow()));
+ if (!dir.mkdirs()) {
+ throw new IOException(
+ String.format(FAILED_TO_CREATE_DIRS_FORMAT_STRING,
+ dir.getAbsolutePath()));
+ }
+ return dir;
+ }
+
+ private File initializeJobDir(String jobName) throws IOException {
+ Objects.requireNonNull(jobsParentDir, "Job parent dir must not be null!");
+ File dir = new File(jobsParentDir.getAbsolutePath(), jobName);
+
+ if (!dir.exists() && !dir.mkdirs()) {
+ throw new IOException(
+ String.format(FAILED_TO_CREATE_DIRS_FORMAT_STRING,
+ dir.getAbsolutePath()));
+ }
+ return dir;
+ }
+
+ private File initializeModelParentDir() throws IOException {
+ File dir = new File(
+ "target/_models_" + System.currentTimeMillis());
+ if (!dir.mkdirs()) {
+ throw new IOException(
+ String.format(FAILED_TO_CREATE_DIRS_FORMAT_STRING,
+ dir.getAbsolutePath()));
+ }
+ return dir;
+ }
+
@Override
public Path getJobStagingArea(String jobName, boolean create)
throws IOException {
- Objects.requireNonNull(jobName, "Job name must not be null!");
- if (jobsParentDir == null && create) {
- jobsParentDir = new File(
- "target/_staging_area_" + System.currentTimeMillis());
- if (!jobsParentDir.mkdirs()) {
- throw new IOException(
- "Failed to mkdirs for" + jobsParentDir.getAbsolutePath());
- }
- }
-
- this.jobDir = new File(jobsParentDir.getAbsolutePath(), jobName);
+ Objects.requireNonNull(jobName, JOB_NAME_MUST_NOT_BE_NULL);
+ Objects.requireNonNull(jobDir, JOB_NAME_MUST_NOT_BE_NULL);
+ this.jobDir = initializeJobDir(jobName);
if (create && !jobDir.exists()) {
if (!jobDir.mkdirs()) {
- throw new IOException("Failed to mkdirs for "
- + jobDir.getAbsolutePath());
+ throw new IOException(
+ String.format(FAILED_TO_CREATE_DIRS_FORMAT_STRING,
+ jobDir.getAbsolutePath()));
}
}
return new Path(jobDir.getAbsolutePath());
@@ -66,15 +113,6 @@ public class MockRemoteDirectoryManager implements
RemoteDirectoryManager {
@Override
public Path getModelDir(String modelName, boolean create)
throws IOException {
- if (modelParentDir == null && create) {
- modelParentDir = new File(
- "target/_models_" + System.currentTimeMillis());
- if (!modelParentDir.mkdirs()) {
- throw new IOException(
- "Failed to mkdirs for " + modelParentDir.getAbsolutePath());
- }
- }
-
File modelDir = new File(modelParentDir.getAbsolutePath(), modelName);
if (create) {
if (!modelDir.exists() && !modelDir.mkdirs()) {
@@ -104,7 +142,6 @@ public class MockRemoteDirectoryManager implements
RemoteDirectoryManager {
public boolean isDir(String uri) throws IOException {
return getDefaultFileSystem().getFileStatus(
new Path(convertToStagingPath(uri))).isDirectory();
-
}
@Override
@@ -117,13 +154,12 @@ public class MockRemoteDirectoryManager implements
RemoteDirectoryManager {
}
private String convertToStagingPath(String uri) throws IOException {
- String ret = uri;
if (isRemote(uri)) {
String dirName = new Path(uri).getName();
- ret = this.jobDir.getAbsolutePath()
+ return this.jobDir.getAbsolutePath()
+ "/" + dirName;
}
- return ret;
+ return uri;
}
/**
@@ -168,4 +204,7 @@ public class MockRemoteDirectoryManager implements
RemoteDirectoryManager {
return 100 * 1024 * 1024;
}
+ public void setJobDir(File jobDir) {
+ this.jobDir = jobDir;
+ }
}
diff --git
a/submarine-server/server-submitter/submitter-yarn/src/test/java/YarnUtilsTest.java
b/submarine-server/server-submitter/submitter-yarn/src/test/java/YarnUtilsTest.java
index 5461141..3e292fb 100644
---
a/submarine-server/server-submitter/submitter-yarn/src/test/java/YarnUtilsTest.java
+++
b/submarine-server/server-submitter/submitter-yarn/src/test/java/YarnUtilsTest.java
@@ -48,7 +48,7 @@ public class YarnUtilsTest {
private MockClientContext getMockClientContext()
throws IOException, YarnException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new MockClientContext("testJob");
JobSubmitter mockJobSubmitter = mock(JobSubmitter.class);
when(mockJobSubmitter.submitJob(
any(ParametersHolder.class))).thenReturn(
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
index 219d6a7..229b9e2 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperations.java
@@ -52,6 +52,24 @@ import java.util.Set;
* well.
*/
public class FileSystemOperations {
+ private static final String TEMP_DIR = System.getProperty("java.io.tmpdir");
+ private static final long BYTES_IN_MB = 1024 * 1024;
+
+ private static class DownloadResult {
+ private final String srcPath;
+ private final String dstPath;
+ private final String suffix;
+ private boolean remote;
+
+ DownloadResult(String srcPath, String dstPath, String suffix,
+ boolean remote) {
+ this.srcPath = srcPath;
+ this.dstPath = dstPath;
+ this.suffix = suffix;
+ this.remote = remote;
+ }
+ }
+
private static final Logger LOG =
LoggerFactory.getLogger(FileSystemOperations.class);
private final SubmarineConfiguration submarineConfig;
@@ -67,54 +85,90 @@ public class FileSystemOperations {
}
/**
- * May download a remote uri(file/dir) and zip.
- * Skip download if local dir
- * Remote uri can be a local dir(won't download)
- * or remote HDFS dir, s3 dir/file .etc
- * */
- public String downloadAndZip(String remoteDir, String zipFileName,
- boolean doZip)
+ * May download a remote URI (file or directory) and zip it if asked.
+ * Skips downloading local directories.
+ * Remote URI can be a local dir or remote HDFS dir,
+ * S3 file or directory, etc.
+ */
+ public String downloadAndZip(String remoteDir, String destFileName)
+ throws IOException {
+ DownloadResult downloadResult = downloadInternal(remoteDir, destFileName);
+ String zipFileUri = zipLocalDirectory(downloadResult);
+
+ if (downloadResult.remote) {
+ deleteFiles(downloadResult.srcPath);
+ }
+ return zipFileUri;
+ }
+
+ public String download(String remoteDir, String zipFileName)
+ throws IOException {
+ DownloadResult downloadResult = downloadInternal(remoteDir, zipFileName);
+ return downloadResult.srcPath;
+ }
+
+ private DownloadResult downloadInternal(String remoteDir, String
destFileName)
throws IOException {
- //Append original modification time and size to zip file name
String suffix;
String srcDir = remoteDir;
- String zipDirPath =
- System.getProperty("java.io.tmpdir") + "/" + zipFileName;
- boolean needDeleteTempDir = false;
- if (remoteDirectoryManager.isRemote(remoteDir)) {
- //Append original modification time and size to zip file name
+
+ String destFilePath = getFilePathInTempDir(destFileName);
+
+ boolean remote = remoteDirectoryManager.isRemote(remoteDir);
+ if (remote) {
+ // Append original modification time and size to zip file name
FileStatus status =
remoteDirectoryManager.getRemoteFileStatus(new Path(remoteDir));
- suffix = "_" + status.getModificationTime()
- + "-" + remoteDirectoryManager.getRemoteFileSize(remoteDir);
+ suffix = getSuffixOfRemoteDirectory(remoteDir, status);
// Download them to temp dir
- boolean downloaded =
- remoteDirectoryManager.copyRemoteToLocal(remoteDir, zipDirPath);
- if (!downloaded) {
- throw new IOException("Failed to download files from "
- + remoteDir);
- }
- LOG.info("Downloaded remote: {} to local: {}", remoteDir, zipDirPath);
- srcDir = zipDirPath;
- needDeleteTempDir = true;
+ downloadRemoteFile(remoteDir, destFilePath);
+ srcDir = destFilePath;
} else {
File localDir = new File(remoteDir);
- suffix = "_" + localDir.lastModified()
- + "-" + localDir.length();
- }
- if (!doZip) {
- return srcDir;
+ suffix = getSuffixOfLocalDirectory(localDir);
}
- // zip a local dir
- String zipFileUri =
- ZipUtilities.zipDir(srcDir, zipDirPath + suffix + ".zip");
- // delete downloaded temp dir
- if (needDeleteTempDir) {
- deleteFiles(srcDir);
+ return new DownloadResult(srcDir, destFilePath, suffix, remote);
+ }
+
+ private String getFilePathInTempDir(String zipFileName) {
+ return new File(TEMP_DIR, zipFileName).getAbsolutePath();
+ }
+
+ private String zipLocalDirectory(DownloadResult downloadResult)
+ throws IOException {
+ String dstFileName = downloadResult.dstPath +
+ downloadResult.suffix + ".zip";
+ return ZipUtilities.zipDir(downloadResult.srcPath,
+ dstFileName);
+ }
+
+ private String getSuffixOfRemoteDirectory(String remoteDir,
+ FileStatus status) throws IOException {
+ return getSuffixOfDirectory(status.getModificationTime(),
+ remoteDirectoryManager.getRemoteFileSize(remoteDir));
+ }
+
+ private String getSuffixOfLocalDirectory(File localDir) {
+ return getSuffixOfDirectory(localDir.lastModified(), localDir.length());
+ }
+
+ private String getSuffixOfDirectory(long modificationTime, long size) {
+ return "_" + modificationTime + "-" + size;
+ }
+
+ private void downloadRemoteFile(String remoteDir, String zipDirPath)
+ throws IOException {
+ boolean downloaded =
+ remoteDirectoryManager.copyRemoteToLocal(remoteDir, zipDirPath);
+ if (!downloaded) {
+ throw new IOException("Failed to download Internal files from "
+ + remoteDir);
}
- return zipFileUri;
+ LOG.info("Downloaded remote file: {} to this local path: {}",
+ remoteDir, zipDirPath);
}
+
public void deleteFiles(String localUri) {
boolean success = FileUtil.fullyDelete(new File(localUri));
if (!success) {
@@ -137,12 +191,18 @@ public class FileSystemOperations {
FileSystem fs = FileSystem.get(yarnConfig);
FileStatus fileStatus = fs.getFileStatus(uploadedFilePath);
- LOG.info("Uploaded file path = " + fileStatus.getPath());
+ LOG.info("Uploaded file path: " + fileStatus.getPath());
+ ConfigFile configFile = new ConfigFile()
+ .srcFile(fileStatus.getPath().toUri().toString())
+ .destFile(destFilename)
+ .type(ConfigFile.TypeEnum.STATIC);
+ addFilesToComponent(comp, configFile);
+ }
- // Set it to component's files list
- comp.getConfiguration().getFiles().add(new ConfigFile().srcFile(
- fileStatus.getPath().toUri().toString()).destFile(destFilename)
- .type(ConfigFile.TypeEnum.STATIC));
+ private void addFilesToComponent(Component comp, ConfigFile... configFiles) {
+ for (ConfigFile configFile : configFiles) {
+ comp.getConfiguration().getFiles().add(configFile);
+ }
}
public Path uploadToRemoteFile(Path stagingDir, String fileToUpload) throws
@@ -153,15 +213,15 @@ public class FileSystemOperations {
File localFile = new File(fileToUpload);
if (!localFile.exists()) {
throw new FileNotFoundException(
- "Trying to upload file=" + localFile.getAbsolutePath()
- + " to remote, but couldn't find local file.");
+ "Trying to upload file " + localFile.getAbsolutePath()
+ + " to remote, but could not find local file!");
}
- String filename = new File(fileToUpload).getName();
+ String filename = localFile.getName();
Path uploadedFilePath = new Path(stagingDir, filename);
if (!uploadedFiles.contains(uploadedFilePath)) {
if (SubmarineLogs.isVerbose()) {
- LOG.info("Copying local file=" + fileToUpload + " to remote="
+ LOG.info("Copying local file " + fileToUpload + " to remote "
+ uploadedFilePath);
}
fs.copyFromLocalFile(new Path(fileToUpload), uploadedFilePath);
@@ -171,26 +231,43 @@ public class FileSystemOperations {
}
public void validFileSize(String uri) throws IOException {
- long actualSizeByte;
- String locationType = "Local";
- if (remoteDirectoryManager.isRemote(uri)) {
- actualSizeByte = remoteDirectoryManager.getRemoteFileSize(uri);
- locationType = "Remote";
+ boolean remote = remoteDirectoryManager.isRemote(uri);
+ long actualSizeInBytes = getFileSizeInBytes(uri, remote);
+ long maxFileSizeInBytes = convertToBytes(getMaxRemoteFileSizeMB());
+
+ String locationType = remote ? "Remote" : "Local";
+ LOG.info("{} file / directory path is {} with size: {} bytes."
+ + " Allowed maximum file / directory size is {} bytes.",
+ locationType, uri, actualSizeInBytes, maxFileSizeInBytes);
+
+ if (actualSizeInBytes > maxFileSizeInBytes) {
+ throw new IOException(
+ String.format("Size of file / directory %s: %d bytes. " +
+ "This exceeded the configured maximum " +
+ "file / directory size, which is %d bytes.",
+ uri, actualSizeInBytes, maxFileSizeInBytes));
+ }
+ }
+
+ private long getFileSizeInBytes(String uri, boolean remote)
+ throws IOException {
+ long actualSizeInBytes;
+ if (remote) {
+ actualSizeInBytes = remoteDirectoryManager.getRemoteFileSize(uri);
} else {
- actualSizeByte = FileUtil.getDU(new File(uri));
+ actualSizeInBytes = FileUtil.getDU(new File(uri));
}
- long maxFileSizeMB = submarineConfig.getLong(
+ return actualSizeInBytes;
+ }
+
+ private long getMaxRemoteFileSizeMB() {
+ return submarineConfig.getLong(
SubmarineConfiguration.ConfVars.
SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB);
- LOG.info("{} fie/dir: {}, size(Byte):{},"
- + " Allowed max file/dir size: {}",
- locationType, uri, actualSizeByte, maxFileSizeMB * 1024 * 1024);
-
- if (actualSizeByte > maxFileSizeMB * 1024 * 1024) {
- throw new IOException(uri + " size(Byte): "
- + actualSizeByte + " exceeds configured max size:"
- + maxFileSizeMB * 1024 * 1024);
- }
+ }
+
+ private long convertToBytes(long fileSizeMB) {
+ return fileSizeMB * BYTES_IN_MB;
}
public void setPermission(Path destPath, FsPermission permission) throws
@@ -211,4 +288,8 @@ public class FileSystemOperations {
public static boolean needHdfs(String content) {
return content != null && content.contains("hdfs://");
}
+
+ public Set<Path> getUploadedFiles() {
+ return uploadedFiles;
+ }
}
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/utils/Localizer.java
b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/utils/Localizer.java
index 1fcdc2a..eac91a1 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/utils/Localizer.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/main/java/org/apache/submarine/server/submitter/yarnservice/utils/Localizer.java
@@ -124,13 +124,12 @@ public class Localizer {
if (localizationState.directory) {
// Special handling of remoteUri directory.
return fsOperations.downloadAndZip(localizationState.remoteUri,
- getLastNameFromPath(localizationState.remoteUri), true);
+ getLastNameFromPath(localizationState.remoteUri));
} else if (localizationState.remote &&
!needHdfs(localizationState.remoteUri)) {
// Non HDFS remote URI.
- // Non directory, we don't need to zip
- return fsOperations.downloadAndZip(localizationState.remoteUri,
- getLastNameFromPath(localizationState.remoteUri), false);
+ return fsOperations.download(localizationState.remoteUri,
+ getLastNameFromPath(localizationState.remoteUri));
}
return localizationState.remoteUri;
}
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/FileUtilitiesForTests.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/FileUtilitiesForTests.java
index 3282c32..cf6fc98 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/FileUtilitiesForTests.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/FileUtilitiesForTests.java
@@ -39,6 +39,9 @@ import static org.junit.Assert.assertTrue;
public class FileUtilitiesForTests {
private static final Logger LOG =
LoggerFactory.getLogger(FileUtilitiesForTests.class);
+
+ public static final String FILE_SCHEME = "file://";
+
private String tempDir;
private List<File> cleanupFiles;
@@ -95,6 +98,16 @@ public class FileUtilitiesForTests {
return file;
}
+ public File createDirInDir(Path dir, String newDir) {
+ File dirTmp = new File(dir.toUri().getPath());
+ if (!dirTmp.exists()) {
+ createDirectory(dirTmp);
+ }
+ File file = new File(dir.toUri().getPath(), newDir);
+ createDirectory(file);
+ return file;
+ }
+
public File createDirectory(Path parent, String dirname) {
File dir =
new File(parent.toUri().getPath() + "/" + new Path(dirname).getName());
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceCliTestUtils.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceCliTestUtils.java
index 13413b8..b41191f 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceCliTestUtils.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceCliTestUtils.java
@@ -24,9 +24,13 @@ import org.apache.submarine.commons.runtime.RuntimeFactory;
import org.apache.submarine.commons.runtime.fs.MemorySubmarineStorage;
import
org.apache.submarine.server.submitter.yarnservice.YarnServiceRuntimeFactory;
+import java.io.IOException;
+
+import static
org.apache.submarine.client.cli.yarnservice.YarnServiceRunJobCliCommonsTest.DEFAULT_JOB_NAME;
+
public class YarnServiceCliTestUtils {
- public static MockClientContext getMockClientContext() {
- MockClientContext mockClientContext = new MockClientContext();
+ public static MockClientContext getMockClientContext() throws IOException {
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
RuntimeFactory runtimeFactory = new YarnServiceRuntimeFactory(
mockClientContext);
mockClientContext.setRuntimeFactory(runtimeFactory);
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliCommonsTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliCommonsTest.java
index 3a732ca..5f1c39a 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliCommonsTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliCommonsTest.java
@@ -40,7 +40,7 @@ import static org.mockito.Mockito.when;
* Common operations shared with test classes using Run job-related actions.
*/
public class YarnServiceRunJobCliCommonsTest {
- static final String DEFAULT_JOB_NAME = "my-job";
+ public static final String DEFAULT_JOB_NAME = "my-job";
static final String DEFAULT_DOCKER_IMAGE = "tf-docker:1.1.0";
static final String DEFAULT_INPUT_PATH = "s3://input";
static final String DEFAULT_CHECKPOINT_PATH = "s3://output";
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliLocalizationTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliLocalizationTest.java
index 0fe37ef..c8a07cb 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliLocalizationTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliLocalizationTest.java
@@ -465,7 +465,8 @@ public class YarnServiceRunJobCliLocalizationTest {
runJobCli.run(params);
} catch (IOException e) {
assertTrue(e.getMessage()
- .contains("104857600 exceeds configured max size:10485760"));
+ .contains("104857600 bytes. This exceeded the configured " +
+ "maximum file / directory size, which is 10485760 bytes"));
// we shouldn't do any download because fail fast
verifyRdmCopyToRemoteLocalCalls(0);
}
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliTest.java
index 33d489a..54bf7f0 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/client/cli/yarnservice/YarnServiceRunJobCliTest.java
@@ -86,7 +86,7 @@ public class YarnServiceRunJobCliTest {
}
@Test
- public void testPrintHelp() {
+ public void testPrintHelp() throws IOException {
MockClientContext mockClientContext =
YarnServiceCliTestUtils.getMockClientContext();
RunJobCli runJobCli = new RunJobCli(mockClientContext);
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperationsTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperationsTest.java
new file mode 100644
index 0000000..4f36212
--- /dev/null
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/FileSystemOperationsTest.java
@@ -0,0 +1,246 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.submarine.server.submitter.yarnservice;
+
+import com.google.common.collect.Lists;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.yarn.service.api.records.Component;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile;
+import org.apache.hadoop.yarn.service.api.records.ConfigFile.TypeEnum;
+import org.apache.submarine.FileUtilitiesForTests;
+import org.apache.submarine.commons.runtime.ClientContext;
+import org.apache.submarine.commons.runtime.MockClientContext;
+import org.apache.submarine.commons.runtime.fs.RemoteDirectoryManager;
+import org.apache.submarine.commons.utils.SubmarineConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+import org.junit.runner.RunWith;
+import org.mockito.junit.MockitoJUnitRunner;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Collections;
+import java.util.List;
+import java.util.Set;
+
+import static org.apache.submarine.FileUtilitiesForTests.FILE_SCHEME;
+import static
org.apache.submarine.client.cli.yarnservice.YarnServiceRunJobCliCommonsTest.DEFAULT_JOB_NAME;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.ArgumentMatchers.anyString;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+@RunWith(MockitoJUnitRunner.Silent.class)
+public class FileSystemOperationsTest {
+ private static final String TARGET_ZIP_FILE = "targetZipFile";
+ private static final String TARGET_ZIP_DIR = "targetZipDir";
+ @Rule
+ public ExpectedException expectedException = ExpectedException.none();
+
+ private FileUtilitiesForTests fileUtils = new FileUtilitiesForTests();
+ private FileSystemOperations fileSystemOperations;
+ private Path stagingDir;
+
+ private Path getStagingDir(MockClientContext mockClientContext)
+ throws IOException {
+ return mockClientContext.getRemoteDirectoryManager()
+ .getJobStagingArea(DEFAULT_JOB_NAME, true);
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ fileUtils.setup();
+ MockClientContext mockClientContext =
+ new MockClientContext(DEFAULT_JOB_NAME);
+ fileSystemOperations = new FileSystemOperations(mockClientContext);
+ stagingDir = getStagingDir(mockClientContext);
+ fileUtils.createDirInTempDir(TARGET_ZIP_DIR);
+ }
+
+ @After
+ public void teardown() throws IOException {
+ fileUtils.teardown();
+ }
+
+ @Test
+ public void testDownloadLocalFileWithSimpleName() throws IOException {
+ String resultFile = fileSystemOperations.download("localFile",
+ TARGET_ZIP_FILE);
+ assertEquals("localFile", resultFile);
+ }
+
+ @Test
+ public void testDownloadLocalFile() throws IOException {
+ String resultFile = fileSystemOperations.download("/tmp/localFile",
+ TARGET_ZIP_FILE);
+ assertEquals("/tmp/localFile", resultFile);
+ }
+
+ @Test
+ public void testDownloadRemoteFile() throws IOException {
+ String remoteUri = "hdfs:///tmp/remoteFile";
+ fileUtils.createFileInDir(stagingDir, remoteUri);
+
+ String resultFilePath = fileSystemOperations.download(remoteUri,
+ TARGET_ZIP_FILE);
+ File resultFile = new File(resultFilePath);
+
+ assertTrue(resultFile.exists());
+ assertEquals(TARGET_ZIP_FILE, resultFile.getName());
+ }
+
+ @Test
+ public void testDownloadAndZip() throws IOException {
+ String remoteDir = "hdfs://remoteDir/";
+ fileUtils.createDirInDir(stagingDir, "remoteDir");
+
+ String resultFilePath = fileSystemOperations.downloadAndZip(remoteDir,
+ TARGET_ZIP_DIR);
+ File resultFile = new File(resultFilePath);
+
+ assertTrue(resultFile.exists());
+ assertTrue(
+ String.format(
+ "Result file name is '%s' and does not start with prefix '%s'",
+ resultFile.getName(), TARGET_ZIP_DIR),
+ resultFile.getName().startsWith(TARGET_ZIP_DIR));
+ }
+
+ @Test(expected = FileNotFoundException.class)
+ public void testUploadToRemoteFileNotExistingFile() throws IOException {
+ fileSystemOperations.uploadToRemoteFile(stagingDir, "notExisting");
+ }
+
+ @Test
+ public void testUploadToRemoteFile() throws IOException {
+ File testFile = fileUtils.createFileInTempDir("testFile");
+ Path path = fileSystemOperations.uploadToRemoteFile(stagingDir,
+ testFile.getAbsolutePath());
+
+ File expectedFile = new File(new File(stagingDir.toString()), "testFile");
+ assertEquals(expectedFile.getAbsolutePath(), path.toString());
+
+ Set<Path> uploadedFiles = fileSystemOperations.getUploadedFiles();
+ assertEquals(1, uploadedFiles.size());
+ List<Path> pathList = Lists.newArrayList(uploadedFiles);
+ Path storedPath = pathList.get(0);
+ assertEquals(path, storedPath);
+ }
+
+ @Test
+ public void testUploadToRemoteFileMultipleFiles() throws IOException {
+ File testFile1 = fileUtils.createFileInTempDir("testFile1");
+ File testFile2 = fileUtils.createFileInTempDir("testFile2");
+ Path path1 = fileSystemOperations.uploadToRemoteFile(stagingDir,
+ testFile1.getAbsolutePath());
+ Path path2 = fileSystemOperations.uploadToRemoteFile(stagingDir,
+ testFile2.getAbsolutePath());
+
+ File expectedFile1 = new File(new File(stagingDir.toString()),
"testFile1");
+ File expectedFile2 = new File(new File(stagingDir.toString()),
"testFile2");
+ assertEquals(expectedFile1.getAbsolutePath(), path1.toString());
+ assertEquals(expectedFile2.getAbsolutePath(), path2.toString());
+
+ Set<Path> uploadedFiles = fileSystemOperations.getUploadedFiles();
+ assertEquals(2, uploadedFiles.size());
+ List<Path> pathList = Lists.newArrayList(uploadedFiles);
+ Collections.sort(pathList);
+ Path storedPath1 = pathList.get(0);
+ Path storedPath2 = pathList.get(1);
+ assertEquals(path1, storedPath1);
+ assertEquals(path2, storedPath2);
+ }
+
+ @Test
+ public void testUploadToRemoteFileAndLocalizeMultipleFiles()
+ throws IOException {
+ Component comp = new Component();
+
+ File testFile1 = fileUtils.createFileInTempDir("testFile1");
+ File testFile2 = fileUtils.createFileInTempDir("testFile2");
+ fileSystemOperations.uploadToRemoteFileAndLocalizeToContainerWorkDir(
+ stagingDir, testFile1.getAbsolutePath(), "testFileDest1", comp);
+ fileSystemOperations.uploadToRemoteFileAndLocalizeToContainerWorkDir(
+ stagingDir, testFile2.getAbsolutePath(), "testFileDest2", comp);
+
+ List<ConfigFile> files = comp.getConfiguration().getFiles();
+ assertEquals(2, files.size());
+
+ ConfigFile configFile1 = files.get(0);
+ assertEquals(TypeEnum.STATIC, configFile1.getType());
+ assertEquals("testFileDest1", configFile1.getDestFile());
+ File expectedTestFile1 =
+ new File(new File(stagingDir.toString()), "testFile1");
+ assertEquals(FILE_SCHEME + expectedTestFile1.getAbsolutePath(),
+ configFile1.getSrcFile());
+
+ ConfigFile configFile2 = files.get(1);
+ assertEquals(TypeEnum.STATIC, configFile2.getType());
+ assertEquals("testFileDest2", configFile2.getDestFile());
+ File expectedTestFile2 =
+ new File(new File(stagingDir.toString()), "testFile2");
+ assertEquals(FILE_SCHEME + expectedTestFile2.getAbsolutePath(),
+ configFile2.getSrcFile());
+ }
+
+ @Test
+ public void testValidFileSize() throws IOException {
+ ClientContext clientContext = mock(ClientContext.class);
+
+ RemoteDirectoryManager remoteDirectoryManager =
+ mock(RemoteDirectoryManager.class);
+ when(clientContext.getRemoteDirectoryManager())
+ .thenReturn(remoteDirectoryManager);
+ when(remoteDirectoryManager.isRemote(anyString())).thenReturn(true);
+ when(remoteDirectoryManager.getRemoteFileSize(anyString()))
+ .thenReturn(20000L);
+
+ SubmarineConfiguration config =
+ SubmarineConfiguration.newInstance();
+ config.setLong(SubmarineConfiguration.ConfVars
+ .SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 21L);
+ when(clientContext.getSubmarineConfig()).thenReturn(config);
+
+ fileSystemOperations = new FileSystemOperations(clientContext);
+ }
+
+ @Test
+ public void testValidFileSizeInvalid() throws IOException {
+ ClientContext clientContext = mock(ClientContext.class);
+
+ RemoteDirectoryManager remoteDirectoryManager =
+ mock(RemoteDirectoryManager.class);
+ when(clientContext.getRemoteDirectoryManager())
+ .thenReturn(remoteDirectoryManager);
+ when(remoteDirectoryManager.isRemote(anyString())).thenReturn(true);
+ when(remoteDirectoryManager.getRemoteFileSize(anyString()))
+ .thenReturn(20000L);
+
+ SubmarineConfiguration config =
+ SubmarineConfiguration.newInstance();
+ config.setLong(SubmarineConfiguration.ConfVars
+ .SUBMARINE_LOCALIZATION_MAX_ALLOWED_FILE_SIZE_MB, 19L);
+ when(clientContext.getSubmarineConfig()).thenReturn(config);
+
+ fileSystemOperations = new FileSystemOperations(clientContext);
+ }
+}
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/command/AbstractTFLaunchCommandTestHelper.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/command/AbstractTFLaunchCommandTestHelper.java
index 366396e..23df04b 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/command/AbstractTFLaunchCommandTestHelper.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/command/AbstractTFLaunchCommandTestHelper.java
@@ -41,6 +41,7 @@ import java.util.Arrays;
import java.util.List;
import static junit.framework.TestCase.assertEquals;
+import static
org.apache.submarine.client.cli.yarnservice.YarnServiceRunJobCliCommonsTest.DEFAULT_JOB_NAME;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
@@ -104,7 +105,7 @@ public abstract class AbstractTFLaunchCommandTestHelper {
private AbstractLaunchCommand createLaunchCommandByTaskType(
TensorFlowRole taskType, TensorFlowRunJobParameters params)
throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
FileSystemOperations fsOperations =
new FileSystemOperations(mockClientContext);
HadoopEnvironmentSetup hadoopEnvSetup =
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/pytorch/PyTorchServiceSpecTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/pytorch/PyTorchServiceSpecTest.java
index 1d4bda3..2587e59 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/pytorch/PyTorchServiceSpecTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/pytorch/PyTorchServiceSpecTest.java
@@ -38,6 +38,7 @@ import org.junit.Test;
import java.io.IOException;
import java.util.List;
+import static
org.apache.submarine.client.cli.yarnservice.YarnServiceRunJobCliCommonsTest.DEFAULT_JOB_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
@@ -62,7 +63,7 @@ public class PyTorchServiceSpecTest {
parameters.setNumWorkers(1);
parameters.setWorkerLaunchCmd("testWorkerLaunchCommand");
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
FileSystemOperations fsOperations =
new FileSystemOperations(mockClientContext);
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorBoardLaunchCommandTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorBoardLaunchCommandTest.java
index e71804f..55ee758 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorBoardLaunchCommandTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorBoardLaunchCommandTest.java
@@ -32,6 +32,8 @@ import org.junit.Test;
import java.io.IOException;
import java.util.List;
+import static
org.apache.submarine.client.cli.yarnservice.YarnServiceRunJobCliCommonsTest.DEFAULT_JOB_NAME;
+
/**
* This class is to test the {@link TensorBoardLaunchCommand}.
*/
@@ -68,7 +70,7 @@ public class TensorBoardLaunchCommandTest extends
@Test
public void testCheckpointPathUndefined() throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
FileSystemOperations fsOperations =
new FileSystemOperations(mockClientContext);
HadoopEnvironmentSetup hadoopEnvSetup =
@@ -86,7 +88,7 @@ public class TensorBoardLaunchCommandTest extends
@Test
public void testCheckpointPathEmptyString() throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
FileSystemOperations fsOperations =
new FileSystemOperations(mockClientContext);
HadoopEnvironmentSetup hadoopEnvSetup =
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorFlowLaunchCommandTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorFlowLaunchCommandTest.java
index 735c29d..91101ae 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorFlowLaunchCommandTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/command/TensorFlowLaunchCommandTest.java
@@ -37,6 +37,8 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
+import static
org.apache.submarine.client.cli.yarnservice.YarnServiceRunJobCliCommonsTest.DEFAULT_JOB_NAME;
+
/**
* This class is to test the implementors of {@link TensorFlowLaunchCommand}.
*/
@@ -112,6 +114,10 @@ public class TensorFlowLaunchCommandTest
throw new IllegalStateException("Unknown tasktype!");
}
+ private MockClientContext createMockClientContext() throws IOException {
+ return new MockClientContext(DEFAULT_JOB_NAME);
+ }
+
@Test
public void testHdfsRelatedEnvironmentIsUndefined() throws IOException {
TensorFlowRunJobParameters params = new TensorFlowRunJobParameters();
@@ -133,15 +139,14 @@ public class TensorFlowLaunchCommandTest
setLaunchCommandToParams(params);
List<String> fileContents =
- testHdfsRelatedEnvironmentIsDefined(taskType,
- params);
+ testHdfsRelatedEnvironmentIsDefined(taskType, params);
assertScriptContainsLaunchCommand(fileContents, params);
assertScriptDoesNotContainLine(fileContents, "export TF_CONFIG=");
}
@Test
public void testLaunchCommandIsNull() throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = createMockClientContext();
FileSystemOperations fsOperations =
new FileSystemOperations(mockClientContext);
HadoopEnvironmentSetup hadoopEnvSetup =
@@ -164,7 +169,7 @@ public class TensorFlowLaunchCommandTest
@Test
public void testLaunchCommandIsEmpty() throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = createMockClientContext();
FileSystemOperations fsOperations =
new FileSystemOperations(mockClientContext);
HadoopEnvironmentSetup hadoopEnvSetup =
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/component/ComponentTestCommons.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/component/ComponentTestCommons.java
index 628c9a1..d6aecc7 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/component/ComponentTestCommons.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/component/ComponentTestCommons.java
@@ -27,6 +27,7 @@ import org.apache.submarine.commons.runtime.Framework;
import org.apache.submarine.commons.runtime.conf.Envs;
import org.apache.submarine.commons.runtime.MockClientContext;
import org.apache.submarine.commons.runtime.api.Role;
+import org.apache.submarine.commons.runtime.fs.MockRemoteDirectoryManager;
import org.apache.submarine.server.submitter.yarnservice.FileSystemOperations;
import
org.apache.submarine.server.submitter.yarnservice.command.AbstractLaunchCommand;
import
org.apache.submarine.server.submitter.yarnservice.command.LaunchCommandFactory;
@@ -35,6 +36,7 @@ import
org.apache.submarine.server.submitter.yarnservice.command.TensorFlowLaunc
import java.io.IOException;
+import static
org.apache.submarine.client.cli.yarnservice.YarnServiceRunJobCliCommonsTest.DEFAULT_JOB_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.mockito.ArgumentMatchers.any;
@@ -75,7 +77,7 @@ public class ComponentTestCommons {
private void setupDependencies(Framework framework) throws IOException {
fsOperations = mock(FileSystemOperations.class);
- mockClientContext = new MockClientContext();
+ mockClientContext = new MockClientContext(DEFAULT_JOB_NAME);
if (framework == Framework.TENSORFLOW) {
mockLaunchCommandFactory = mock(TensorFlowLaunchCommandFactory.class);
@@ -109,4 +111,11 @@ public class ComponentTestCommons {
assertEquals(4000,
(int) Integer.valueOf(component.getResource().getMemory()));
}
+
+ void makeRemoteDirectoryManagerHaveNullJobName() {
+ MockRemoteDirectoryManager mockRdm =
+ (MockRemoteDirectoryManager)
+ mockClientContext.getRemoteDirectoryManager();
+ mockRdm.setJobDir(null);
+ }
}
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/component/TensorBoardComponentTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/component/TensorBoardComponentTest.java
index 6ccdaf3..2ba44ef 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/component/TensorBoardComponentTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/tensorflow/component/TensorBoardComponentTest.java
@@ -83,9 +83,9 @@ public class TensorBoardComponentTest {
parameters.setTensorboardResource(testCommons.resource);
parameters.setName(null);
+ testCommons.makeRemoteDirectoryManagerHaveNullJobName();
TensorBoardComponent tensorBoardComponent =
createTensorBoardComponent(parameters);
-
expectedException.expect(NullPointerException.class);
expectedException.expectMessage("Job name must not be null");
tensorBoardComponent.createComponent();
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/KerberosPrincipalFactoryTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/KerberosPrincipalFactoryTest.java
index f99f98c..e1bda5b 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/KerberosPrincipalFactoryTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/KerberosPrincipalFactoryTest.java
@@ -32,6 +32,7 @@ import org.junit.Test;
import java.io.File;
import java.io.IOException;
+import static
org.apache.submarine.client.cli.yarnservice.YarnServiceRunJobCliCommonsTest.DEFAULT_JOB_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
@@ -60,7 +61,7 @@ public class KerberosPrincipalFactoryTest {
@Test
public void testCreatePrincipalEmptyPrincipalAndKeytab() throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
RunJobParameters parameters = mock(RunJobParameters.class);
when(parameters.getPrincipal()).thenReturn("");
@@ -76,7 +77,7 @@ public class KerberosPrincipalFactoryTest {
}
@Test
public void testCreatePrincipalEmptyPrincipalString() throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
RunJobParameters parameters = mock(RunJobParameters.class);
when(parameters.getPrincipal()).thenReturn("");
@@ -93,7 +94,7 @@ public class KerberosPrincipalFactoryTest {
@Test
public void testCreatePrincipalEmptyKeyTabString() throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
RunJobParameters parameters = mock(RunJobParameters.class);
when(parameters.getPrincipal()).thenReturn("principal");
@@ -111,7 +112,7 @@ public class KerberosPrincipalFactoryTest {
@Test
public void testCreatePrincipalNonEmptyPrincipalAndKeytab()
throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
RunJobParameters parameters = mock(RunJobParameters.class);
when(parameters.getPrincipal()).thenReturn("principal");
@@ -130,7 +131,7 @@ public class KerberosPrincipalFactoryTest {
@Test
public void testCreatePrincipalDistributedKeytab() throws IOException {
- MockClientContext mockClientContext = new MockClientContext();
+ MockClientContext mockClientContext = new
MockClientContext(DEFAULT_JOB_NAME);
String jobname = "testJobname";
String keytab = "testKeytab";
File keytabFile = createKeytabFile(keytab);
diff --git
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/LocalizerTest.java
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/LocalizerTest.java
index db2f54e..0a7886b 100644
---
a/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/LocalizerTest.java
+++
b/submarine-server/server-submitter/submitter-yarnservice/src/test/java/org/apache/submarine/server/submitter/yarnservice/utils/LocalizerTest.java
@@ -249,7 +249,7 @@ public class LocalizerTest {
String remoteUri = "hdfs://remotedir1/remotedir2";
when(fsOperations.uploadToRemoteFile(any(Path.class), anyString()))
.thenReturn(new Path(remoteUri));
- when(fsOperations.downloadAndZip(anyString(), anyString(), eq(true)))
+ when(fsOperations.downloadAndZip(anyString(), anyString()))
.thenReturn("remotedir2.zip");
Localization localization = new Localization();
@@ -271,7 +271,7 @@ public class LocalizerTest {
when(fsOperations.uploadToRemoteFile(any(Path.class), anyString()))
.thenReturn(new Path(remoteUri));
String zipFileName = "remotedir2_221424.zip";
- when(fsOperations.downloadAndZip(anyString(), anyString(), eq(true)))
+ when(fsOperations.downloadAndZip(anyString(), anyString()))
.thenReturn(zipFileName);
Localization localization = new Localization();
@@ -309,7 +309,7 @@ public class LocalizerTest {
when(fsOperations.uploadToRemoteFile(any(Path.class), anyString()))
.thenReturn(new Path(remoteUri));
String downloadedFileName = "remotedir2_221424";
- when(fsOperations.downloadAndZip(anyString(), anyString(), eq(false)))
+ when(fsOperations.download(anyString(), anyString()))
.thenReturn(downloadedFileName);
Localization localization = new Localization();
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
