This is an automated email from the ASF dual-hosted git repository.
jeagles pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/tez.git
The following commit(s) were added to refs/heads/master by this push:
new 7a3e378 TEZ-4082. Reduce excessive getFileLinkInfo calls in Tez
7a3e378 is described below
commit 7a3e378b59dad2afe1a068669620846b87d6e732
Author: Jonathan Eagles <[email protected]>
AuthorDate: Tue Aug 27 10:40:51 2019 -0500
TEZ-4082. Reduce excessive getFileLinkInfo calls in Tez
---
.../java/org/apache/tez/client/TezClientUtils.java | 57 ++++++++--------------
.../java/org/apache/tez/common/TezCommonUtils.java | 17 +------
.../org/apache/tez/common/TestTezCommonUtils.java | 36 ++++++++++----
tez-ui/src/main/webapp/bower-shrinkwrap.json | 44 ++++++++---------
4 files changed, 70 insertions(+), 84 deletions(-)
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index cd3ae6b..2b21024 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -19,6 +19,7 @@
package org.apache.tez.client;
import java.io.File;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStream;
import java.net.InetSocketAddress;
@@ -121,31 +122,6 @@ public class TezClientUtils {
private static Logger LOG = LoggerFactory.getLogger(TezClientUtils.class);
private static final int UTF8_CHUNK_SIZE = 16 * 1024;
- private static FileStatus[] getLRFileStatus(String fileName, Configuration
conf) throws
- IOException {
- URI uri;
- try {
- uri = new URI(fileName);
- } catch (URISyntaxException e) {
- String message = "Invalid URI defined in configuration for"
- + " location of TEZ jars. providedURI=" + fileName;
- LOG.error(message);
- throw new TezUncheckedException(message, e);
- }
-
- Path p = new Path(uri);
- FileSystem fs = p.getFileSystem(conf);
- p = fs.resolvePath(p.makeQualified(fs.getUri(),
- fs.getWorkingDirectory()));
- FileSystem targetFS = p.getFileSystem(conf);
- if (targetFS.isDirectory(p)) {
- return targetFS.listStatus(p);
- } else {
- FileStatus fStatus = targetFS.getFileStatus(p);
- return new FileStatus[]{fStatus};
- }
- }
-
/**
* Setup LocalResource map for Tez jars based on provided Configuration
*
@@ -217,8 +193,16 @@ public class TezClientUtils {
}
Path p = new Path(u);
FileSystem remoteFS = p.getFileSystem(conf);
- p = remoteFS.resolvePath(p.makeQualified(remoteFS.getUri(),
- remoteFS.getWorkingDirectory()));
+ FileStatus targetStatus = remoteFS.getFileLinkStatus(p);
+ p = targetStatus.getPath();
+
+ FileStatus[] fileStatuses;
+ FileSystem targetFS = p.getFileSystem(conf);
+ if (targetStatus.isDirectory()) {
+ fileStatuses = targetFS.listStatus(p);
+ } else {
+ fileStatuses = new FileStatus[]{targetStatus};
+ }
LocalResourceType type = null;
@@ -232,8 +216,6 @@ public class TezClientUtils {
type = LocalResourceType.FILE;
}
- FileStatus [] fileStatuses = getLRFileStatus(configUri, conf);
-
for (FileStatus fStatus : fileStatuses) {
String linkName;
if (fStatus.isDirectory()) {
@@ -329,13 +311,16 @@ public class TezClientUtils {
Path stagingArea)
throws IOException {
FileSystem fs = stagingArea.getFileSystem(conf);
- String realUser;
- String currentUser;
UserGroupInformation ugi = UserGroupInformation.getLoginUser();
- realUser = ugi.getShortUserName();
- currentUser = UserGroupInformation.getCurrentUser().getShortUserName();
- if (fs.exists(stagingArea)) {
- FileStatus fsStatus = fs.getFileStatus(stagingArea);
+ String realUser = ugi.getShortUserName();
+ String currentUser =
UserGroupInformation.getCurrentUser().getShortUserName();
+ FileStatus fsStatus = null;
+ try {
+ fsStatus = fs.getFileStatus(stagingArea);
+ } catch (FileNotFoundException e) {
+ TezCommonUtils.mkDirForAM(fs, stagingArea);
+ }
+ if (fsStatus != null) {
String owner = fsStatus.getOwner();
if (!(owner.equals(currentUser) || owner.equals(realUser))) {
throw new IOException("The ownership on the staging directory "
@@ -350,8 +335,6 @@ public class TezClientUtils {
+ TezCommonUtils.TEZ_AM_DIR_PERMISSION);
fs.setPermission(stagingArea, TezCommonUtils.TEZ_AM_DIR_PERMISSION);
}
- } else {
- TezCommonUtils.mkDirForAM(fs, stagingArea);
}
return fs;
}
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
index 16165e2..fc4789f 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezCommonUtils.java
@@ -79,21 +79,8 @@ public class TezCommonUtils {
* @return Fully qualified staging directory
*/
public static Path getTezBaseStagingPath(Configuration conf) {
- String stagingDirStr = conf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
- TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT);
- Path baseStagingDir;
- try {
- Path p = new Path(stagingDirStr);
- FileSystem fs = p.getFileSystem(conf);
- if (!fs.exists(p)) {
- mkDirForAM(fs, p);
- LOG.info("Stage directory " + p + " doesn't exist and is created");
- }
- baseStagingDir = fs.resolvePath(p);
- } catch (IOException e) {
- throw new TezUncheckedException(e);
- }
- return baseStagingDir;
+ return new Path(conf.get(TezConfiguration.TEZ_AM_STAGING_DIR,
+ TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT));
}
/**
diff --git
a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
index 3929c4b..52df2c6 100644
--- a/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
+++ b/tez-api/src/test/java/org/apache/tez/common/TestTezCommonUtils.java
@@ -57,7 +57,7 @@ public class TestTezCommonUtils {
LOG.info("Starting mini clusters");
try {
conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, TEST_ROOT_DIR);
- dfsCluster = new
MiniDFSCluster.Builder(conf).numDataNodes(3).format(true).racks(null)
+ dfsCluster = new
MiniDFSCluster.Builder(conf).numDataNodes(1).format(true).racks(null)
.build();
remoteFs = dfsCluster.getFileSystem();
RESOLVED_STAGE_DIR = remoteFs.getUri() + STAGE_DIR;
@@ -80,19 +80,23 @@ public class TestTezCommonUtils {
}
// Testing base staging dir
- @Test(timeout = 5000)
+ @Test
public void testTezBaseStagingPath() throws Exception {
Configuration localConf = new Configuration();
// Check if default works with localFS
localConf.unset(TezConfiguration.TEZ_AM_STAGING_DIR);
localConf.set("fs.defaultFS", "file:///");
Path stageDir = TezCommonUtils.getTezBaseStagingPath(localConf);
- Assert.assertEquals(stageDir.toString(), "file:" +
TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT);
+ TezCommonUtils.mkDirForAM(stageDir.getFileSystem(localConf), stageDir);
+ Path resolveStageDir =
stageDir.getFileSystem(localConf).resolvePath(stageDir);
+ Assert.assertEquals(resolveStageDir.toString(), "file:" +
TezConfiguration.TEZ_AM_STAGING_DIR_DEFAULT);
// check if user set something, indeed works
conf.set(TezConfiguration.TEZ_AM_STAGING_DIR, STAGE_DIR);
stageDir = TezCommonUtils.getTezBaseStagingPath(conf);
- Assert.assertEquals(stageDir.toString(), RESOLVED_STAGE_DIR);
+ TezCommonUtils.mkDirForAM(stageDir.getFileSystem(conf), stageDir);
+ resolveStageDir = stageDir.getFileSystem(conf).resolvePath(stageDir);
+ Assert.assertEquals(resolveStageDir.toString(), RESOLVED_STAGE_DIR);
}
// Testing System staging dir if createed
@@ -111,7 +115,8 @@ public class TestTezCommonUtils {
}
Assert.assertFalse(fs.exists(stagePath));
Path stageDir = TezCommonUtils.createTezSystemStagingPath(conf, strAppId);
- Assert.assertEquals(stageDir.toString(), expectedStageDir);
+ Path resolveStageDir = stageDir.getFileSystem(conf).resolvePath(stageDir);
+ Assert.assertEquals(resolveStageDir.toString(), expectedStageDir);
Assert.assertTrue(fs.exists(stagePath));
}
@@ -122,7 +127,9 @@ public class TestTezCommonUtils {
Path stageDir = TezCommonUtils.getTezSystemStagingPath(conf, strAppId);
String expectedStageDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId;
- Assert.assertEquals(stageDir.toString(), expectedStageDir);
+ TezCommonUtils.mkDirForAM(stageDir.getFileSystem(conf), stageDir);
+ Path resolvedStageDir = stageDir.getFileSystem(conf).resolvePath(stageDir);
+ Assert.assertEquals(resolvedStageDir.toString(), expectedStageDir);
}
// Testing conf staging dir
@@ -134,7 +141,9 @@ public class TestTezCommonUtils {
String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId +
Path.SEPARATOR
+ TezConstants.TEZ_PB_BINARY_CONF_NAME;
- Assert.assertEquals(confStageDir.toString(), expectedDir);
+ TezCommonUtils.mkDirForAM(confStageDir.getFileSystem(conf), confStageDir);
+ Path resolvedConfStageDir =
confStageDir.getFileSystem(conf).resolvePath(confStageDir);
+ Assert.assertEquals(resolvedConfStageDir.toString(), expectedDir);
}
// Testing session jars staging dir
@@ -146,7 +155,9 @@ public class TestTezCommonUtils {
String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId +
Path.SEPARATOR
+ TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME;
- Assert.assertEquals(confStageDir.toString(), expectedDir);
+ TezCommonUtils.mkDirForAM(confStageDir.getFileSystem(conf), confStageDir);
+ Path resolvedConfStageDir =
confStageDir.getFileSystem(conf).resolvePath(confStageDir);
+ Assert.assertEquals(resolvedConfStageDir.toString(), expectedDir);
}
// Testing bin plan staging dir
@@ -158,7 +169,9 @@ public class TestTezCommonUtils {
String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId +
Path.SEPARATOR
+ TezConstants.TEZ_PB_PLAN_BINARY_NAME;
- Assert.assertEquals(confStageDir.toString(), expectedDir);
+ TezCommonUtils.mkDirForAM(confStageDir.getFileSystem(conf), confStageDir);
+ Path resolvedConfStageDir =
confStageDir.getFileSystem(conf).resolvePath(confStageDir);
+ Assert.assertEquals(resolvedConfStageDir.toString(), expectedDir);
}
// Testing text plan staging dir
@@ -167,12 +180,15 @@ public class TestTezCommonUtils {
String strAppId = "testAppId";
String dagPBName = "testDagPBName";
Path tezSysStagingPath = TezCommonUtils.getTezSystemStagingPath(conf,
strAppId);
+ TezCommonUtils.mkDirForAM(tezSysStagingPath.getFileSystem(conf),
tezSysStagingPath);
Path confStageDir =
TezCommonUtils.getTezTextPlanStagingPath(tezSysStagingPath, strAppId,
dagPBName);
String expectedDir = RESOLVED_STAGE_DIR + Path.SEPARATOR
+ TezCommonUtils.TEZ_SYSTEM_SUB_DIR + Path.SEPARATOR + strAppId +
Path.SEPARATOR
+ strAppId + "-" + dagPBName + "-" +
TezConstants.TEZ_PB_PLAN_TEXT_NAME;
- Assert.assertEquals(confStageDir.toString(), expectedDir);
+ TezCommonUtils.createFileForAM(confStageDir.getFileSystem(conf),
confStageDir);
+ Path resolvedConfStageDir =
confStageDir.getFileSystem(conf).resolvePath(confStageDir);
+ Assert.assertEquals(resolvedConfStageDir.toString(), expectedDir);
}
// Testing recovery path staging dir
diff --git a/tez-ui/src/main/webapp/bower-shrinkwrap.json
b/tez-ui/src/main/webapp/bower-shrinkwrap.json
index 357d576..edcbae3 100644
--- a/tez-ui/src/main/webapp/bower-shrinkwrap.json
+++ b/tez-ui/src/main/webapp/bower-shrinkwrap.json
@@ -1,72 +1,72 @@
{
"https://github.com/FortAwesome/Font-Awesome.git": {
- "4.5.0": "593ad563a987977f14102be935d0abc2a172903e"
+ "4.5.0": "4.5.0"
},
"https://github.com/Teleborder/FileSaver.js.git": {
- "1.20150507.2": "b7cf622909258086bc63ad764d08fcaed780ab42"
+ "1.20150507.2": "1.20150507.2"
},
"https://github.com/adamwdraper/Numeral-js.git": {
- "1.5.3": "f97f14bb8bab988f28f1d854525b4cfeff8ec9e1"
+ "1.5.3": "1.5.3"
},
"https://github.com/components/codemirror.git": {
- "5.11.0": "7d43f32bb56f83a9c47addb3f91170b3102f3ead"
+ "5.11.0": "5.11.0"
},
"https://github.com/components/ember-data.git": {
- "2.1.0": "d8b4d3092f67afe22d9d374c40d719d557915fa3"
+ "2.1.0": "2.1.0"
},
"https://github.com/components/ember.git": {
- "2.2.0": "49e042ca89922ed96b27488c2a98add280ae7123"
+ "2.2.0": "2.2.0"
},
"https://github.com/components/jqueryui.git": {
- "1.11.4": "c34f8dbf3ba57b3784b93f26119f436c0e8288e1"
+ "1.11.4": "1.11.4"
},
"https://github.com/dockyard/ember-qunit-notifications.git": {
- "0.1.0": "a83277aa7a1c0545c66e6d133caebb9a620e71ad"
+ "0.1.0": "0.1.0"
},
"https://github.com/dockyard/qunit-notifications.git": {
- "0.1.1": "7a13f6dba5a340e1cb9e0b64c1c711e4d7edaca1"
+ "0.1.1": "0.1.1"
},
"https://github.com/ember-cli/ember-cli-shims.git": {
- "0.0.6": "dcab43b58d5698690050bb9a46ead5c8663c7da1"
+ "0.0.6": "0.0.6"
},
"https://github.com/ember-cli/ember-cli-test-loader.git": {
- "0.2.1": "3348d801089279296c38f31ae14d9c4d115ce154"
+ "0.2.1": "0.2.1"
},
"https://github.com/ember-cli/ember-load-initializers.git": {
- "0.1.7": "7bb21488563bd1bba23e903a812bf5815beddd1a"
+ "0.1.7": "0.1.7"
},
"https://github.com/jquery/jquery-dist.git": {
- "2.1.4": "7751e69b615c6eca6f783a81e292a55725af6b85"
+ "2.1.4": "2.1.4"
},
"https://github.com/jquery/jquery-mousewheel.git": {
- "3.1.13": "67289b6b2aa0066d7d78a5807f520387135ffb22"
+ "3.1.13": "3.1.13"
},
"https://github.com/jquery/qunit.git": {
- "1.19.0": "467e7e34652ad7d5883ce9c568461cf8c5e172a8"
+ "1.19.0": "1.19.0"
},
"https://github.com/moment/moment-timezone.git": {
- "0.5.0": "74a2e9378ecf4a31a168f3049f086565c8d66814"
+ "0.5.0": "0.5.0"
},
"https://github.com/moment/moment.git": {
- "2.12.0": "d3d7488b4d60632854181cb0a9af325d57fb3d51"
+ "2.12.0": "2.12.0"
},
"https://github.com/rwjblue/ember-qunit-builds.git": {
- "0.4.16": "142c4066a5458bef9dfcb92b70152b9c01d79188"
+ "0.4.16": "0.4.16"
},
"https://github.com/sreenaths/more-js.git": {
"0.8.6": "f1d9ccdaf7ff74c26b6ee341067a5a5ef33bd64a",
"0.8.8": "0.8.8"
},
"https://github.com/sreenaths/snippet-ss.git": {
- "1.11.0": "c1abc566f4e001b7f1939b6dbdd911eadc969cf9"
+ "1.11.0": "1.11.0"
},
"https://github.com/sreenaths/zip.js.git": {
- "1.0.0": "ec67ad22eba116083ea3ef2fe0b40ccc953513ce"
+ "1.0.0": "1.0.0"
},
"https://github.com/stefanpenner/loader.js.git": {
- "3.3.0": "ac909550c9544325632542bbea97531cc60bc628"
+ "3.3.0": "3.3.0"
},
"https://github.com/twbs/bootstrap.git": {
- "3.3.6": "81df608a40bf0629a1dc08e584849bb1e43e0b7a"
+ "3.3.6": "3.3.6"
}
}
\ No newline at end of file
