MAPREDUCE-6881. Fix warnings from Spotbugs in hadoop-mapreduce. Contributed by Weiwei Yang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3ed3062f Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3ed3062f Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3ed3062f Branch: refs/heads/HDFS-10467 Commit: 3ed3062fe3979ff55a411b730a8eee2b2c96d6b3 Parents: 28eb2aa Author: Akira Ajisaka <aajis...@apache.org> Authored: Thu Apr 27 15:45:33 2017 +0900 Committer: Akira Ajisaka <aajis...@apache.org> Committed: Thu Apr 27 15:45:33 2017 +0900 ---------------------------------------------------------------------- .../hadoop/mapred/LocalContainerLauncher.java | 46 ++++++++++++-------- .../hadoop/mapreduce/v2/app/MRAppMaster.java | 14 +++--- .../java/org/apache/hadoop/mapred/JVMId.java | 2 +- .../org/apache/hadoop/mapred/Operation.java | 14 ++++-- .../mapreduce/v2/hs/HistoryFileManager.java | 5 --- .../org/apache/hadoop/examples/pi/Parser.java | 8 +++- 6 files changed, 53 insertions(+), 36 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java index 190d988..0b942b0 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java @@ -27,6 +27,8 @@ import java.lang.management.ThreadMXBean; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Set; +import java.util.Collections; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; @@ -81,7 +83,7 @@ public class LocalContainerLauncher extends AbstractService implements private static final Log LOG = LogFactory.getLog(LocalContainerLauncher.class); private FileContext curFC = null; - private final HashSet<File> localizedFiles; + private Set<File> localizedFiles = new HashSet<File>(); private final AppContext context; private final TaskUmbilicalProtocol umbilical; private final ClassLoader jobClassLoader; @@ -121,9 +123,12 @@ public class LocalContainerLauncher extends AbstractService implements // users who do that get what they deserve (and will have to disable // uberization in order to run correctly). File[] curLocalFiles = curDir.listFiles(); - localizedFiles = new HashSet<File>(curLocalFiles.length); - for (int j = 0; j < curLocalFiles.length; ++j) { - localizedFiles.add(curLocalFiles[j]); + if (curLocalFiles != null) { + HashSet<File> lf = new HashSet<File>(curLocalFiles.length); + for (int j = 0; j < curLocalFiles.length; ++j) { + lf.add(curLocalFiles[j]); + } + localizedFiles = Collections.unmodifiableSet(lf); } // Relocalization note/future FIXME (per chrisdo, 20110315): At moment, @@ -521,26 +526,29 @@ public class LocalContainerLauncher extends AbstractService implements */ private void relocalize() { File[] curLocalFiles = curDir.listFiles(); - for (int j = 0; j < curLocalFiles.length; ++j) { - if (!localizedFiles.contains(curLocalFiles[j])) { - // found one that wasn't there before: delete it - boolean deleted = false; - try { - if (curFC != null) { - // this is recursive, unlike File delete(): - deleted = curFC.delete(new Path(curLocalFiles[j].getName()),true); + if (curLocalFiles != null) { + for (int j = 0; j < curLocalFiles.length; ++j) { + if (!localizedFiles.contains(curLocalFiles[j])) { + // found one that wasn't there before: delete it + boolean deleted = false; + try { + if (curFC != null) { + // this is recursive, unlike File delete(): + deleted = + curFC.delete(new Path(curLocalFiles[j].getName()), true); + } + } catch (IOException e) { + deleted = false; + } + if (!deleted) { + LOG.warn("Unable to delete unexpected local file/dir " + + curLocalFiles[j].getName() + + ": insufficient permissions?"); } - } catch (IOException e) { - deleted = false; - } - if (!deleted) { - LOG.warn("Unable to delete unexpected local file/dir " - + curLocalFiles[j].getName() + ": insufficient permissions?"); } } } } - } // end EventHandler /** http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java index bb7abb6..ba7f929 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java @@ -572,13 +572,15 @@ public class MRAppMaster extends CompositeService { private boolean isJobNamePatternMatch(JobConf conf, String jobTempDir) { // Matched staging files should be preserved after job is finished. if (conf.getKeepTaskFilesPattern() != null && jobTempDir != null) { - String jobFileName = Paths.get(jobTempDir).getFileName().toString(); - Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern()); - Matcher matcher = pattern.matcher(jobFileName); - return matcher.find(); - } else { - return false; + java.nio.file.Path pathName = Paths.get(jobTempDir).getFileName(); + if (pathName != null) { + String jobFileName = pathName.toString(); + Pattern pattern = Pattern.compile(conf.getKeepTaskFilesPattern()); + Matcher matcher = pattern.matcher(jobFileName); + return matcher.find(); + } } + return false; } private boolean isKeepFailedTaskFiles(JobConf conf) { http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java index 77a7f8a..6e0e1c4 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/JVMId.java @@ -98,7 +98,7 @@ class JVMId { int jobComp = this.jobId.compareTo(that.jobId); if(jobComp == 0) { if(this.isMap == that.isMap) { - return Long.valueOf(this.jvmId).compareTo(that.jvmId); + return Long.compare(this.jvmId, that.jvmId); } else { return this.isMap ? -1 : 1; } http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java index 4df9be4..a5552f8 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapred/Operation.java @@ -34,12 +34,20 @@ public enum Operation { KILL_TASK(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB), SET_JOB_PRIORITY(QueueACL.ADMINISTER_JOBS, JobACL.MODIFY_JOB), SUBMIT_JOB(QueueACL.SUBMIT_JOB, null); - - public QueueACL qACLNeeded; - public JobACL jobACLNeeded; + + private final QueueACL qACLNeeded; + private final JobACL jobACLNeeded; Operation(QueueACL qACL, JobACL jobACL) { this.qACLNeeded = qACL; this.jobACLNeeded = jobACL; } + + public QueueACL getqACLNeeded() { + return qACLNeeded; + } + + public JobACL getJobACLNeeded() { + return jobACLNeeded; + } } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java index bd8d9c2..6b98e47 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java @@ -407,17 +407,12 @@ public class HistoryFileManager extends AbstractService { } JobId jobId = jobIndexInfo.getJobId(); - List<Path> paths = new ArrayList<Path>(2); if (historyFile == null) { LOG.info("No file for job-history with " + jobId + " found in cache!"); - } else { - paths.add(historyFile); } if (confFile == null) { LOG.info("No file for jobConf with " + jobId + " found in cache!"); - } else { - paths.add(confFile); } if (summaryFile == null || !intermediateDoneDirFc.util().exists( http://git-wip-us.apache.org/repos/asf/hadoop/blob/3ed3062f/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java ---------------------------------------------------------------------- diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java index a2db9d1..60b3638 100644 --- a/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java +++ b/hadoop-mapreduce-project/hadoop-mapreduce-examples/src/main/java/org/apache/hadoop/examples/pi/Parser.java @@ -67,8 +67,12 @@ public final class Parser { private void parse(File f, Map<Parameter, List<TaskResult>> sums) throws IOException { if (f.isDirectory()) { println("Process directory " + f); - for(File child : f.listFiles()) - parse(child, sums); + File[] files = f.listFiles(); + if (files != null) { + for(File child : files) { + parse(child, sums); + } + } } else if (f.getName().endsWith(".txt")) { println("Parse file " + f); final Map<Parameter, List<TaskResult>> m = new TreeMap<Parameter, List<TaskResult>>(); --------------------------------------------------------------------- To unsubscribe, e-mail: common-commits-unsubscr...@hadoop.apache.org For additional commands, e-mail: common-commits-h...@hadoop.apache.org
