Repository: oozie Updated Branches: refs/heads/master 9937e4c4a -> 678f1b2b2
OOZIE-2943 Fix Findbugs warnings in oozie-sharelib-pig (Jan Hentschel via gezapeti) Project: http://git-wip-us.apache.org/repos/asf/oozie/repo Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/678f1b2b Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/678f1b2b Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/678f1b2b Branch: refs/heads/master Commit: 678f1b2b2cc9a2456c7e3d9f693d2637f1f03257 Parents: 9937e4c Author: Gezapeti Cseh <[email protected]> Authored: Tue Jun 27 11:58:22 2017 +0200 Committer: Gezapeti Cseh <[email protected]> Committed: Tue Jun 27 11:58:22 2017 +0200 ---------------------------------------------------------------------- release-log.txt | 1 + .../oozie/action/hadoop/OoziePigStats.java | 8 +- .../org/apache/oozie/action/hadoop/PigMain.java | 33 ++++----- .../oozie/action/hadoop/PigMainWithOldAPI.java | 78 +++++++++++--------- 4 files changed, 61 insertions(+), 59 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/oozie/blob/678f1b2b/release-log.txt ---------------------------------------------------------------------- diff --git a/release-log.txt b/release-log.txt index f862b39..97884b6 100644 --- a/release-log.txt +++ b/release-log.txt @@ -1,5 +1,6 @@ -- Oozie 5.0.0 release (trunk - unreleased) +OOZIE-2943 Fix Findbugs warnings in oozie-sharelib-pig (Jan Hentschel via gezapeti) OOZIE-2846 TestSSLServerConnectorFactory fails with Jetty 9.3 and 9.4 (gezapeti) OOZIE-2936 Fix Javadoc warnings from build (Jan Hentschel via gezapeti) OOZIE-2920 Document Distcp can copy files within a cluster (Artem Ervits via rkanter) http://git-wip-us.apache.org/repos/asf/oozie/blob/678f1b2b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java index c3a31b5..b2fcfc4 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/OoziePigStats.java @@ -119,18 +119,16 @@ public class OoziePigStats extends ActionStats { jobStatsGroup.put("MULTI_STORE_COUNTERS", toJSONFromMultiStoreCounters(jobStats.getMultiStoreCounters())); return jobStatsGroup; - } // multistorecounters to JSON @SuppressWarnings("unchecked") private static JSONObject toJSONFromMultiStoreCounters(Map<String, Long> map) { JSONObject group = new JSONObject(); - for (String cName : map.keySet()) { - group.put(cName, map.get(cName)); + for (Map.Entry cName : map.entrySet()) { + group.put(cName.getKey(), cName.getValue()); } return group; - } // hadoop counters to JSON @@ -151,7 +149,5 @@ public class OoziePigStats extends ActionStats { groups.put(gName, group); } return groups; - } - } http://git-wip-us.apache.org/repos/asf/oozie/blob/678f1b2b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java index 5a9123a..b6b30eb 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMain.java @@ -33,15 +33,14 @@ import org.apache.hadoop.fs.Path; import java.io.BufferedWriter; import java.io.FileNotFoundException; -import java.io.FileWriter; import java.io.OutputStream; import java.io.FileOutputStream; import java.io.BufferedReader; -import java.io.FileReader; import java.io.File; import java.io.FileInputStream; import java.io.IOException; import java.io.InputStreamReader; +import java.io.OutputStreamWriter; import java.util.Arrays; import java.util.HashSet; import java.util.Map; @@ -52,7 +51,7 @@ import java.util.Set; import java.util.regex.Pattern; public class PigMain extends LauncherMain { - private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>(); + private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<>(); public static final int STRING_BUFFER_SIZE = 100; public static final String LOG_EXPANDED_PIG_SCRIPT = LauncherMapper.ACTION_PREFIX + "pig.log.expandedscript"; @@ -127,14 +126,14 @@ public class PigMain extends LauncherMain { String callerId = "oozie:" + System.getProperty("oozie.job.id"); pigProperties.setProperty("pig.log.trace.id", callerId); - OutputStream os = new FileOutputStream("pig.properties"); - pigProperties.store(os, ""); - os.close(); + try (OutputStream os = new FileOutputStream("pig.properties")) { + pigProperties.store(os, ""); + } logMasking("pig.properties:", Arrays.asList("password"), (Iterable<Map.Entry<String, String>>)(Iterable<?>) pigProperties.entrySet()); - List<String> arguments = new ArrayList<String>(); + List<String> arguments = new ArrayList<>(); String script = actionConf.get(PigActionExecutor.PIG_SCRIPT); if (script == null) { @@ -247,7 +246,7 @@ public class PigMain extends LauncherMain { * @param arguments */ private void logExpandedScript(String script, List<String> arguments) { - List<String> dryrunArgs = new ArrayList<String>(); + List<String> dryrunArgs = new ArrayList<>(); dryrunArgs.addAll(arguments); dryrunArgs.add("-dryrun"); try { @@ -304,13 +303,13 @@ public class PigMain extends LauncherMain { System.err.println("Pig logfile dump:"); System.err.println(); try { - BufferedReader reader = new BufferedReader(new FileReader(pigLog)); - String line = reader.readLine(); - while (line != null) { - System.err.println(line); - line = reader.readLine(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(pigLog), "UTF-8"))) { + String line = reader.readLine(); + while (line != null) { + System.err.println(line); + line = reader.readLine(); + } } - reader.close(); } catch (FileNotFoundException e) { System.err.println("pig log file: " + pigLog + " not found."); @@ -330,9 +329,9 @@ public class PigMain extends LauncherMain { protected void runPigJob(String[] args, String pigLog, boolean resetSecurityManager, boolean retrieveStats) throws Exception { // running as from the command line boolean pigRunnerExists = true; - Class klass; + try { - klass = Class.forName("org.apache.pig.PigRunner"); + Class.forName("org.apache.pig.PigRunner"); } catch (ClassNotFoundException ex) { pigRunnerExists = false; @@ -405,7 +404,7 @@ public class PigMain extends LauncherMain { private static void writeExternalData(String data, File f) throws IOException { BufferedWriter out = null; try { - out = new BufferedWriter(new FileWriter(f)); + out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(f), "UTF-8")); out.write(data); } finally { http://git-wip-us.apache.org/repos/asf/oozie/blob/678f1b2b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java ---------------------------------------------------------------------- diff --git a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java index 0ee4b0b..df31655 100644 --- a/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java +++ b/sharelib/pig/src/main/java/org/apache/oozie/action/hadoop/PigMainWithOldAPI.java @@ -22,11 +22,12 @@ import org.apache.pig.Main; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import java.io.FileInputStream; import java.io.FileNotFoundException; +import java.io.InputStreamReader; import java.io.OutputStream; import java.io.FileOutputStream; import java.io.BufferedReader; -import java.io.FileReader; import java.io.File; import java.io.IOException; import java.util.HashSet; @@ -37,7 +38,7 @@ import java.util.Properties; import java.util.Set; public class PigMainWithOldAPI extends LauncherMain { - private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<String>(); + private static final Set<String> DISALLOWED_PIG_OPTIONS = new HashSet<>(); static { DISALLOWED_PIG_OPTIONS.add("-4"); @@ -98,9 +99,9 @@ public class PigMainWithOldAPI extends LauncherMain { System.out.println("Non-kerberos execution"); } - OutputStream os = new FileOutputStream("pig.properties"); - pigProperties.store(os, ""); - os.close(); + try (OutputStream os = new FileOutputStream("pig.properties")) { + pigProperties.store(os, ""); + } System.out.println(); System.out.println("pig.properties content:"); @@ -110,7 +111,7 @@ public class PigMainWithOldAPI extends LauncherMain { System.out.println("------------------------"); System.out.println(); - List<String> arguments = new ArrayList<String>(); + List<String> arguments = new ArrayList<>(); String script = actionConf.get("oozie.pig.script"); if (script == null) { @@ -123,13 +124,16 @@ public class PigMainWithOldAPI extends LauncherMain { System.out.println("Pig script [" + script + "] content: "); System.out.println("------------------------"); - BufferedReader br = new BufferedReader(new FileReader(script)); - String line = br.readLine(); - while (line != null) { - System.out.println(line); + + String line; + try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(script), "UTF-8"))) { line = br.readLine(); + while (line != null) { + System.out.println(line); + line = br.readLine(); + } } - br.close(); + System.out.println("------------------------"); System.out.println(); @@ -216,13 +220,14 @@ public class PigMainWithOldAPI extends LauncherMain { System.err.println("Pig logfile dump:"); System.err.println(); try { - BufferedReader reader = new BufferedReader(new FileReader(pigLog)); - line = reader.readLine(); - while (line != null) { - System.err.println(line); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(new FileInputStream(pigLog), + "UTF-8"))) { line = reader.readLine(); + while (line != null) { + System.err.println(line); + line = reader.readLine(); + } } - reader.close(); } catch (FileNotFoundException e) { System.err.println("pig log file: " + pigLog + " not found."); @@ -239,9 +244,11 @@ public class PigMainWithOldAPI extends LauncherMain { // harvesting and recording Hadoop Job IDs Properties jobIds = getHadoopJobIds(logFile); File file = new File(System.getProperty(OUTPUT_PROPERTIES)); - os = new FileOutputStream(file); - jobIds.store(os, ""); - os.close(); + + try (OutputStream os = new FileOutputStream(file)) { + jobIds.store(os, ""); + } + System.out.println(" Hadoop Job IDs executed by Pig: " + jobIds.getProperty(HADOOP_JOBS)); System.out.println(); } @@ -254,34 +261,33 @@ public class PigMainWithOldAPI extends LauncherMain { private static final String JOB_ID_LOG_PREFIX = "HadoopJobId: "; protected Properties getHadoopJobIds(String logFile) throws IOException { - int jobCount = 0; Properties props = new Properties(); StringBuffer sb = new StringBuffer(100); - if (new File(logFile).exists() == false) { + if (!new File(logFile).exists()) { System.err.println("pig log file: " + logFile + " not present. Therefore no Hadoop jobids found"); props.setProperty(HADOOP_JOBS, ""); } else { - BufferedReader br = new BufferedReader(new FileReader(logFile)); - String line = br.readLine(); - String separator = ""; - while (line != null) { - if (line.contains(JOB_ID_LOG_PREFIX)) { - int jobIdStarts = line.indexOf(JOB_ID_LOG_PREFIX) + JOB_ID_LOG_PREFIX.length(); - String jobId = line.substring(jobIdStarts); - int jobIdEnds = jobId.indexOf(" "); - if (jobIdEnds > -1) { - jobId = jobId.substring(0, jobId.indexOf(" ")); + try (BufferedReader br = new BufferedReader(new InputStreamReader(new FileInputStream(logFile), "UTF-8"))) { + String line = br.readLine(); + String separator = ""; + while (line != null) { + if (line.contains(JOB_ID_LOG_PREFIX)) { + int jobIdStarts = line.indexOf(JOB_ID_LOG_PREFIX) + JOB_ID_LOG_PREFIX.length(); + String jobId = line.substring(jobIdStarts); + int jobIdEnds = jobId.indexOf(" "); + if (jobIdEnds > -1) { + jobId = jobId.substring(0, jobId.indexOf(" ")); + } + sb.append(separator).append(jobId); + separator = ","; } - sb.append(separator).append(jobId); - separator = ","; + line = br.readLine(); } - line = br.readLine(); } - br.close(); + props.setProperty(HADOOP_JOBS, sb.toString()); } return props; } - }
