Repository: sqoop Updated Branches: refs/heads/sqoop2 b151ba384 -> 9f076edb0
SQOOP-2516: Sqoop2: Findbugs: Fix warning in execution module (Colin Ma via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/9f076edb Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/9f076edb Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/9f076edb Branch: refs/heads/sqoop2 Commit: 9f076edb0265fda24705320e944511dd0fee772e Parents: b151ba3 Author: Jarek Jarcec Cecho <[email protected]> Authored: Tue Aug 25 09:07:19 2015 -0700 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Tue Aug 25 09:07:19 2015 -0700 ---------------------------------------------------------------------- .../mapreduce/MapreduceExecutionEngine.java | 1 + .../sqoop/job/mr/MRConfigurationUtils.java | 33 +++++++++++--------- .../apache/sqoop/job/mr/SqoopInputFormat.java | 3 +- .../org/apache/sqoop/job/mr/SqoopMapper.java | 2 +- .../sqoop/job/mr/SqoopNullOutputFormat.java | 2 +- .../org/apache/sqoop/job/mr/SqoopSplit.java | 6 ++++ 6 files changed, 29 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java index 3f79325..c8d210e 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/execution/mapreduce/MapreduceExecutionEngine.java @@ -46,6 +46,7 @@ public class MapreduceExecutionEngine extends ExecutionEngine { } public void prepareJob(JobRequest jobRequest) { + assert jobRequest instanceof MRJobRequest; MRJobRequest mrJobRequest = (MRJobRequest)jobRequest; // Add jar dependencies http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java index e07ddd7..1e1b237 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/MRConfigurationUtils.java @@ -31,7 +31,9 @@ import org.apache.sqoop.schema.Schema; import org.apache.sqoop.utils.ClassUtils; import org.json.simple.JSONObject; +import java.io.IOException; import java.io.InputStream; +import java.nio.charset.Charset; import java.util.Properties; /** @@ -89,12 +91,12 @@ public final class MRConfigurationUtils { switch (type) { case FROM: job.getConfiguration().set(MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK, obj.getClass().getName()); - job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes()); + job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8"))); break; case TO: job.getConfiguration().set(MR_JOB_CONFIG_CLASS_TO_CONNECTOR_LINK, obj.getClass().getName()); - job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes()); + job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_CONNECTOR_LINK_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8"))); break; } } @@ -109,12 +111,12 @@ public final class MRConfigurationUtils { switch (type) { case FROM: job.getConfiguration().set(MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, obj.getClass().getName()); - job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes()); + job.getCredentials().addSecretKey(MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8"))); break; case TO: job.getConfiguration().set(MR_JOB_CONFIG_CLASS_TO_CONNECTOR_JOB, obj.getClass().getName()); - job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes()); + job.getCredentials().addSecretKey(MR_JOB_CONFIG_TO_JOB_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8"))); break; } } @@ -128,7 +130,7 @@ public final class MRConfigurationUtils { */ public static void setDriverConfig(Job job, Object obj) { job.getConfiguration().set(MR_JOB_CONFIG_DRIVER_CONFIG_CLASS, obj.getClass().getName()); - job.getCredentials().addSecretKey(MR_JOB_CONFIG_DRIVER_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes()); + job.getCredentials().addSecretKey(MR_JOB_CONFIG_DRIVER_CONFIG_KEY, ConfigUtils.toJson(obj).getBytes(Charset.forName("UTF-8"))); } /** @@ -142,10 +144,10 @@ public final class MRConfigurationUtils { String jsonSchema = SchemaSerialization.extractSchema(schema).toJSONString(); switch (type) { case FROM: - job.getCredentials().addSecretKey(SCHEMA_FROM_KEY,jsonSchema.getBytes()); + job.getCredentials().addSecretKey(SCHEMA_FROM_KEY,jsonSchema.getBytes(Charset.forName("UTF-8"))); return; case TO: - job.getCredentials().addSecretKey(SCHEMA_TO_KEY, jsonSchema.getBytes()); + job.getCredentials().addSecretKey(SCHEMA_TO_KEY, jsonSchema.getBytes(Charset.forName("UTF-8"))); return; } } @@ -156,6 +158,7 @@ public final class MRConfigurationUtils { * @return Configuration object */ public static Object getConnectorLinkConfig(Direction type, Configuration configuration) { + assert configuration instanceof JobConf; switch (type) { case FROM: return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_LINK, MR_JOB_CONFIG_FROM_CONNECTOR_LINK_KEY); @@ -174,6 +177,7 @@ public final class MRConfigurationUtils { * @return Configuration object */ public static Object getConnectorJobConfig(Direction type, Configuration configuration) { + assert configuration instanceof JobConf; switch (type) { case FROM: return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_CLASS_FROM_CONNECTOR_JOB, MR_JOB_CONFIG_FROM_JOB_CONFIG_KEY); @@ -192,6 +196,7 @@ public final class MRConfigurationUtils { * @return Configuration object */ public static Object getDriverConfig(Configuration configuration) { + assert configuration instanceof JobConf; return loadConfiguration((JobConf) configuration, MR_JOB_CONFIG_DRIVER_CONFIG_CLASS, MR_JOB_CONFIG_DRIVER_CONFIG_KEY); } @@ -204,6 +209,7 @@ public final class MRConfigurationUtils { * @param configuration MapReduce configuration object */ public static Schema getConnectorSchema(Direction type, Configuration configuration) { + assert configuration instanceof JobConf; switch (type) { case FROM: return getSchemaFromBytes(((JobConf) configuration).getCredentials().getSecretKey(SCHEMA_FROM_KEY)); @@ -228,7 +234,7 @@ public final class MRConfigurationUtils { return null; } - JSONObject jsonSchema = JSONUtils.parse(new String(bytes)); + JSONObject jsonSchema = JSONUtils.parse(new String(bytes, Charset.forName("UTF-8"))); return SchemaSerialization.restoreSchema(jsonSchema); } @@ -248,7 +254,7 @@ public final class MRConfigurationUtils { return null; } - String json = new String(configuration.getCredentials().getSecretKey(valueProperty)); + String json = new String(configuration.getCredentials().getSecretKey(valueProperty), Charset.forName("UTF-8")); // Fill it with JSON data ConfigUtils.fillValues(json, object); @@ -262,15 +268,14 @@ public final class MRConfigurationUtils { } public static void configureLogging() { - try { + try (InputStream resourceAsStream = + SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties")) { Properties props = new Properties(); - InputStream resourceAsStream = - SqoopMapper.class.getResourceAsStream("/META-INF/log4j.properties"); props.load(resourceAsStream); PropertyConfigurator.configure(props); - } catch (Exception e) { + } catch (RuntimeException | IOException e) { System.err.println("Encountered exception while configuring logging " + - "for sqoop: " + e); + "for sqoop: " + e); } } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java index 5aef878..67189a1 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopInputFormat.java @@ -75,8 +75,7 @@ public class SqoopInputFormat extends InputFormat<SqoopSplit, NullWritable> { List<InputSplit> splits = new LinkedList<InputSplit>(); for (Partition partition : partitions) { LOG.debug("Partition: " + partition); - SqoopSplit split = new SqoopSplit(); - split.setPartition(partition); + SqoopSplit split = new SqoopSplit(partition); splits.add(split); } http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java index 14fdfdc..937ef5a 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopMapper.java @@ -149,7 +149,7 @@ public class SqoopMapper extends Mapper<SqoopSplit, NullWritable, SqoopWritable, toIDF.setObjectData(matcher.getMatchingData(fromIDF.getObjectData())); // NOTE: We do not use the reducer to do the writing (a.k.a LOAD in ETL). Hence the mapper sets up the writable context.write(writable, NullWritable.get()); - } catch (Exception e) { + } catch (RuntimeException | IOException | InterruptedException e) { throw new SqoopException(MRExecutionError.MAPRED_EXEC_0013, e); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java index 6134106..88ab98e 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopNullOutputFormat.java @@ -55,7 +55,7 @@ public class SqoopNullOutputFormat extends OutputFormat<SqoopWritable, NullWrita return new SqoopDestroyerOutputCommitter(); } - class SqoopDestroyerOutputCommitter extends OutputCommitter { + private static class SqoopDestroyerOutputCommitter extends OutputCommitter { @Override public void setupJob(JobContext jobContext) { } http://git-wip-us.apache.org/repos/asf/sqoop/blob/9f076edb/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java ---------------------------------------------------------------------- diff --git a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java index b81f4d6..bb6b2b0 100644 --- a/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java +++ b/execution/mapreduce/src/main/java/org/apache/sqoop/job/mr/SqoopSplit.java @@ -35,6 +35,12 @@ public class SqoopSplit extends InputSplit implements Writable { private Partition partition; + public SqoopSplit() {} + + public SqoopSplit(Partition partition) { + this.partition = partition; + } + public void setPartition(Partition partition) { this.partition = partition; }
