Repository: sqoop Updated Branches: refs/heads/trunk e0fc46e94 -> 4ee8b6843
SQOOP-1392: Create the temporary directory inside task working dir rather then in tmp (Richard via Jarek Jarcec Cecho) Project: http://git-wip-us.apache.org/repos/asf/sqoop/repo Commit: http://git-wip-us.apache.org/repos/asf/sqoop/commit/4ee8b684 Tree: http://git-wip-us.apache.org/repos/asf/sqoop/tree/4ee8b684 Diff: http://git-wip-us.apache.org/repos/asf/sqoop/diff/4ee8b684 Branch: refs/heads/trunk Commit: 4ee8b6843a1bec0cb6ad7a50c9219b12eae5f35e Parents: e0fc46e Author: Jarek Jarcec Cecho <[email protected]> Authored: Mon Sep 1 10:52:15 2014 +0200 Committer: Jarek Jarcec Cecho <[email protected]> Committed: Mon Sep 1 10:52:15 2014 +0200 ---------------------------------------------------------------------- src/java/com/cloudera/sqoop/lib/LargeObjectLoader.java | 5 ++--- src/java/org/apache/sqoop/lib/LargeObjectLoader.java | 5 ++--- src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java | 3 +-- src/java/org/apache/sqoop/mapreduce/HBaseBulkImportMapper.java | 5 +---- src/java/org/apache/sqoop/mapreduce/ParquetImportMapper.java | 4 +--- .../org/apache/sqoop/mapreduce/SequenceFileImportMapper.java | 3 +-- src/java/org/apache/sqoop/mapreduce/TextImportMapper.java | 3 +-- .../org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java | 3 +-- src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java | 4 ++-- 9 files changed, 12 insertions(+), 23 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/java/com/cloudera/sqoop/lib/LargeObjectLoader.java ---------------------------------------------------------------------- diff --git a/src/java/com/cloudera/sqoop/lib/LargeObjectLoader.java b/src/java/com/cloudera/sqoop/lib/LargeObjectLoader.java index 43df7fc..8966579 100644 --- a/src/java/com/cloudera/sqoop/lib/LargeObjectLoader.java +++ b/src/java/com/cloudera/sqoop/lib/LargeObjectLoader.java @@ -47,10 +47,9 @@ public class LargeObjectLoader extends org.apache.sqoop.lib.LargeObjectLoader { /** * Create a new LargeObjectLoader. * @param conf the Configuration to use - * @param workPath the HDFS working directory for this task. */ - public LargeObjectLoader(Configuration conf, Path workPath) + public LargeObjectLoader(Configuration conf) throws IOException { - super(conf, workPath); + super(conf); } } http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/java/org/apache/sqoop/lib/LargeObjectLoader.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/lib/LargeObjectLoader.java b/src/java/org/apache/sqoop/lib/LargeObjectLoader.java index bc51277..a0f94d2 100644 --- a/src/java/org/apache/sqoop/lib/LargeObjectLoader.java +++ b/src/java/org/apache/sqoop/lib/LargeObjectLoader.java @@ -69,12 +69,11 @@ public class LargeObjectLoader implements Closeable { /** * Create a new LargeObjectLoader. * @param conf the Configuration to use - * @param workPath the HDFS working directory for this task. */ - public LargeObjectLoader(Configuration conf, Path workPath) + public LargeObjectLoader(Configuration conf) throws IOException { this.conf = conf; - this.workPath = workPath; + this.workPath = new Path(System.getProperty("java.io.tmpdir"), "SQOOP"); this.fs = FileSystem.get(conf); this.curBlobWriter = null; this.curClobWriter = null; http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java b/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java index 6fc656f..6adad79 100644 --- a/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/AvroImportMapper.java @@ -52,8 +52,7 @@ public class AvroImportMapper throws IOException, InterruptedException { Configuration conf = context.getConfiguration(); schema = AvroJob.getMapOutputSchema(conf); - lobLoader = new LargeObjectLoader(conf, - FileOutputFormat.getWorkOutputPath(context)); + lobLoader = new LargeObjectLoader(conf); bigDecimalFormatString = conf.getBoolean( ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT, ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT); http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/java/org/apache/sqoop/mapreduce/HBaseBulkImportMapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/HBaseBulkImportMapper.java b/src/java/org/apache/sqoop/mapreduce/HBaseBulkImportMapper.java index 9c9d6cd..10891e9 100644 --- a/src/java/org/apache/sqoop/mapreduce/HBaseBulkImportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/HBaseBulkImportMapper.java @@ -54,10 +54,7 @@ public class HBaseBulkImportMapper protected void setup(Context context) throws IOException, InterruptedException { this.conf = context.getConfiguration(); - Path largeFilePath = new Path(this.conf.get("sqoop.hbase.lob.extern.dir", - "/tmp/sqoop-hbase-" + context.getTaskAttemptID())); - this.lobLoader = new LargeObjectLoader(context.getConfiguration(), - largeFilePath); + this.lobLoader = new LargeObjectLoader(this.conf); // Get the implementation of PutTransformer to use. // By default, we call toString() on every non-null field. http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/java/org/apache/sqoop/mapreduce/ParquetImportMapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/ParquetImportMapper.java b/src/java/org/apache/sqoop/mapreduce/ParquetImportMapper.java index cc2982c..effbadd 100644 --- a/src/java/org/apache/sqoop/mapreduce/ParquetImportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/ParquetImportMapper.java @@ -42,9 +42,7 @@ public class ParquetImportMapper @Override protected void setup(Context context) throws IOException, InterruptedException { - Configuration conf = context.getConfiguration(); - Path workPath = new Path("/tmp/sqoop-parquet-" + context.getTaskAttemptID()); - lobLoader = new LargeObjectLoader(conf, workPath); + lobLoader = new LargeObjectLoader(context.getConfiguration()); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/java/org/apache/sqoop/mapreduce/SequenceFileImportMapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/SequenceFileImportMapper.java b/src/java/org/apache/sqoop/mapreduce/SequenceFileImportMapper.java index 528eeec..fde5d66 100644 --- a/src/java/org/apache/sqoop/mapreduce/SequenceFileImportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/SequenceFileImportMapper.java @@ -38,8 +38,7 @@ public class SequenceFileImportMapper @Override protected void setup(Context context) throws IOException, InterruptedException { - this.lobLoader = new LargeObjectLoader(context.getConfiguration(), - FileOutputFormat.getWorkOutputPath(context)); + this.lobLoader = new LargeObjectLoader(context.getConfiguration()); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/java/org/apache/sqoop/mapreduce/TextImportMapper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/TextImportMapper.java b/src/java/org/apache/sqoop/mapreduce/TextImportMapper.java index a63faed..f0bca93 100644 --- a/src/java/org/apache/sqoop/mapreduce/TextImportMapper.java +++ b/src/java/org/apache/sqoop/mapreduce/TextImportMapper.java @@ -45,8 +45,7 @@ public class TextImportMapper @Override protected void setup(Context context) throws IOException, InterruptedException { - this.lobLoader = new LargeObjectLoader(context.getConfiguration(), - FileOutputFormat.getWorkOutputPath(context)); + this.lobLoader = new LargeObjectLoader(context.getConfiguration()); } @Override http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java ---------------------------------------------------------------------- diff --git a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java index c2fc2e5..598483d 100644 --- a/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java +++ b/src/java/org/apache/sqoop/mapreduce/hcat/SqoopHCatImportHelper.java @@ -112,8 +112,7 @@ public class SqoopHCatImportHelper { hCatFullTableSchema.append(hfs); } fieldCount = hCatFullTableSchema.size(); - lobLoader = new LargeObjectLoader(conf, new Path(jobInfo.getTableInfo() - .getTableLocation())); + lobLoader = new LargeObjectLoader(conf); bigDecimalFormatString = conf.getBoolean( ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT, ImportJobBase.PROPERTY_BIGDECIMAL_FORMAT_DEFAULT); http://git-wip-us.apache.org/repos/asf/sqoop/blob/4ee8b684/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java ---------------------------------------------------------------------- diff --git a/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java b/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java index 7dada96..c6bebf7 100644 --- a/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java +++ b/src/test/com/cloudera/sqoop/lib/TestLargeObjectLoader.java @@ -49,14 +49,14 @@ public class TestLargeObjectLoader extends TestCase { conf.set(CommonArgs.FS_DEFAULT_NAME, CommonArgs.LOCAL_FS); } String tmpDir = System.getProperty("test.build.data", "/tmp/"); - this.outDir = new Path(new Path(tmpDir), "testLobLoader"); + this.outDir = new Path(System.getProperty("java.io.tmpdir")); FileSystem fs = FileSystem.get(conf); if (fs.exists(outDir)) { fs.delete(outDir, true); } fs.mkdirs(outDir); - loader = new LargeObjectLoader(conf, outDir); + loader = new LargeObjectLoader(conf); } public void testReadClobRef()
