Repository: hbase Updated Branches: refs/heads/master 652929c0f -> a93ba89b5
HBASE-13625 Use HDFS for HFileOutputFormat2 partitioner's path (Stephen Yuan Jiang) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/a93ba89b Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/a93ba89b Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/a93ba89b Branch: refs/heads/master Commit: a93ba89b5b0ab2d34224a9958e0aea72d5b71934 Parents: 652929c Author: tedyu <[email protected]> Authored: Wed May 6 07:31:15 2015 -0700 Committer: tedyu <[email protected]> Committed: Wed May 6 07:31:15 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/security/SecureBulkLoadUtil.java | 2 +- hbase-common/src/main/resources/hbase-default.xml | 14 ++++++++++++++ .../hadoop/hbase/mapreduce/HFileOutputFormat2.java | 2 +- .../hadoop/hbase/mapreduce/TestHFileOutputFormat.java | 5 ++++- .../hbase/mapreduce/TestHFileOutputFormat2.java | 5 ++++- 5 files changed, 24 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/a93ba89b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java index 30959a0..3dca0a3 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java @@ -37,6 +37,6 @@ public class SecureBulkLoadUtil { } public static Path getBaseStagingDir(Configuration conf) { - return new Path(conf.get(BULKLOAD_STAGING_DIR, "/tmp/hbase-staging")); + return new Path(conf.get(BULKLOAD_STAGING_DIR)); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/a93ba89b/hbase-common/src/main/resources/hbase-default.xml ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index f107fb7..a5b12d3 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -63,6 +63,20 @@ possible configurations would overwhelm and obscure the important. machine restart.</description> </property> <property > + <name>hbase.fs.tmp.dir</name> + <value>/user/${user.name}/hbase-staging</value> + <description>A staging directory in default file system (HDFS) + for keeping temporary data. + </description> + </property> + <property > + <name>hbase.bulkload.staging.dir</name> + <value>${hbase.fs.tmp.dir}</value> + <description>A staging directory in default file system (HDFS) + for bulk loading. + </description> + </property> + <property > <name>hbase.cluster.distributed</name> <value>false</value> <description>The mode the cluster will be in. Possible values are http://git-wip-us.apache.org/repos/asf/hbase/blob/a93ba89b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index c96a2bf..4e94308 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -588,7 +588,7 @@ public class HFileOutputFormat2 Configuration conf = job.getConfiguration(); // create the partitions file FileSystem fs = FileSystem.get(conf); - Path partitionsPath = new Path(conf.get("hadoop.tmp.dir"), "partitions_" + UUID.randomUUID()); + Path partitionsPath = new Path(conf.get("hbase.fs.tmp.dir"), "partitions_" + UUID.randomUUID()); fs.makeQualified(partitionsPath); writePartitions(conf, partitionsPath, splitPoints); fs.deleteOnExit(partitionsPath); http://git-wip-us.apache.org/repos/asf/hbase/blob/a93ba89b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java index f4ccc4e..314b7b2 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java @@ -335,7 +335,9 @@ public class TestHFileOutputFormat { @Test public void testJobConfiguration() throws Exception { - Job job = new Job(util.getConfiguration()); + Configuration conf = new Configuration(this.util.getConfiguration()); + conf.set("hbase.fs.tmp.dir", util.getDataTestDir("testJobConfiguration").toString()); + Job job = new Job(conf); job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration")); HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class); RegionLocator regionLocator = Mockito.mock(RegionLocator.class); @@ -820,6 +822,7 @@ public class TestHFileOutputFormat { // We turn off the sequence file compression, because DefaultCodec // pollutes the GZip codec pool with an incompatible compressor. conf.set("io.seqfile.compression.type", "NONE"); + conf.set("hbase.fs.tmp.dir", dir.toString()); Job job = new Job(conf, "testLocalMRIncrementalLoad"); job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings")); setupRandomGeneratorMapper(job); http://git-wip-us.apache.org/repos/asf/hbase/blob/a93ba89b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index 178eb57..7aa5dc4 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -337,7 +337,9 @@ public class TestHFileOutputFormat2 { @Test public void testJobConfiguration() throws Exception { - Job job = new Job(util.getConfiguration()); + Configuration conf = new Configuration(this.util.getConfiguration()); + conf.set("hbase.fs.tmp.dir", util.getDataTestDir("testJobConfiguration").toString()); + Job job = new Job(conf); job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration")); Table table = Mockito.mock(Table.class); RegionLocator regionLocator = Mockito.mock(RegionLocator.class); @@ -823,6 +825,7 @@ public class TestHFileOutputFormat2 { // We turn off the sequence file compression, because DefaultCodec // pollutes the GZip codec pool with an incompatible compressor. conf.set("io.seqfile.compression.type", "NONE"); + conf.set("hbase.fs.tmp.dir", dir.toString()); Job job = new Job(conf, "testLocalMRIncrementalLoad"); job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings")); setupRandomGeneratorMapper(job);
