Repository: hbase Updated Branches: refs/heads/branch-1.1 5422fb87f -> 0a751589a
HBASE-13625 Use HDFS for HFileOutputFormat2 partitioner's path (Stephen Yuan Jiang) Project: http://git-wip-us.apache.org/repos/asf/hbase/repo Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/0a751589 Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/0a751589 Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/0a751589 Branch: refs/heads/branch-1.1 Commit: 0a751589a097243a097eca8f5127e610eec64e58 Parents: 5422fb8 Author: tedyu <[email protected]> Authored: Wed May 6 07:30:40 2015 -0700 Committer: tedyu <[email protected]> Committed: Wed May 6 07:30:40 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hbase/security/SecureBulkLoadUtil.java | 2 +- hbase-common/src/main/resources/hbase-default.xml | 14 ++++++++++++++ .../hadoop/hbase/mapreduce/HFileOutputFormat2.java | 2 +- .../hadoop/hbase/mapreduce/TestHFileOutputFormat.java | 5 ++++- .../hbase/mapreduce/TestHFileOutputFormat2.java | 5 ++++- 5 files changed, 24 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hbase/blob/0a751589/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java ---------------------------------------------------------------------- diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java index 2fde925..04bfbb5 100644 --- a/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java +++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/security/SecureBulkLoadUtil.java @@ -37,6 +37,6 @@ public class SecureBulkLoadUtil { } public static Path getBaseStagingDir(Configuration conf) { - return new Path(conf.get(BULKLOAD_STAGING_DIR, "/tmp/hbase-staging")); + return new Path(conf.get(BULKLOAD_STAGING_DIR)); } } http://git-wip-us.apache.org/repos/asf/hbase/blob/0a751589/hbase-common/src/main/resources/hbase-default.xml ---------------------------------------------------------------------- diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml index 69b1f8b..f773d21 100644 --- a/hbase-common/src/main/resources/hbase-default.xml +++ b/hbase-common/src/main/resources/hbase-default.xml @@ -63,6 +63,20 @@ possible configurations would overwhelm and obscure the important. machine restart.</description> </property> <property > + <name>hbase.fs.tmp.dir</name> + <value>/user/${user.name}/hbase-staging</value> + <description>A staging directory in default file system (HDFS) + for keeping temporary data. + </description> + </property> + <property > + <name>hbase.bulkload.staging.dir</name> + <value>${hbase.fs.tmp.dir}</value> + <description>A staging directory in default file system (HDFS) + for bulk loading. + </description> + </property> + <property > <name>hbase.cluster.distributed</name> <value>false</value> <description>The mode the cluster will be in. Possible values are http://git-wip-us.apache.org/repos/asf/hbase/blob/0a751589/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java index 465be56..5935b86 100644 --- a/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java +++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat2.java @@ -588,7 +588,7 @@ public class HFileOutputFormat2 Configuration conf = job.getConfiguration(); // create the partitions file FileSystem fs = FileSystem.get(conf); - Path partitionsPath = new Path(conf.get("hadoop.tmp.dir"), "partitions_" + UUID.randomUUID()); + Path partitionsPath = new Path(conf.get("hbase.fs.tmp.dir"), "partitions_" + UUID.randomUUID()); fs.makeQualified(partitionsPath); writePartitions(conf, partitionsPath, splitPoints); fs.deleteOnExit(partitionsPath); http://git-wip-us.apache.org/repos/asf/hbase/blob/0a751589/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java index 5a25ddd..895c09c 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat.java @@ -333,7 +333,9 @@ public class TestHFileOutputFormat { @Test public void testJobConfiguration() throws Exception { - Job job = new Job(util.getConfiguration()); + Configuration conf = new Configuration(this.util.getConfiguration()); + conf.set("hbase.fs.tmp.dir", util.getDataTestDir("testJobConfiguration").toString()); + Job job = new Job(conf); job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration")); HTableDescriptor tableDescriptor = Mockito.mock(HTableDescriptor.class); RegionLocator regionLocator = Mockito.mock(RegionLocator.class); @@ -820,6 +822,7 @@ public class TestHFileOutputFormat { // We turn off the sequence file compression, because DefaultCodec // pollutes the GZip codec pool with an incompatible compressor. conf.set("io.seqfile.compression.type", "NONE"); + conf.set("hbase.fs.tmp.dir", dir.toString()); Job job = new Job(conf, "testLocalMRIncrementalLoad"); job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings")); setupRandomGeneratorMapper(job); http://git-wip-us.apache.org/repos/asf/hbase/blob/0a751589/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java ---------------------------------------------------------------------- diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java index e3c30b1..2e4d016 100644 --- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java +++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapreduce/TestHFileOutputFormat2.java @@ -337,7 +337,9 @@ public class TestHFileOutputFormat2 { @Test public void testJobConfiguration() throws Exception { - Job job = new Job(util.getConfiguration()); + Configuration conf = new Configuration(this.util.getConfiguration()); + conf.set("hbase.fs.tmp.dir", util.getDataTestDir("testJobConfiguration").toString()); + Job job = new Job(conf); job.setWorkingDirectory(util.getDataTestDir("testJobConfiguration")); RegionLocator regionLocator = Mockito.mock(RegionLocator.class); setupMockStartKeys(regionLocator); @@ -822,6 +824,7 @@ public class TestHFileOutputFormat2 { // We turn off the sequence file compression, because DefaultCodec // pollutes the GZip codec pool with an incompatible compressor. conf.set("io.seqfile.compression.type", "NONE"); + conf.set("hbase.fs.tmp.dir", dir.toString()); Job job = new Job(conf, "testLocalMRIncrementalLoad"); job.setWorkingDirectory(util.getDataTestDirOnTestFS("testColumnFamilySettings")); setupRandomGeneratorMapper(job);
