Updated Branches: refs/heads/master fd957a920 -> 3f9841136
CRUNCH-249: Fix HFileTargetIT failure under hadoop2 Project: http://git-wip-us.apache.org/repos/asf/crunch/repo Commit: http://git-wip-us.apache.org/repos/asf/crunch/commit/3f984113 Tree: http://git-wip-us.apache.org/repos/asf/crunch/tree/3f984113 Diff: http://git-wip-us.apache.org/repos/asf/crunch/diff/3f984113 Branch: refs/heads/master Commit: 3f98411364cec32a0a8c6681dfaabd43caa4dd60 Parents: fd957a9 Author: Chao Shi <[email protected]> Authored: Mon Aug 12 11:35:23 2013 +0800 Committer: Chao Shi <[email protected]> Committed: Mon Aug 12 11:35:23 2013 +0800 ---------------------------------------------------------------------- .../apache/crunch/io/hbase/HFileTargetIT.java | 36 ++++++++++++++++++++ pom.xml | 4 +-- 2 files changed, 38 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/crunch/blob/3f984113/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java ---------------------------------------------------------------------- diff --git a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java index 86d965b..8c1b3f4 100644 --- a/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java +++ b/crunch-hbase/src/it/java/org/apache/crunch/io/hbase/HFileTargetIT.java @@ -53,6 +53,8 @@ import org.apache.hadoop.hbase.regionserver.KeyValueScanner; import org.apache.hadoop.hbase.regionserver.StoreFile; import org.apache.hadoop.hbase.regionserver.StoreFileScanner; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.mapred.JobConf; +import org.apache.hadoop.mapred.MiniMRCluster; import org.junit.AfterClass; import org.junit.Before; import org.junit.BeforeClass; @@ -63,11 +65,13 @@ import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; +import java.lang.reflect.Field; import java.util.List; import java.util.Map; import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; public class HFileTargetIT implements Serializable { @@ -96,6 +100,38 @@ public class HFileTargetIT implements Serializable { splits[i] = new byte[] { b }; } admin.createTable(htable, splits); + + // Set classpath for yarn, otherwise it won't be able to find MRAppMaster + // (see CRUNCH-249 and HBASE-8528). + HBASE_TEST_UTILITY.getConfiguration().setBoolean("yarn.is.minicluster", true); + dirtyFixForJobHistoryServerAddress(); + } + + /** + * We need to set the address of JobHistory server, as it randomly picks a unused port + * to listen. Unfortunately, HBaseTestingUtility neither does that nor provides a way + * for us to know the picked address. We have to access it using reflection. + * + * This is necessary when testing with MRv2, but does no harm to MRv1. + */ + private static void dirtyFixForJobHistoryServerAddress() { + try { + // Retrieve HBASE_TEST_UTILITY.mrCluster via reflection, as it is private. + Field mrClusterField = HBaseTestingUtility.class.getDeclaredField("mrCluster"); + mrClusterField.setAccessible(true); + MiniMRCluster mrCluster = (MiniMRCluster) mrClusterField.get(HBASE_TEST_UTILITY); + JobConf jobConf = mrCluster.createJobConf(); + Configuration conf = HBASE_TEST_UTILITY.getConfiguration(); + String proprety = "mapreduce.jobhistory.address"; + String value = jobConf.get(proprety); + if (value != null) { // maybe null if we're running MRv1 + conf.set(proprety, value); + } + } catch (IllegalAccessException e) { + throw new AssertionError(e); + } catch (NoSuchFieldException e) { + throw new AssertionError(e); + } } @AfterClass http://git-wip-us.apache.org/repos/asf/crunch/blob/3f984113/pom.xml ---------------------------------------------------------------------- diff --git a/pom.xml b/pom.xml index 369041f..e972ce2 100644 --- a/pom.xml +++ b/pom.xml @@ -433,10 +433,10 @@ under the License. </activation> <properties> <hadoop.version>2.0.4-alpha</hadoop.version> - <!-- NOTE: You must build HBase 0.94.1 from source using: + <!-- NOTE: You must build HBase 0.94.3 from source using: mvn clean install -Dhadoop.profile=2.0 for this to work. --> - <hbase.version>0.94.1</hbase.version> + <hbase.version>0.94.3</hbase.version> <commons-lang.version>2.5</commons-lang.version> <slf4j.version>1.6.1</slf4j.version> </properties>
