Author: tedyu
Date: Fri Feb 24 15:57:01 2012
New Revision: 1293306
URL: http://svn.apache.org/viewvc?rev=1293306&view=rev
Log:
HBASE-5317 Fix TestHFileOutputFormat to work against hadoop 0.23
(Gregory Taylor)
Modified:
hbase/branches/0.92/CHANGES.txt
hbase/branches/0.92/pom.xml
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
Modified: hbase/branches/0.92/CHANGES.txt
URL:
http://svn.apache.org/viewvc/hbase/branches/0.92/CHANGES.txt?rev=1293306&r1=1293305&r2=1293306&view=diff
==============================================================================
--- hbase/branches/0.92/CHANGES.txt (original)
+++ hbase/branches/0.92/CHANGES.txt Fri Feb 24 15:57:01 2012
@@ -43,6 +43,8 @@ Release 0.92.1 - Unreleased
hbase.root.dir better
HBASE-5466 Opening a table also opens the metatable and never closes it
(Ashley Taylor)
+ HBASE-5317 Fix TestHFileOutputFormat to work against hadoop 0.23
+ (Gregory Taylor)
IMPROVEMENTS
HBASE-5197 [replication] Handle socket timeouts in ReplicationSource
Modified: hbase/branches/0.92/pom.xml
URL:
http://svn.apache.org/viewvc/hbase/branches/0.92/pom.xml?rev=1293306&r1=1293305&r2=1293306&view=diff
==============================================================================
--- hbase/branches/0.92/pom.xml (original)
+++ hbase/branches/0.92/pom.xml Fri Feb 24 15:57:01 2012
@@ -1728,6 +1728,25 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <artifactId>maven-dependency-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>create-mrapp-generated-classpath</id>
+ <phase>generate-test-resources</phase>
+ <goals>
+ <goal>build-classpath</goal>
+ </goals>
+ <configuration>
+ <!-- needed to run the unit test for DS to generate
+ the required classpath that is required in the env
+ of the launch container in the mini mr/yarn cluster
+ -->
+
<outputFile>${project.build.directory}/test-classes/mrapp-generated-classpath</outputFile>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
</profile>
Modified:
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java?rev=1293306&r1=1293305&r2=1293306&view=diff
==============================================================================
---
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
(original)
+++
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/HFileOutputFormat.java
Fri Feb 24 15:57:01 2012
@@ -333,7 +333,8 @@ public class HFileOutputFormat extends F
// Set compression algorithms based on column families
configureCompression(table, conf);
-
+
+ TableMapReduceUtil.addDependencyJars(job);
LOG.info("Incremental table output configured.");
}
Modified:
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java?rev=1293306&r1=1293305&r2=1293306&view=diff
==============================================================================
---
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
(original)
+++
hbase/branches/0.92/src/main/java/org/apache/hadoop/hbase/mapreduce/TableMapReduceUtil.java
Fri Feb 24 15:57:01 2012
@@ -24,6 +24,8 @@ import java.io.ByteArrayOutputStream;
import java.io.DataInputStream;
import java.io.DataOutputStream;
import java.io.IOException;
+import java.lang.reflect.Method;
+import java.lang.reflect.InvocationTargetException;
import java.net.URL;
import java.net.URLDecoder;
import java.util.Enumeration;
@@ -467,7 +469,7 @@ public class TableMapReduceUtil {
for (Class clazz : classes) {
if (clazz == null) continue;
- String pathStr = findContainingJar(clazz);
+ String pathStr = findOrCreateJar(clazz);
if (pathStr == null) {
LOG.warn("Could not find jar for class " + clazz +
" in order to ship it to the cluster.");
@@ -487,7 +489,46 @@ public class TableMapReduceUtil {
StringUtils.arrayToString(jars.toArray(new String[0])));
}
- /**
+ /**
+ * If org.apache.hadoop.util.JarFinder is available (0.23+ hadoop),
+ * finds the Jar for a class or creates it if it doesn't exist. If
+ * the class is in a directory in the classpath, it creates a Jar
+ * on the fly with the contents of the directory and returns the path
+ * to that Jar. If a Jar is created, it is created in
+ * the system temporary directory.
+ *
+ * Otherwise, returns an existing jar that contains a class of the
+ * same name.
+ *
+ * @param my_class the class to find.
+ * @return a jar file that contains the class, or null.
+ * @throws IOException
+ */
+ private static String findOrCreateJar(Class my_class)
+ throws IOException {
+ try {
+ Class<?> jarFinder = Class.forName("org.apache.hadoop.util.JarFinder");
+ // hadoop-0.23 has a JarFinder class that will create the jar
+ // if it doesn't exist. Note that this is needed to run the mapreduce
+ // unit tests post-0.23, because mapreduce v2 requires the relevant jars
+ // to be in the mr cluster to do output, split, etc. At unit test time,
+ // the hbase jars do not exist, so we need to create some. Note that we
+ // can safely fall back to findContainingJars for pre-0.23 mapreduce.
+ Method m = jarFinder.getMethod("getJar", Class.class);
+ return (String)m.invoke(null,my_class);
+ } catch (InvocationTargetException ite) {
+ // function was properly called, but threw it's own exception
+ throw new IOException(ite.getCause());
+ } catch (Exception e) {
+ // ignore all other exceptions. related to reflection failure
+ }
+
+ LOG.debug("New JarFinder: org.apache.hadoop.util.JarFinder.getJar " +
+ "not available. Using old findContainingJar");
+ return findContainingJar(my_class);
+}
+
+ /**
* Find a jar that contains a class of the same name, if any.
* It will return a jar file, even if that is not the first thing
* on the class path that has a class with the same name.
Modified:
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1293306&r1=1293305&r2=1293306&view=diff
==============================================================================
---
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
(original)
+++
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
Fri Feb 24 15:57:01 2012
@@ -601,6 +601,17 @@ public class HBaseTestingUtility {
}
/**
+ * Returns the path to the default root dir the minicluster uses.
+ * Note: this does not cause the root dir to be created.
+ * @return Fully qualified path for the default hbase root dir
+ * @throws IOException
+ */
+ public Path getDefaultRootDirPath() throws IOException {
+ FileSystem fs = FileSystem.get(this.conf);
+ return new Path(fs.makeQualified(fs.getHomeDirectory()),"hbase");
+ }
+
+ /**
* Creates an hbase rootdir in user home directory. Also creates hbase
* version file. Normally you won't make use of this method. Root hbasedir
* is created for you as part of mini cluster startup. You'd only use this
@@ -610,7 +621,7 @@ public class HBaseTestingUtility {
*/
public Path createRootDir() throws IOException {
FileSystem fs = FileSystem.get(this.conf);
- Path hbaseRootdir = fs.makeQualified(fs.getHomeDirectory());
+ Path hbaseRootdir = getDefaultRootDirPath();
this.conf.set(HConstants.HBASE_DIR, hbaseRootdir.toString());
fs.mkdirs(hbaseRootdir);
FSUtils.setVersion(fs, hbaseRootdir);
Modified:
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
URL:
http://svn.apache.org/viewvc/hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java?rev=1293306&r1=1293305&r2=1293306&view=diff
==============================================================================
---
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
(original)
+++
hbase/branches/0.92/src/test/java/org/apache/hadoop/hbase/client/TestMetaMigrationRemovingHTD.java
Fri Feb 24 15:57:01 2012
@@ -77,10 +77,10 @@ public class TestMetaMigrationRemovingHT
Configuration conf = TEST_UTIL.getConfiguration();
FsShell shell = new FsShell(conf);
FileSystem fs = FileSystem.get(conf);
- // Minihbase roots itself in user home directory up in minidfs.
- Path homedir = fs.getHomeDirectory();
+ // find where hbase will root itself, so we can copy filesystem there
+ Path hbaseRootDir = TEST_UTIL.getDefaultRootDirPath();
doFsCommand(shell,
- new String [] {"-put", untar.toURI().toString(), homedir.toString()});
+ new String [] {"-put", untar.toURI().toString(),
hbaseRootDir.toString()});
// See whats in minihdfs.
doFsCommand(shell, new String [] {"-lsr", "/"});
TEST_UTIL.startMiniHBaseCluster(1, 1);
