This is an automated email from the ASF dual-hosted git repository. mwalch pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/accumulo.git
The following commit(s) were added to refs/heads/master by this push: new c265ea5 Remove 'accumulo-util hadoop-jar' command (#872) c265ea5 is described below commit c265ea5b16171032419b164e809f5478f70bbba8 Author: Mike Walch <mwa...@apache.org> AuthorDate: Sat Jan 5 17:54:17 2019 -0500 Remove 'accumulo-util hadoop-jar' command (#872) * Command doesn't work well now that Accumulo does not match Hadoop's dependenices * Users should create shaded jar instead and submit using Hadoop's 'yarn jar' command * Command was used by StandaloneClusterControl but only by unit test --- assemble/bin/accumulo-util | 69 ---------------------- .../standalone/StandaloneClusterControl.java | 37 +----------- .../standalone/StandaloneClusterControlTest.java | 49 --------------- 3 files changed, 2 insertions(+), 153 deletions(-) diff --git a/assemble/bin/accumulo-util b/assemble/bin/accumulo-util index d243cd4..13eb802 100755 --- a/assemble/bin/accumulo-util +++ b/assemble/bin/accumulo-util @@ -22,7 +22,6 @@ Usage: accumulo-util <command> (<argument> ...) Commands: build-native Builds Accumulo native libraries dump-zoo Dumps data in ZooKeeper - hadoop-jar Runs 'hadoop jar' command with Accumulo jars gen-monitor-cert Generates Accumulo monitor certficate load-jars-hdfs Loads Accumulo jars in lib/ to HDFS for VFS classloader @@ -182,71 +181,6 @@ function load_jars_hdfs() { "$HADOOP" fs -rm "$SYSTEM_CONTEXT_HDFS_DIR/slf4j*.jar" > /dev/null } -function hadoop_jar() { - if [[ -x "$HADOOP_HOME/bin/hadoop" ]]; then - HADOOP="$HADOOP_HOME/bin/hadoop" - else - HADOOP=$(which hadoop) - fi - if [[ ! -x "$HADOOP" ]]; then - echo "Could not find 'hadoop' command. Please set hadoop on your PATH or set HADOOP_HOME" - exit 1 - fi - if [[ -z "$ZOOKEEPER_HOME" ]]; then - echo "ZOOKEEPER_HOME must be set!" - exit 1 - fi - - ZOOKEEPER_CMD="ls -1 $ZOOKEEPER_HOME/zookeeper-[0-9]*[^csn].jar " - if [[ $(eval "$ZOOKEEPER_CMD" | wc -l) -ne 1 ]] ; then - echo "Not exactly one zookeeper jar in $ZOOKEEPER_HOME" - exit 1 - fi - ZOOKEEPER_LIB=$(eval "$ZOOKEEPER_CMD") - - CORE_LIB="${lib}/accumulo-core.jar" - THRIFT_LIB="${lib}/libthrift.jar" - JCOMMANDER_LIB="${lib}/jcommander.jar" - COMMONS_VFS_LIB="${lib}/commons-vfs2.jar" - GUAVA_LIB="${lib}/guava.jar" - HTRACE_LIB="${lib}/htrace-core.jar" - - USERJARS=" " - for arg in "$@"; do - if [[ "$arg" != "-libjars" ]] && [[ -z "$TOOLJAR" ]]; then - TOOLJAR="$arg" - shift - elif [[ "$arg" != "-libjars" ]] && [[ -z "$CLASSNAME" ]]; then - CLASSNAME="$arg" - shift - elif [[ -z "$USERJARS" ]]; then - USERJARS=$(echo "$arg" | tr "," " ") - shift - elif [[ "$arg" = "-libjars" ]]; then - USERJARS="" - shift - else - break - fi - done - - LIB_JARS="$THRIFT_LIB,$CORE_LIB,$ZOOKEEPER_LIB,$JCOMMANDER_LIB,$COMMONS_VFS_LIB,$GUAVA_LIB,$HTRACE_LIB" - H_JARS="$THRIFT_LIB:$CORE_LIB:$ZOOKEEPER_LIB:$JCOMMANDER_LIB:$COMMONS_VFS_LIB:$GUAVA_LIB:$HTRACE_LIB" - - for jar in $USERJARS; do - LIB_JARS="$LIB_JARS,$jar" - H_JARS="$H_JARS:$jar" - done - export HADOOP_CLASSPATH="$H_JARS:$HADOOP_CLASSPATH" - - if [[ -z "$CLASSNAME" || -z "$TOOLJAR" ]]; then - echo "Usage: accumulo-util hadoop-jar path/to/myTool.jar my.tool.class.Name [-libjars my1.jar,my2.jar]" 1>&2 - exit 1 - fi - - exec "$HADOOP" jar "$TOOLJAR" "$CLASSNAME" -libjars "$LIB_JARS" "$@" -} - function main() { SOURCE="${BASH_SOURCE[0]}" while [ -h "${SOURCE}" ]; do @@ -266,9 +200,6 @@ function main() { dump-zoo) "$bin"/accumulo org.apache.accumulo.server.util.DumpZookeeper "${@:2}" ;; - hadoop-jar) - hadoop_jar "${@:2}" - ;; gen-monitor-cert) gen_monitor_cert ;; diff --git a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java index c374e86..13c33f7 100644 --- a/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java +++ b/minicluster/src/main/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControl.java @@ -53,7 +53,7 @@ public class StandaloneClusterControl implements ClusterControl { private static final Logger log = LoggerFactory.getLogger(StandaloneClusterControl.class); private static final String ACCUMULO_SERVICE_SCRIPT = "accumulo-service", - ACCUMULO_SCRIPT = "accumulo", ACCUMULO_UTIL_SCRIPT = "accumulo-util"; + ACCUMULO_SCRIPT = "accumulo"; private static final String MASTER_HOSTS_FILE = "masters", GC_HOSTS_FILE = "gc", TSERVER_HOSTS_FILE = "tservers", TRACER_HOSTS_FILE = "tracers", MONITOR_HOSTS_FILE = "monitor"; @@ -65,7 +65,7 @@ public class StandaloneClusterControl implements ClusterControl { private String serverCmdPrefix; protected RemoteShellOptions options; - protected String accumuloServicePath, accumuloPath, accumuloUtilPath; + protected String accumuloServicePath, accumuloPath; @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "code runs in same security context as user who provided input file name") @@ -81,11 +81,6 @@ public class StandaloneClusterControl implements ClusterControl { File bin = new File(accumuloHome, "bin"); this.accumuloServicePath = new File(bin, ACCUMULO_SERVICE_SCRIPT).getAbsolutePath(); this.accumuloPath = new File(bin, ACCUMULO_SCRIPT).getAbsolutePath(); - this.accumuloUtilPath = new File(bin, ACCUMULO_UTIL_SCRIPT).getAbsolutePath(); - } - - String getAccumuloUtilPath() { - return this.accumuloUtilPath; } protected Entry<Integer,String> exec(String hostname, String[] command) throws IOException { @@ -133,34 +128,6 @@ public class StandaloneClusterControl implements ClusterControl { return msg.replaceAll("[\r\n]", ""); } - public Entry<Integer,String> execMapreduceWithStdout(Class<?> clz, String[] args) - throws IOException { - String host = "localhost"; - List<String> cmd = new ArrayList<>(); - cmd.add(getAccumuloUtilPath()); - cmd.add("hadoop-jar"); - cmd.add(getJarFromClass(clz)); - cmd.add(clz.getName()); - for (String arg : args) { - cmd.add("'" + arg + "'"); - } - log.info("Running: '{}' on {}", StringUtils.join(cmd, " "), host); - return exec(host, cmd.toArray(new String[cmd.size()])); - } - - String getJarFromClass(Class<?> clz) { - CodeSource source = clz.getProtectionDomain().getCodeSource(); - if (source == null) { - throw new RuntimeException("Could not get CodeSource for class"); - } - URL jarUrl = source.getLocation(); - String jar = jarUrl.getPath(); - if (!jar.endsWith(".jar")) { - throw new RuntimeException("Need to have a jar to run mapreduce: " + jar); - } - return jar; - } - @Override public void adminStopAll() throws IOException { String master = getHosts(MASTER_HOSTS_FILE).get(0); diff --git a/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java b/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java index 1c31f89..5e23b83 100644 --- a/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java +++ b/minicluster/src/test/java/org/apache/accumulo/cluster/standalone/StandaloneClusterControlTest.java @@ -16,19 +16,10 @@ */ package org.apache.accumulo.cluster.standalone; -import static org.easymock.EasyMock.anyObject; -import static org.easymock.EasyMock.aryEq; -import static org.easymock.EasyMock.eq; -import static org.easymock.EasyMock.expect; -import static org.easymock.EasyMock.replay; -import static org.easymock.EasyMock.verify; import static org.junit.Assert.assertEquals; -import org.easymock.EasyMock; import org.junit.Test; -import com.google.common.collect.Maps; - public class StandaloneClusterControlTest { @Test @@ -46,44 +37,4 @@ public class StandaloneClusterControlTest { assertEquals(accumuloHome + "/bin/accumulo", control.accumuloPath); assertEquals(accumuloHome + "/bin/accumulo-service", control.accumuloServicePath); } - - @Test - public void mapreduceLaunchesLocally() throws Exception { - final String accumuloUtilPath = "/usr/lib/accumulo/bin/accumulo-util"; - final String jar = "/home/user/my_project.jar"; - final Class<?> clz = Object.class; - final String myClass = clz.getName(); - StandaloneClusterControl control = EasyMock.createMockBuilder(StandaloneClusterControl.class) - .addMockedMethod("exec", String.class, String[].class) - .addMockedMethod("getAccumuloUtilPath").addMockedMethod("getJarFromClass", Class.class) - .createMock(); - - final String[] toolArgs = {"-u", "user", "-p", "password"}; - final String[] expectedCommands = new String[4 + toolArgs.length]; - - int i = 0; - expectedCommands[i++] = accumuloUtilPath; - expectedCommands[i++] = "hadoop-jar"; - expectedCommands[i++] = jar; - expectedCommands[i++] = myClass; - for (int j = 0; j < toolArgs.length; j++) { - expectedCommands[i + j] = quote(toolArgs[j]); - } - - expect(control.getAccumuloUtilPath()).andReturn(accumuloUtilPath); - expect(control.getJarFromClass(anyObject(Class.class))).andReturn(jar); - expect(control.exec(eq("localhost"), aryEq(expectedCommands))) - .andReturn(Maps.immutableEntry(0, "")); - - replay(control); - - // Give a fake Class -- we aren't verifying the actual class passed in - control.execMapreduceWithStdout(clz, toolArgs); - - verify(control); - } - - private String quote(String word) { - return "'" + word + "'"; - } }